From 77a8f8db4067271bac10e278cb8576da71dd9558 Mon Sep 17 00:00:00 2001 From: Eric Forgy Date: Wed, 14 Feb 2018 01:47:31 +0800 Subject: [PATCH 01/35] Use HTTP.jl (#87) * Use HTTP.jl * Support SSLContext --- REQUIRE | 8 +- src/WebSockets.jl | 334 +++++++++++++++++++++-------------- test/runtests.jl | 436 ++++++++++++++++++++++++++-------------------- 3 files changed, 454 insertions(+), 324 deletions(-) diff --git a/REQUIRE b/REQUIRE index 3f44968..29b8b53 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,6 +1,2 @@ -julia 0.5 -Compat 0.28.0 -HttpCommon -HttpServer -Codecs -MbedTLS +julia 0.6 +HTTP \ No newline at end of file diff --git a/src/WebSockets.jl b/src/WebSockets.jl index f7ccc08..95ac3f1 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -20,14 +20,11 @@ an http server. """ module WebSockets -using HttpCommon -using HttpServer -using Codecs -using MbedTLS -using Compat; import Compat.String +using HTTP +import HTTP: digest, MD_SHA1 + export WebSocket, - WebSocketHandler, write, read, close, @@ -42,23 +39,29 @@ const TCPSock = Base.TCPSocket init_socket(sock) = Base.buffer_writes(sock) -type WebSocketClosedError <: Exception end +struct WebSocketClosedError <: Exception end Base.showerror(io::IO, e::WebSocketClosedError) = print(io, "Error: client disconnected") +struct WebSocketError <: Exception + status::Int16 + message::String +end + """ A WebSocket is a wrapper over a TcpSocket. It takes care of wrapping outgoing data in a frame and unwrapping (and concatenating) incoming data. """ -type WebSocket - id::Int - socket::TCPSock +mutable struct WebSocket{T <: IO} <: IO + socket::T + server::Bool state::ReadyState - function WebSocket(id::Int,socket::TCPSock) + function WebSocket{T}(socket::T,server::Bool) where T init_socket(socket) - new(id, socket, CONNECTED) + new(socket, server, CONNECTED) end end +WebSocket(socket,server) = WebSocket{typeof(socket)}(socket,server) # WebSocket Frames # @@ -131,50 +134,36 @@ function addsubproto(name) return true end """ - write_fragment(io, islast, data::Array{UInt8}, opcode) + write_fragment(io, islast, opcode, hasmask, data::Array{UInt8}) Write the raw frame to a bufffer """ -function write_fragment(io::IO, islast::Bool, data::Array{UInt8}, opcode) +function write_fragment(io::IO, islast::Bool, opcode, hasmask::Bool, data::Array{UInt8}) l = length(data) b1::UInt8 = (islast ? 0b1000_0000 : 0b0000_0000) | opcode - # TODO: Do the mask xor thing?? - # 1. set bit 8 to 1, - # 2. set a mask - # 3. xor data with mask + mask::UInt8 = hasmask ? 0b1000_0000 : 0b0000_0000 + write(io, b1) if l <= 125 - write(io, b1) - write(io, @compat UInt8(l)) - write(io, data) + write(io, mask | UInt8(l)) elseif l <= typemax(UInt16) - write(io, b1) - write(io, @compat UInt8(126)) - write(io, hton(@compat UInt16(l))) - write(io, data) + write(io, mask | UInt8(126)) + write(io, hton(UInt16(l))) elseif l <= typemax(UInt64) - write(io, b1) - write(io, @compat UInt8(127)) - write(io, hton(@compat UInt64(l))) - write(io, data) + write(io, mask | UInt8(127)) + write(io, hton(UInt64(l))) else error("Attempted to send too much data for one websocket fragment\n") end -end - -""" - write_fragment(io, islast, data::String, opcode) -A version of send_fragment for text data. -""" -function write_fragment(io::IO, islast::Bool, data::String, opcode) - write_fragment(io, islast, data.data, opcode) + hasmask && write(io,mask!(data)) + write(io, data) end """ Write without interruptions""" -function locked_write(io::IO, islast::Bool, data, opcode) +function locked_write(io::IO, islast::Bool, opcode, hasmask, data) isa(io, TCPSock) && lock(io.lock) try - write_fragment(io, islast, Vector{UInt8}(data), opcode) + write_fragment(io, islast, opcode, hasmask, Vector{UInt8}(data)) finally if isa(io, TCPSock) flush(io) @@ -189,7 +178,7 @@ function Base.write(ws::WebSocket,data::String) @show ws error("Attempted write to closed WebSocket\n") end - locked_write(ws.socket, true, data, OPCODE_TEXT) + locked_write(ws.socket, true, OPCODE_TEXT, !ws.server, data) end """ Write binary data; will be sent as one frame.""" @@ -198,22 +187,22 @@ function Base.write(ws::WebSocket, data::Array{UInt8}) @show ws error("attempt to write to closed WebSocket\n") end - locked_write(ws.socket, true, data, OPCODE_BINARY) + locked_write(ws.socket, true, OPCODE_BINARY, !ws.server, data) end -function write_ping(io::IO, data = "") - locked_write(io, true, data, OPCODE_PING) +function write_ping(io::IO, hasmask, data = "") + locked_write(io, true, OPCODE_PING, hasmask, data) end """ Send a ping message, optionally with data.""" -send_ping(ws, data...) = write_ping(ws.socket, data...) +send_ping(ws, data...) = write_ping(ws.socket, !ws.server, data...) -function write_pong(io::IO, data = "") - locked_write(io, true, data, OPCODE_PONG) +function write_pong(io::IO, hasmask, data = "") + locked_write(io, true, OPCODE_PONG, hasmask, data) end """ Send a pong message, optionally with data.""" -send_pong(ws, data...) = write_pong(ws.socket, data...) +send_pong(ws, data...) = write_pong(ws.socket, !ws.server, data...) """ close(ws::WebSocket) @@ -225,7 +214,7 @@ function Base.close(ws::WebSocket) end # Ask client to acknowledge closing the connection - locked_write(ws.socket, true, "", OPCODE_CLOSE) + locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, "") ws.state = CLOSING # Wait till the client responds with an OPCODE_CLOSE. This process is @@ -242,7 +231,7 @@ function Base.close(ws::WebSocket) try while ws.state === CLOSING wsf = read_frame(ws.socket) - # ALERT: stuff might get lost in ether here + # ALERT: stuff might get lost in ether here if is_control_frame(wsf) && (wsf.opcode == OPCODE_CLOSE) ws.state = CLOSED end @@ -262,7 +251,7 @@ Base.isopen(ws::WebSocket) = (ws.state === CONNECTED) && isopen(ws.socket) """ Represents one (received) message frame.""" -type WebSocketFragment +mutable struct WebSocketFragment is_last::Bool rsv1::Bool rsv2::Bool @@ -314,7 +303,7 @@ function handle_control_frame(ws::WebSocket,wsf::WebSocketFragment) # acknowledged by replying with an empty CLOSE frame and cleaning # up try - locked_write(ws.socket, true, "", OPCODE_CLOSE) + locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, "") catch exception # On sudden disconnects, the other side may be gone before the # close acknowledgement can be sent. This will cause an @@ -333,7 +322,7 @@ function handle_control_frame(ws::WebSocket,wsf::WebSocketFragment) throw(WebSocketClosedError()) elseif wsf.opcode == OPCODE_PING - write_pong(ws.socket,wsf.data) + send_pong(ws,wsf.data) elseif wsf.opcode == OPCODE_PONG # Nothing to do here; no reply is needed for a pong message. else # %xB-F are reserved for further control frames @@ -343,7 +332,8 @@ end """ Read a frame: turn bytes from the websocket into a WebSocketFragment.""" function read_frame(io::IO) - a = read(io,UInt8) + ab = read(io,2) + a = ab[1] fin = a & 0b1000_0000 >>> 7 # If fin, then is final fragment rsv1 = a & 0b0100_0000 # If not 0, fail. rsv2 = a & 0b0010_0000 # If not 0, fail. @@ -351,13 +341,14 @@ function read_frame(io::IO) opcode = a & 0b0000_1111 # If not known code, fail. # TODO: add validation somewhere to ensure rsv, opcode, mask, etc are valid. - b = read(io,UInt8) - mask = b & 0b1000_0000 >>> 7 # If not 1, fail. + b = ab[2] + mask = b & 0b1000_0000 >>> 7 + hasmask = mask != 0 - if mask != 1 - error("WebSocket reader cannot handle incoming messages without mask. " * - "See http://tools.ietf.org/html/rfc6455#section-5.3") - end + # if mask != 1 + # error("WebSocket reader cannot handle incoming messages without mask. " * + # "See http://tools.ietf.org/html/rfc6455#section-5.3") + # end payload_len::UInt64 = b & 0b0111_1111 if payload_len == 126 @@ -366,17 +357,10 @@ function read_frame(io::IO) payload_len = ntoh(read(io,UInt64)) # 8 bytes end - maskkey = Array{UInt8,1}(4) - for i in 1:4 - maskkey[i] = read(io,UInt8) - end + maskkey = hasmask ? read(io,4) : UInt8[] - data = Array{UInt8,1}(payload_len) - for i in 1:payload_len - d = read(io, UInt8) - d = xor(d , maskkey[mod(i - 1, 4) + 1]) - data[i] = d - end + data = read(io,Int(payload_len)) + hasmask && mask!(data,maskkey) return WebSocketFragment(fin,rsv1,rsv2,rsv3,opcode,mask,payload_len,maskkey,data) end @@ -422,78 +406,170 @@ value. This is done in three steps: This function then returns the string of the base64-encoded value. """ function generate_websocket_key(key) - hashed_key = digest(MD_SHA1, key*"258EAFA5-E914-47DA-95CA-C5AB0DC85B11") - String(encode(Base64, hashed_key)) + hashkey = "$(key)258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + return base64encode(digest(MD_SHA1, hashkey)) end -""" -Responds to a WebSocket handshake request. -Checks for required headers and subprotocols; sends Response(400) if they're missing or bad. Otherwise, transforms client key into accept value, and sends Reponse(101). -Function returns true for accepted handshakes. -""" -function websocket_handshake(request,client) - if !haskey(request.headers, "Sec-WebSocket-Key") - Base.write(client.sock, Response(400)) - return false +# """ +# Responds to a WebSocket handshake request. +# Checks for required headers and subprotocols; sends Response(400) if they're missing or bad. Otherwise, transforms client key into accept value, and sends Reponse(101). +# Function returns true for accepted handshakes. +# """ +# function websocket_handshake(request,client) +# if !haskey(request.headers, "Sec-WebSocket-Key") +# Base.write(client.sock, Response(400)) +# return false +# end +# if get(request.headers, "Sec-WebSocket-Version", "13") != "13" +# response = Response(400) +# response.headers["Sec-WebSocket-Version"] = "13" +# Base.write(client.sock, response) +# return false +# end + +# key = request.headers["Sec-WebSocket-Key"] +# if length(decode(Base64,key)) != 16 # Key must be 16 bytes +# Base.write(client.sock, Response(400)) +# return false +# end +# resp_key = generate_websocket_key(key) + +# response = Response(101) +# response.headers["Upgrade"] = "websocket" +# response.headers["Connection"] = "Upgrade" +# response.headers["Sec-WebSocket-Accept"] = resp_key + +# if haskey(request.headers, "Sec-WebSocket-Protocol") +# if hasprotocol(request.headers["Sec-WebSocket-Protocol"]) +# response.headers["Sec-WebSocket-Protocol"] = request.headers["Sec-WebSocket-Protocol"] +# else +# Base.write(client.sock, Response(400)) +# return false +# end +# end + +# Base.write(client.sock, response) +# return true +# end + +function mask!(data, mask=rand(UInt8, 4)) + for i in 1:length(data) + data[i] = data[i] ⊻ mask[((i-1) % 4)+1] end - if get(request.headers, "Sec-WebSocket-Version", "13") != "13" - response = Response(400) - response.headers["Sec-WebSocket-Version"] = "13" - Base.write(client.sock, response) - return false + return mask +end + +function open(f::Function, url; binary=false, verbose=false, kw...) + + key = HTTP.base64encode(rand(UInt8, 16)) + + headers = [ + "Upgrade" => "websocket", + "Connection" => "Upgrade", + "Sec-WebSocket-Key" => key, + "Sec-WebSocket-Version" => "13" + ] + + HTTP.open("GET", url, headers; + reuse_limit=0, verbose=verbose ? 2 : 0, kw...) do http + + HTTP.startread(http) + + status = http.message.status + if status != 101 + return + end + + check_upgrade(http) + + if HTTP.header(http, "Sec-WebSocket-Accept") != generate_websocket_key(key) + throw(WebSocketError(0, "Invalid Sec-WebSocket-Accept\n" * + "$(http.message)")) + end + + io = HTTP.ConnectionPool.getrawstream(http) + f(WebSocket(io,false)) end +end - key = request.headers["Sec-WebSocket-Key"] - if length(decode(Base64,key)) != 16 # Key must be 16 bytes - Base.write(client.sock, Response(400)) - return false +# mutable struct WebSocket +# id::Int +# socket::TCPSock +# state::ReadyState + +# function WebSocket(id::Int,socket::TCPSock) +# init_socket(socket) +# new(id, socket, CONNECTED) +# end +# end + +# function listen(f::Function, host::String="localhost", port::UInt16=UInt16(8081); binary=false, verbose=false) +# HTTP.listen(host, port; verbose=verbose) do http +# upgrade(f, http; binary=binary) +# end +# end + +function upgrade(f::Function, http::HTTP.Stream; binary=false) + + check_upgrade(http) + if !HTTP.hasheader(http, "Sec-WebSocket-Version", "13") + throw(WebSocketError(0, "Expected \"Sec-WebSocket-Version: 13\"!\n$(http.message)")) end - resp_key = generate_websocket_key(key) - response = Response(101) - response.headers["Upgrade"] = "websocket" - response.headers["Connection"] = "Upgrade" - response.headers["Sec-WebSocket-Accept"] = resp_key - - if haskey(request.headers, "Sec-WebSocket-Protocol") - if hasprotocol(request.headers["Sec-WebSocket-Protocol"]) - response.headers["Sec-WebSocket-Protocol"] = request.headers["Sec-WebSocket-Protocol"] - else - Base.write(client.sock, Response(400)) - return false - end - end - - Base.write(client.sock, response) - return true -end + HTTP.setstatus(http, 101) + HTTP.setheader(http, "Upgrade" => "websocket") + HTTP.setheader(http, "Connection" => "Upgrade") + key = HTTP.header(http, "Sec-WebSocket-Key") + HTTP.setheader(http, "Sec-WebSocket-Accept" => generate_websocket_key(key)) + + HTTP.startwrite(http) -""" Implement the WebSocketInterface, for compatilibility with HttpServer.""" -immutable WebSocketHandler <: HttpServer.WebSocketInterface - handle::Function + io = HTTP.ConnectionPool.getrawstream(http) + f(WebSocket(io, true)) end -import HttpServer: handle, is_websocket_handshake -""" -Performs handshake. If successfull, establishes WebSocket type and calls -handler with the WebSocket and the original request. On exit from handler, closes websocket. No return value. -""" -function handle(handler::WebSocketHandler, req::Request, client::HttpServer.Client) - websocket_handshake(req, client) || return - sock = WebSocket(client.id, client.sock) - handler.handle(req, sock) - if isopen(sock) - try - close(sock) - end +function check_upgrade(http) + if !HTTP.hasheader(http, "Upgrade", "websocket") + throw(WebSocketError(0, "Expected \"Upgrade: websocket\"!\n$(http.message)")) + end + + if !HTTP.hasheader(http, "Connection", "upgrade") + throw(WebSocketError(0, "Expected \"Connection: upgrade\"!\n$(http.message)")) end end -function is_websocket_handshake(handler::WebSocketHandler, req::Request) - is_get = req.method == "GET" - # "upgrade" for Chrome and "keep-alive, upgrade" for Firefox. - is_upgrade = contains(lowercase(get(req.headers, "Connection", "")),"upgrade") - is_websockets = lowercase(get(req.headers, "Upgrade", "")) == "websocket" - return is_get && is_upgrade && is_websockets + +function is_upgrade(r::HTTP.Message) + (r isa HTTP.Request && r.method == "GET" || r.status == 101) && + HTTP.hasheader(r, "Connection", "upgrade") && + HTTP.hasheader(r, "Upgrade", "websocket") end +# """ Implement the WebSocketInterface, for compatilibility with HttpServer.""" +# struct WebSocketHandler <: HttpServer.WebSocketInterface +# handle::Function +# end + +# import HttpServer: handle, is_websocket_handshake +# """ +# Performs handshake. If successfull, establishes WebSocket type and calls +# handler with the WebSocket and the original request. On exit from handler, closes websocket. No return value. +# """ +# function handle(handler::WebSocketHandler, req::Request, client::HttpServer.Client) +# websocket_handshake(req, client) || return +# sock = WebSocket(client.id, client.sock) +# handler.handle(req, sock) +# if isopen(sock) +# try +# close(sock) +# end +# end +# end +# function is_websocket_handshake(handler::WebSocketHandler, req::Request) +# is_get = req.method == "GET" +# # "upgrade" for Chrome and "keep-alive, upgrade" for Firefox. +# is_upgrade = contains(lowercase(get(req.headers, "Connection", "")),"upgrade") +# is_websockets = lowercase(get(req.headers, "Upgrade", "")) == "websocket" +# return is_get && is_upgrade && is_websockets +# end + end # module WebSockets diff --git a/test/runtests.jl b/test/runtests.jl index ff25684..a68b1f1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,204 +1,262 @@ +# using WebSockets +# using Compat; import Compat.String +# using Base.Test +# import WebSockets: generate_websocket_key, +# write_fragment, +# read_frame, +# is_websocket_handshake, +# websocket_handshake, +# handle +# import HttpCommon: Request + + using WebSockets -using Compat; import Compat.String +using WebSockets.HTTP +using HTTP.IOExtras using Base.Test -import WebSockets: generate_websocket_key, - write_fragment, - read_frame, - is_websocket_handshake, - websocket_handshake, - handle -import HttpCommon: Request -@sync yield() # avoid mixing of output with possible deprecation warnings from .juliarc -info("Starting test WebSockets...") -#is_control_frame is one line, checking one bit. -#get_websocket_key grabs a header. -#is_websocket_handshake grabs a header. -#generate_websocket_key makes a call to a library. -info("Test generate_websocket_key") -@test generate_websocket_key("dGhlIHNhbXBsZSBub25jZQ==") == "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" - -# Test writing - -function xor_payload(maskkey, data) - out = Array{UInt8,1}(length(data)) - for i in 1:length(data) - d = data[i] - d = @compat xor(d , maskkey[mod(i - 1, 4) + 1]) - out[i] = d - end - out -end -const io = IOBuffer() - -info("Test length less than 126") -for len = [8, 125], op = (rand(UInt8) & 0b1111), fin=[true, false] - - test_str = randstring(len) - write_fragment(io, fin, Vector{UInt8}(test_str), op) - - frame = take!(io) - - @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] - @test frame[2] == UInt8(len) - @test String(frame[3:end]) == test_str - - # Check to see if reading message without a mask fails - in_buf = IOBuffer(String(frame)) - @test_throws ErrorException read_frame(in_buf) - close(in_buf) - - # add a mask - maskkey = rand(UInt8, 4) - data = vcat( - frame[1], - frame[2] | 0b1000_0000, - maskkey, - xor_payload(maskkey, frame[3:end]) - ) - frame_back = read_frame(IOBuffer(data)) - - @test frame_back.is_last == fin - @test frame_back.rsv1 == false - @test frame_back.rsv2 == false - @test frame_back.rsv3 == false - @test frame_back.opcode == op - @test frame_back.is_masked == true - @test frame_back.payload_len == len - @test all(map(==, frame_back.maskkey, maskkey)) - @test test_str == String(frame_back.data) -end +import WebSockets: CONNECTED, CLOSING, CLOSED -info("Test length 126 or more") -for len = 126:129, op = 0b1111, fin=[true, false] - - test_str = randstring(len) - write_fragment(io, fin, Vector{UInt8}(test_str), op) - - frame = take!(io) - - @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] - @test frame[2] == 126 - - @test bits(frame[4])*bits(frame[3]) == bits(hton(UInt16(len))) - - # add a mask - maskkey = rand(UInt8, 4) - data = vcat( - frame[1], - frame[2] | 0b1000_0000, - frame[3], - frame[4], - maskkey, - xor_payload(maskkey, frame[5:end]) - ) - frame_back = read_frame(IOBuffer(data)) - - @test frame_back.is_last == fin - @test frame_back.rsv1 == false - @test frame_back.rsv2 == false - @test frame_back.rsv3 == false - @test frame_back.opcode == op - @test frame_back.is_masked == true - @test frame_back.payload_len == len - @test all(map(==, frame_back.maskkey, maskkey)) - @test test_str == String(frame_back.data) -end +@testset "WebSockets" begin -# TODO: test for length > typemax(Uint32) - -info("Tests for is_websocket_handshake") -chromeheaders = Dict{String, String}( - "Connection"=>"Upgrade", - "Upgrade"=>"websocket" - ) -chromerequest = HttpCommon.Request( - "GET", - "", - chromeheaders, - "" - ) - -firefoxheaders = Dict{String, String}( - "Connection"=>"keep-alive, Upgrade", - "Upgrade"=>"websocket" - ) - -firefoxrequest= Request( - "GET", - "", - firefoxheaders, - "" - ) - -wshandler = WebSocketHandler((x,y)->nothing);#Dummy wshandler - -for request in [chromerequest, firefoxrequest] - @test is_websocket_handshake(wshandler,request) == true -end +info("Testing ws...") +WebSockets.open("ws://echo.websocket.org") do ws + write(ws, "Foo") + @test String(read(ws)) == "Foo" -info("Test of handshake response") -takefirstline(buf) = split(buf |> take! |> String, "\r\n")[1] - -take!(io) -Base.write(io, "test") -@test takefirstline(io) == "test" - -info("Test reject / switch format") -const SWITCH = "HTTP/1.1 101 Switching Protocols " -const REJECT = "HTTP/1.1 400 Bad Request " -Base.write(io, WebSockets.Response(400)) -@test takefirstline(io) == REJECT -Base.write(io, WebSockets.Response(101)) -@test takefirstline(io) == SWITCH - -function handshakeresponse(request) - cli = HttpServer.Client(2, IOBuffer()) - websocket_handshake(request, cli) - takefirstline(cli.sock) + close(ws) end +sleep(1) -info("Test simple handshakes that are unacceptable") -for request in [chromerequest, firefoxrequest] - @test handshakeresponse(request) == REJECT - push!(request.headers, "Sec-WebSocket-Version" => "13") - @test handshakeresponse(request) == REJECT - push!(request.headers, "Sec-WebSocket-Key" => "mumbojumbobo") - @test handshakeresponse(request) == REJECT - push!(request.headers, "Sec-WebSocket-Version" => "11") - push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") - @test handshakeresponse(request) == REJECT -end +info("Testing wss...") +WebSockets.open("wss://echo.websocket.org") do ws + write(ws, "Foo") + @test String(read(ws)) == "Foo" -info("Test simple handshakes, acceptable") -for request in [chromerequest, firefoxrequest] - push!(request.headers, "Sec-WebSocket-Version" => "13") - push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") - @test handshakeresponse(request) == SWITCH + close(ws) end - -info("Test unacceptable subprotocol handshake subprotocol") -for request in [chromerequest, firefoxrequest] - push!(request.headers, "Sec-WebSocket-Version" => "13") - push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") - push!(request.headers, "Sec-WebSocket-Protocol" => "my.server/json-zmq") - @test handshakeresponse(request) == REJECT +sleep(1) + +p = UInt16(8000) +@async HTTP.listen("127.0.0.1",p) do http + if WebSockets.is_upgrade(http.message) + WebSockets.upgrade(http) do ws + while ws.state == CONNECTED + data = String(read(ws)) + write(ws,data) + end + end + end end -info("add simple subprotocol to acceptable list") -@test true == WebSockets.addsubproto("xml") +sleep(2) -info("add subprotocol with difficult name") -@test true == WebSockets.addsubproto("my.server/json-zmq") +info("Testing local server...") +WebSockets.open("ws://127.0.0.1:$(p)") do ws + write(ws, "Foo") + @test String(read(ws)) == "Foo" -info("Test handshake subprotocol now acceptable") -for request in [chromerequest, firefoxrequest] - push!(request.headers, "Sec-WebSocket-Version" => "13") - push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") - push!(request.headers, "Sec-WebSocket-Protocol" => "xml") - @test handshakeresponse(request) == SWITCH - push!(request.headers, "Sec-WebSocket-Protocol" => "my.server/json-zmq") - @test handshakeresponse(request) == SWITCH + write(ws, "Bar") + @test String(read(ws)) == "Bar" + + close(ws) end -close(io) -include("browsertest.jl") \ No newline at end of file + +end # testset + + + +# @sync yield() # avoid mixing of output with possible deprecation warnings from .juliarc +# info("Starting test WebSockets...") +# #is_control_frame is one line, checking one bit. +# #get_websocket_key grabs a header. +# #is_websocket_handshake grabs a header. +# #generate_websocket_key makes a call to a library. +# info("Test generate_websocket_key") +# @test generate_websocket_key("dGhlIHNhbXBsZSBub25jZQ==") == "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" + +# # Test writing + +# function xor_payload(maskkey, data) +# out = Array{UInt8,1}(length(data)) +# for i in 1:length(data) +# d = data[i] +# d = @compat xor(d , maskkey[mod(i - 1, 4) + 1]) +# out[i] = d +# end +# out +# end + +# const io = IOBuffer() + +# info("Test length less than 126") +# for len = [8, 125], op = (rand(UInt8) & 0b1111), fin=[true, false] + +# test_str = randstring(len) +# write_fragment(io, fin, Vector{UInt8}(test_str), op) + +# frame = take!(io) + +# @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] +# @test frame[2] == UInt8(len) +# @test String(frame[3:end]) == test_str + +# # Check to see if reading message without a mask fails +# in_buf = IOBuffer(String(frame)) +# @test_throws ErrorException read_frame(in_buf) +# close(in_buf) + +# # add a mask +# maskkey = rand(UInt8, 4) +# data = vcat( +# frame[1], +# frame[2] | 0b1000_0000, +# maskkey, +# xor_payload(maskkey, frame[3:end]) +# ) +# frame_back = read_frame(IOBuffer(data)) + +# @test frame_back.is_last == fin +# @test frame_back.rsv1 == false +# @test frame_back.rsv2 == false +# @test frame_back.rsv3 == false +# @test frame_back.opcode == op +# @test frame_back.is_masked == true +# @test frame_back.payload_len == len +# @test all(map(==, frame_back.maskkey, maskkey)) +# @test test_str == String(frame_back.data) +# end + +# info("Test length 126 or more") +# for len = 126:129, op = 0b1111, fin=[true, false] + +# test_str = randstring(len) +# write_fragment(io, fin, Vector{UInt8}(test_str), op) + +# frame = take!(io) + +# @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] +# @test frame[2] == 126 + +# @test bits(frame[4])*bits(frame[3]) == bits(hton(UInt16(len))) + +# # add a mask +# maskkey = rand(UInt8, 4) +# data = vcat( +# frame[1], +# frame[2] | 0b1000_0000, +# frame[3], +# frame[4], +# maskkey, +# xor_payload(maskkey, frame[5:end]) +# ) +# frame_back = read_frame(IOBuffer(data)) + +# @test frame_back.is_last == fin +# @test frame_back.rsv1 == false +# @test frame_back.rsv2 == false +# @test frame_back.rsv3 == false +# @test frame_back.opcode == op +# @test frame_back.is_masked == true +# @test frame_back.payload_len == len +# @test all(map(==, frame_back.maskkey, maskkey)) +# @test test_str == String(frame_back.data) +# end + +# # TODO: test for length > typemax(Uint32) + +# info("Tests for is_websocket_handshake") +# chromeheaders = Dict{String, String}( +# "Connection"=>"Upgrade", +# "Upgrade"=>"websocket" +# ) +# chromerequest = HttpCommon.Request( +# "GET", +# "", +# chromeheaders, +# "" +# ) + +# firefoxheaders = Dict{String, String}( +# "Connection"=>"keep-alive, Upgrade", +# "Upgrade"=>"websocket" +# ) + +# firefoxrequest= Request( +# "GET", +# "", +# firefoxheaders, +# "" +# ) + +# wshandler = WebSocketHandler((x,y)->nothing);#Dummy wshandler + +# for request in [chromerequest, firefoxrequest] +# @test is_websocket_handshake(wshandler,request) == true +# end + +# info("Test of handshake response") +# takefirstline(buf) = split(buf |> take! |> String, "\r\n")[1] + +# take!(io) +# Base.write(io, "test") +# @test takefirstline(io) == "test" + +# info("Test reject / switch format") +# const SWITCH = "HTTP/1.1 101 Switching Protocols " +# const REJECT = "HTTP/1.1 400 Bad Request " +# Base.write(io, WebSockets.Response(400)) +# @test takefirstline(io) == REJECT +# Base.write(io, WebSockets.Response(101)) +# @test takefirstline(io) == SWITCH + +# function handshakeresponse(request) +# cli = HttpServer.Client(2, IOBuffer()) +# websocket_handshake(request, cli) +# takefirstline(cli.sock) +# end + +# info("Test simple handshakes that are unacceptable") +# for request in [chromerequest, firefoxrequest] +# @test handshakeresponse(request) == REJECT +# push!(request.headers, "Sec-WebSocket-Version" => "13") +# @test handshakeresponse(request) == REJECT +# push!(request.headers, "Sec-WebSocket-Key" => "mumbojumbobo") +# @test handshakeresponse(request) == REJECT +# push!(request.headers, "Sec-WebSocket-Version" => "11") +# push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") +# @test handshakeresponse(request) == REJECT +# end + +# info("Test simple handshakes, acceptable") +# for request in [chromerequest, firefoxrequest] +# push!(request.headers, "Sec-WebSocket-Version" => "13") +# push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") +# @test handshakeresponse(request) == SWITCH +# end + +# info("Test unacceptable subprotocol handshake subprotocol") +# for request in [chromerequest, firefoxrequest] +# push!(request.headers, "Sec-WebSocket-Version" => "13") +# push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") +# push!(request.headers, "Sec-WebSocket-Protocol" => "my.server/json-zmq") +# @test handshakeresponse(request) == REJECT +# end + +# info("add simple subprotocol to acceptable list") +# @test true == WebSockets.addsubproto("xml") + +# info("add subprotocol with difficult name") +# @test true == WebSockets.addsubproto("my.server/json-zmq") + +# info("Test handshake subprotocol now acceptable") +# for request in [chromerequest, firefoxrequest] +# push!(request.headers, "Sec-WebSocket-Version" => "13") +# push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") +# push!(request.headers, "Sec-WebSocket-Protocol" => "xml") +# @test handshakeresponse(request) == SWITCH +# push!(request.headers, "Sec-WebSocket-Protocol" => "my.server/json-zmq") +# @test handshakeresponse(request) == SWITCH +# end +# close(io) +# include("browsertest.jl") \ No newline at end of file From 1dd534be2c67b476b55bf59816259261d2d52245 Mon Sep 17 00:00:00 2001 From: Eric Forgy Date: Fri, 16 Feb 2018 18:23:13 +0800 Subject: [PATCH 02/35] Http.jl (#88) * Use HTTP.jl * Support SSLContext * Get tests to pass * Fix chat example * examples/server.jl is working. * Restrict tests to Julia v0.6 (v0.5 no longer supported). * Almost done. Last test is nearly passing... --- .travis.yml | 1 - appveyor.yml | 2 - examples/chat-client.html | 35 ++- examples/chat.jl | 32 ++- examples/server.jl | 6 - src/HTTP.jl | 76 +++++ src/HttpServer.jl | 74 +++++ src/WebSockets.jl | 160 +---------- test/HTTP.jl | 53 ++++ test/HttpServer.jl | 211 ++++++++++++++ test/browsertest.html | 36 ++- test/browsertest.jl | 16 +- test/browsertest2.html | 9 + ...ndler_functions_websockets_general_test.jl | 77 ++--- ...r_functions_websockets_subprotocol_test.jl | 97 ++++--- test/runtests.jl | 264 +----------------- 16 files changed, 598 insertions(+), 551 deletions(-) create mode 100644 src/HTTP.jl create mode 100644 src/HttpServer.jl create mode 100644 test/HTTP.jl create mode 100644 test/HttpServer.jl diff --git a/.travis.yml b/.travis.yml index 00dd9c9..ea5faa0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ os: - linux - osx julia: - - 0.5 - 0.6 sudo: false notifications: diff --git a/appveyor.yml b/appveyor.yml index aa55302..75c64a0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,9 +1,7 @@ environment: matrix: - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.5/julia-0.5-latest-win64.exe" - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe" - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.5/julia-0.5-latest-win32.exe" # HttpCommon not building yet - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" # HttpCommon not building yet - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" branches: diff --git a/examples/chat-client.html b/examples/chat-client.html index 5ae1c61..f1847f8 100644 --- a/examples/chat-client.html +++ b/examples/chat-client.html @@ -65,6 +65,14 @@

Select a userinput

var $chatForm = document.querySelector("#say_message"); var $chatInput = $chatForm.querySelector("input[name=say]"); + + function uuid4() { + return ([1e7]+-1e3+-4e3+-8e3+-1e11).replace(/[018]/g, c => + (c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> c / 4).toString(16) + ) + } + var id = uuid4(); + function addContent(html) { var $content = document.querySelector("#content"); var div = document.createElement("div"); @@ -86,7 +94,11 @@

Select a userinput

if( !uname.replace(/\s/gi,'').length ) { alert("Please select a valid userinput"); } else { - connection.send('setusername:'+ uname); + var msg = { + "id": id, + "userName": uname + }; + connection.send(JSON.stringify(msg)); $userName.innerHTML = uname; $welcome.style.display = "none"; $chat.style.display = "block"; @@ -94,12 +106,16 @@

Select a userinput

} function whenChatMessage() { - var msg = $chatInput.value; - if(!msg.replace(/\s/gi,'').length) { + var content = $chatInput.value; + if( !content.replace(/\s/gi,'').length) { /* nothing to do */ } else { - connection.send('say:'+ msg); - addContent(`

${you}: ${msg}

`); + var msg = { + "id": id, + "say": content + }; + connection.send(JSON.stringify(msg)); + addContent(`

${you}: ${content}

`); $chatInput.focus(); } } @@ -110,19 +126,14 @@

Select a userinput

whenChatMessage(); return false; }) - - $chatInput.addEventListener("keypress", (e) => { - if( e.keyCode === 13 ) { whenChatMessage(); } ; - return false; - }, false) - + $userForm.addEventListener("submit", function(e){ e.preventDefault(); e.stopImmediatePropagation(); whenUserName(); return false; }); - + const connection = new SocketConnection(onMessageReceived); connection.start(); diff --git a/examples/chat.jl b/examples/chat.jl index 8983e4d..dfaa307 100644 --- a/examples/chat.jl +++ b/examples/chat.jl @@ -1,38 +1,44 @@ using HttpServer using WebSockets +using JSON +struct User + name::String + client::WebSocket +end #global Dict to store open connections in -global connections = Dict{Int,WebSocket}() -global usernames = Dict{Int,String}() +global connections = Dict{String,User}() function decodeMessage( msg ) - String(copy(msg)) + JSON.parse(String(copy(msg))) end wsh = WebSocketHandler() do req, client global connections - @show connections[client.id] = client while true msg = read(client) msg = decodeMessage(msg) - if startswith(msg, "setusername:") - println("SETTING USERNAME: $msg") - usernames[client.id] = msg[13:end] + id = msg["id"] + if haskey(msg,"userName") && !haskey(connections,id) + uname = msg["userName"] + println("SETTING USERNAME: $(uname)") + connections[id] = User(uname,client) end - if startswith(msg, "say:") - println("EMITTING MESSAGE: $msg") + if haskey(msg,"say") + content = msg["say"] + println("EMITTING MESSAGE: $(content)") for (k,v) in connections - if k != client.id - write(v, (usernames[client.id] * ": " * msg[5:end])) + if k != id + write(v.client, (v.name * ": " * content)) end end end end end -onepage = readstring(Pkg.dir("WebSockets","examples","chat-client.html")) httph = HttpHandler() do req::Request, res::Response - Response(onepage) + onepage = readstring(Pkg.dir("WebSockets","examples","chat-client.html")) + Response(onepage) end server = Server(httph, wsh) diff --git a/examples/server.jl b/examples/server.jl index 0cf883b..78ac7da 100644 --- a/examples/server.jl +++ b/examples/server.jl @@ -1,10 +1,6 @@ using HttpServer using WebSockets -#global Dict to store open connections in -global connections = Dict{Int,WebSocket}() -global usernames = Dict{Int,String}() - function decodeMessage( msg ) String(copy(msg)) end @@ -19,8 +15,6 @@ function eval_or_describe_error(strmsg) end wsh = WebSocketHandler() do req, client - global connections - connections[client.id] = client while true val = client |> read |> decodeMessage |> eval_or_describe_error output = String(take!(Base.mystreamvar)) diff --git a/src/HTTP.jl b/src/HTTP.jl new file mode 100644 index 0000000..f713c42 --- /dev/null +++ b/src/HTTP.jl @@ -0,0 +1,76 @@ +info("Loading HTTP methods...") + +function open(f::Function, url; binary=false, verbose=false, kw...) + + key = base64encode(rand(UInt8, 16)) + + headers = [ + "Upgrade" => "websocket", + "Connection" => "Upgrade", + "Sec-WebSocket-Key" => key, + "Sec-WebSocket-Version" => "13" + ] + + HTTP.open("GET", url, headers; + reuse_limit=0, verbose=verbose ? 2 : 0, kw...) do http + + HTTP.startread(http) + + status = http.message.status + if status != 101 + return + end + + check_upgrade(http) + + if HTTP.header(http, "Sec-WebSocket-Accept") != generate_websocket_key(key) + throw(WebSocketError(0, "Invalid Sec-WebSocket-Accept\n" * + "$(http.message)")) + end + + io = HTTP.ConnectionPool.getrawstream(http) + f(WebSocket(io,false)) + end +end + +function upgrade(f::Function, http::HTTP.Stream; binary=false) + + check_upgrade(http) + if !HTTP.hasheader(http, "Sec-WebSocket-Version", "13") + throw(WebSocketError(0, "Expected \"Sec-WebSocket-Version: 13\"!\n$(http.message)")) + end + + HTTP.setstatus(http, 101) + HTTP.setheader(http, "Upgrade" => "websocket") + HTTP.setheader(http, "Connection" => "Upgrade") + key = HTTP.header(http, "Sec-WebSocket-Key") + HTTP.setheader(http, "Sec-WebSocket-Accept" => generate_websocket_key(key)) + + HTTP.startwrite(http) + + io = HTTP.ConnectionPool.getrawstream(http) + f(WebSocket(io, true)) +end + +function check_upgrade(http) + if !HTTP.hasheader(http, "Upgrade", "websocket") + throw(WebSocketError(0, "Expected \"Upgrade: websocket\"!\n$(http.message)")) + end + + if !HTTP.hasheader(http, "Connection", "upgrade") + throw(WebSocketError(0, "Expected \"Connection: upgrade\"!\n$(http.message)")) + end +end + +function is_upgrade(r::HTTP.Message) + (r isa HTTP.Request && r.method == "GET" || r.status == 101) && + HTTP.hasheader(r, "Connection", "upgrade") && + HTTP.hasheader(r, "Upgrade", "websocket") +end + +# function listen(f::Function, host::String="localhost", port::UInt16=UInt16(8081); binary=false, verbose=false) +# HTTP.listen(host, port; verbose=verbose) do http +# upgrade(f, http; binary=binary) +# end +# end + diff --git a/src/HttpServer.jl b/src/HttpServer.jl new file mode 100644 index 0000000..6d8523f --- /dev/null +++ b/src/HttpServer.jl @@ -0,0 +1,74 @@ +info("Loading HttpServer methods...") + +export WebSocketHandler + + +""" +Responds to a WebSocket handshake request. +Checks for required headers and subprotocols; sends Response(400) if they're missing or bad. Otherwise, transforms client key into accept value, and sends Reponse(101). +Function returns true for accepted handshakes. +""" +function websocket_handshake(request,client) + if !haskey(request.headers, "Sec-WebSocket-Key") + Base.write(client.sock, HttpServer.Response(400)) + return false + end + if get(request.headers, "Sec-WebSocket-Version", "13") != "13" + response = HttpServer.Response(400) + response.headers["Sec-WebSocket-Version"] = "13" + Base.write(client.sock, response) + return false + end + + key = request.headers["Sec-WebSocket-Key"] + if length(base64decode(key)) != 16 # Key must be 16 bytes + Base.write(client.sock, HttpServer.Response(400)) + return false + end + resp_key = generate_websocket_key(key) + + response = HttpServer.Response(101) + response.headers["Upgrade"] = "websocket" + response.headers["Connection"] = "Upgrade" + response.headers["Sec-WebSocket-Accept"] = resp_key + + if haskey(request.headers, "Sec-WebSocket-Protocol") + if hasprotocol(request.headers["Sec-WebSocket-Protocol"]) + response.headers["Sec-WebSocket-Protocol"] = request.headers["Sec-WebSocket-Protocol"] + else + Base.write(client.sock, HttpServer.Response(400)) + return false + end + end + + Base.write(client.sock, response) + return true +end + +""" Implement the WebSocketInterface, for compatilibility with HttpServer.""" +struct WebSocketHandler <: HttpServer.WebSocketInterface + handle::Function +end + +""" +Performs handshake. If successfull, establishes WebSocket type and calls +handler with the WebSocket and the original request. On exit from handler, closes websocket. No return value. +""" +function HttpServer.handle(handler::WebSocketHandler, req::HttpServer.Request, client::HttpServer.Client) + websocket_handshake(req, client) || return + sock = WebSocket(client.sock,true) + handler.handle(req, sock) + if isopen(sock) + try + close(sock) + end + end +end + +function HttpServer.is_websocket_handshake(handler::WebSocketHandler, req::HttpServer.Request) + is_get = req.method == "GET" + # "upgrade" for Chrome and "keep-alive, upgrade" for Firefox. + is_upgrade = contains(lowercase(get(req.headers, "Connection", "")),"upgrade") + is_websockets = lowercase(get(req.headers, "Upgrade", "")) == "websocket" + return is_get && is_upgrade && is_websockets +end \ No newline at end of file diff --git a/src/WebSockets.jl b/src/WebSockets.jl index 95ac3f1..bb90be3 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -20,9 +20,8 @@ an http server. """ module WebSockets -using HTTP -import HTTP: digest, MD_SHA1 - +import MbedTLS: digest, MD_SHA1 +using Requires export WebSocket, write, @@ -410,48 +409,6 @@ function generate_websocket_key(key) return base64encode(digest(MD_SHA1, hashkey)) end -# """ -# Responds to a WebSocket handshake request. -# Checks for required headers and subprotocols; sends Response(400) if they're missing or bad. Otherwise, transforms client key into accept value, and sends Reponse(101). -# Function returns true for accepted handshakes. -# """ -# function websocket_handshake(request,client) -# if !haskey(request.headers, "Sec-WebSocket-Key") -# Base.write(client.sock, Response(400)) -# return false -# end -# if get(request.headers, "Sec-WebSocket-Version", "13") != "13" -# response = Response(400) -# response.headers["Sec-WebSocket-Version"] = "13" -# Base.write(client.sock, response) -# return false -# end - -# key = request.headers["Sec-WebSocket-Key"] -# if length(decode(Base64,key)) != 16 # Key must be 16 bytes -# Base.write(client.sock, Response(400)) -# return false -# end -# resp_key = generate_websocket_key(key) - -# response = Response(101) -# response.headers["Upgrade"] = "websocket" -# response.headers["Connection"] = "Upgrade" -# response.headers["Sec-WebSocket-Accept"] = resp_key - -# if haskey(request.headers, "Sec-WebSocket-Protocol") -# if hasprotocol(request.headers["Sec-WebSocket-Protocol"]) -# response.headers["Sec-WebSocket-Protocol"] = request.headers["Sec-WebSocket-Protocol"] -# else -# Base.write(client.sock, Response(400)) -# return false -# end -# end - -# Base.write(client.sock, response) -# return true -# end - function mask!(data, mask=rand(UInt8, 4)) for i in 1:length(data) data[i] = data[i] ⊻ mask[((i-1) % 4)+1] @@ -459,117 +416,8 @@ function mask!(data, mask=rand(UInt8, 4)) return mask end -function open(f::Function, url; binary=false, verbose=false, kw...) - - key = HTTP.base64encode(rand(UInt8, 16)) - - headers = [ - "Upgrade" => "websocket", - "Connection" => "Upgrade", - "Sec-WebSocket-Key" => key, - "Sec-WebSocket-Version" => "13" - ] - - HTTP.open("GET", url, headers; - reuse_limit=0, verbose=verbose ? 2 : 0, kw...) do http - - HTTP.startread(http) - - status = http.message.status - if status != 101 - return - end - - check_upgrade(http) - - if HTTP.header(http, "Sec-WebSocket-Accept") != generate_websocket_key(key) - throw(WebSocketError(0, "Invalid Sec-WebSocket-Accept\n" * - "$(http.message)")) - end - - io = HTTP.ConnectionPool.getrawstream(http) - f(WebSocket(io,false)) - end -end - -# mutable struct WebSocket -# id::Int -# socket::TCPSock -# state::ReadyState - -# function WebSocket(id::Int,socket::TCPSock) -# init_socket(socket) -# new(id, socket, CONNECTED) -# end -# end - -# function listen(f::Function, host::String="localhost", port::UInt16=UInt16(8081); binary=false, verbose=false) -# HTTP.listen(host, port; verbose=verbose) do http -# upgrade(f, http; binary=binary) -# end -# end - -function upgrade(f::Function, http::HTTP.Stream; binary=false) - - check_upgrade(http) - if !HTTP.hasheader(http, "Sec-WebSocket-Version", "13") - throw(WebSocketError(0, "Expected \"Sec-WebSocket-Version: 13\"!\n$(http.message)")) - end - - HTTP.setstatus(http, 101) - HTTP.setheader(http, "Upgrade" => "websocket") - HTTP.setheader(http, "Connection" => "Upgrade") - key = HTTP.header(http, "Sec-WebSocket-Key") - HTTP.setheader(http, "Sec-WebSocket-Accept" => generate_websocket_key(key)) - - HTTP.startwrite(http) - - io = HTTP.ConnectionPool.getrawstream(http) - f(WebSocket(io, true)) -end - -function check_upgrade(http) - if !HTTP.hasheader(http, "Upgrade", "websocket") - throw(WebSocketError(0, "Expected \"Upgrade: websocket\"!\n$(http.message)")) - end - - if !HTTP.hasheader(http, "Connection", "upgrade") - throw(WebSocketError(0, "Expected \"Connection: upgrade\"!\n$(http.message)")) - end -end - -function is_upgrade(r::HTTP.Message) - (r isa HTTP.Request && r.method == "GET" || r.status == 101) && - HTTP.hasheader(r, "Connection", "upgrade") && - HTTP.hasheader(r, "Upgrade", "websocket") -end +@require HTTP include("HTTP.jl") -# """ Implement the WebSocketInterface, for compatilibility with HttpServer.""" -# struct WebSocketHandler <: HttpServer.WebSocketInterface -# handle::Function -# end - -# import HttpServer: handle, is_websocket_handshake -# """ -# Performs handshake. If successfull, establishes WebSocket type and calls -# handler with the WebSocket and the original request. On exit from handler, closes websocket. No return value. -# """ -# function handle(handler::WebSocketHandler, req::Request, client::HttpServer.Client) -# websocket_handshake(req, client) || return -# sock = WebSocket(client.id, client.sock) -# handler.handle(req, sock) -# if isopen(sock) -# try -# close(sock) -# end -# end -# end -# function is_websocket_handshake(handler::WebSocketHandler, req::Request) -# is_get = req.method == "GET" -# # "upgrade" for Chrome and "keep-alive, upgrade" for Firefox. -# is_upgrade = contains(lowercase(get(req.headers, "Connection", "")),"upgrade") -# is_websockets = lowercase(get(req.headers, "Upgrade", "")) == "websocket" -# return is_get && is_upgrade && is_websockets -# end +@require HttpServer include("HttpServer.jl") end # module WebSockets diff --git a/test/HTTP.jl b/test/HTTP.jl new file mode 100644 index 0000000..25d7b36 --- /dev/null +++ b/test/HTTP.jl @@ -0,0 +1,53 @@ +@testset "HTTP" begin + +using HTTP +using WebSockets +# using HTTP.IOExtras +using Base.Test + +import WebSockets: CONNECTED, CLOSING, CLOSED + +info("Testing ws...") +WebSockets.open("ws://echo.websocket.org") do ws + write(ws, "Foo") + @test String(read(ws)) == "Foo" + + close(ws) +end +sleep(1) + +info("Testing wss...") +WebSockets.open("wss://echo.websocket.org") do ws + write(ws, "Foo") + @test String(read(ws)) == "Foo" + + close(ws) +end +sleep(1) + +p = UInt16(8000) +@async HTTP.listen("127.0.0.1",p) do http + if WebSockets.is_upgrade(http.message) + WebSockets.upgrade(http) do ws + while ws.state == CONNECTED + data = String(read(ws)) + write(ws,data) + end + end + end +end + +sleep(2) + +info("Testing local server...") +WebSockets.open("ws://127.0.0.1:$(p)") do ws + write(ws, "Foo") + @test String(read(ws)) == "Foo" + + write(ws, "Bar") + @test String(read(ws)) == "Bar" + + close(ws) +end + +end # testset \ No newline at end of file diff --git a/test/HttpServer.jl b/test/HttpServer.jl new file mode 100644 index 0000000..7a7351a --- /dev/null +++ b/test/HttpServer.jl @@ -0,0 +1,211 @@ +@testset "HttpServer" begin + +using HttpServer +using WebSockets +import WebSockets: generate_websocket_key, + write_fragment, + read_frame, + websocket_handshake + # is_websocket_handshake, + # handle +import HttpCommon: Request, Response + + +@sync yield() # avoid mixing of output with possible deprecation warnings from .juliarc +info("Starting test WebSockets...") +#is_control_frame is one line, checking one bit. +#get_websocket_key grabs a header. +#is_websocket_handshake grabs a header. +#generate_websocket_key makes a call to a library. +info("Test generate_websocket_key") +@test generate_websocket_key("dGhlIHNhbXBsZSBub25jZQ==") == "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" + +# Test writing + +function xor_payload(maskkey, data) + out = Array{UInt8,1}(length(data)) + for i in 1:length(data) + d = data[i] + d = xor(d , maskkey[mod(i - 1, 4) + 1]) + out[i] = d + end + out +end + +const io = IOBuffer() + +info("Test length less than 126") +for len = [8, 125], op = (rand(UInt8) & 0b1111), fin=[true, false] + + test_str = randstring(len) + write_fragment(io, fin, op, false, Vector{UInt8}(test_str)) + + frame = take!(io) + + @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] + @test frame[2] == UInt8(len) + @test String(frame[3:end]) == test_str + + # The new WebSockets can handle both client and server so can handle no masks + # =========================================================================== + # # Check to see if reading message without a mask fails + # in_buf = IOBuffer(String(frame)) + # @test_throws ErrorException read_frame(in_buf) + # close(in_buf) + + # add a mask + maskkey = rand(UInt8, 4) + data = vcat( + frame[1], + frame[2] | 0b1000_0000, + maskkey, + xor_payload(maskkey, frame[3:end]) + ) + frame_back = read_frame(IOBuffer(data)) + + @test frame_back.is_last == fin + @test frame_back.rsv1 == false + @test frame_back.rsv2 == false + @test frame_back.rsv3 == false + @test frame_back.opcode == op + @test frame_back.is_masked == true + @test frame_back.payload_len == len + @test all(map(==, frame_back.maskkey, maskkey)) + @test test_str == String(frame_back.data) +end + +info("Test length 126 or more") +for len = 126:129, op = 0b1111, fin=[true, false] + + test_str = randstring(len) + write_fragment(io, fin, op, false, Vector{UInt8}(test_str)) + + frame = take!(io) + + @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] + @test frame[2] == 126 + + @test bits(frame[4])*bits(frame[3]) == bits(hton(UInt16(len))) + + # add a mask + maskkey = rand(UInt8, 4) + data = vcat( + frame[1], + frame[2] | 0b1000_0000, + frame[3], + frame[4], + maskkey, + xor_payload(maskkey, frame[5:end]) + ) + frame_back = read_frame(IOBuffer(data)) + + @test frame_back.is_last == fin + @test frame_back.rsv1 == false + @test frame_back.rsv2 == false + @test frame_back.rsv3 == false + @test frame_back.opcode == op + @test frame_back.is_masked == true + @test frame_back.payload_len == len + @test all(map(==, frame_back.maskkey, maskkey)) + @test test_str == String(frame_back.data) +end + +# TODO: test for length > typemax(Uint32) + +info("Tests for is_websocket_handshake") +chromeheaders = Dict{String, String}( + "Connection"=>"Upgrade", + "Upgrade"=>"websocket" + ) +chromerequest = HttpCommon.Request( + "GET", + "", + chromeheaders, + "" + ) + +firefoxheaders = Dict{String, String}( + "Connection"=>"keep-alive, Upgrade", + "Upgrade"=>"websocket" + ) + +firefoxrequest= Request( + "GET", + "", + firefoxheaders, + "" + ) + +wshandler = WebSocketHandler((x,y)->nothing);#Dummy wshandler + +for request in [chromerequest, firefoxrequest] + @test HttpServer.is_websocket_handshake(wshandler,request) == true +end + +info("Test of handshake response") +takefirstline(buf) = split(buf |> take! |> String, "\r\n")[1] + +take!(io) +Base.write(io, "test") +@test takefirstline(io) == "test" + +info("Test reject / switch format") +const SWITCH = "HTTP/1.1 101 Switching Protocols " +const REJECT = "HTTP/1.1 400 Bad Request " +Base.write(io, Response(400)) +@test takefirstline(io) == REJECT +Base.write(io, Response(101)) +@test takefirstline(io) == SWITCH + +function handshakeresponse(request) + cli = HttpServer.Client(2, IOBuffer()) + websocket_handshake(request, cli) + takefirstline(cli.sock) +end + +info("Test simple handshakes that are unacceptable") +for request in [chromerequest, firefoxrequest] + @test handshakeresponse(request) == REJECT + push!(request.headers, "Sec-WebSocket-Version" => "13") + @test handshakeresponse(request) == REJECT + push!(request.headers, "Sec-WebSocket-Key" => "mumbojumbobo") + @test handshakeresponse(request) == REJECT + push!(request.headers, "Sec-WebSocket-Version" => "11") + push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") + @test handshakeresponse(request) == REJECT +end + +info("Test simple handshakes, acceptable") +for request in [chromerequest, firefoxrequest] + push!(request.headers, "Sec-WebSocket-Version" => "13") + push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") + @test handshakeresponse(request) == SWITCH +end + +info("Test unacceptable subprotocol handshake subprotocol") +for request in [chromerequest, firefoxrequest] + push!(request.headers, "Sec-WebSocket-Version" => "13") + push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") + push!(request.headers, "Sec-WebSocket-Protocol" => "my.server/json-zmq") + @test handshakeresponse(request) == REJECT +end + +info("add simple subprotocol to acceptable list") +@test true == WebSockets.addsubproto("xml") + +info("add subprotocol with difficult name") +@test true == WebSockets.addsubproto("my.server/json-zmq") + +info("Test handshake subprotocol now acceptable") +for request in [chromerequest, firefoxrequest] + push!(request.headers, "Sec-WebSocket-Version" => "13") + push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") + push!(request.headers, "Sec-WebSocket-Protocol" => "xml") + @test handshakeresponse(request) == SWITCH + push!(request.headers, "Sec-WebSocket-Protocol" => "my.server/json-zmq") + @test handshakeresponse(request) == SWITCH +end +close(io) +include("browsertest.jl") + +end \ No newline at end of file diff --git a/test/browsertest.html b/test/browsertest.html index 63216ff..941031a 100644 --- a/test/browsertest.html +++ b/test/browsertest.html @@ -24,6 +24,14 @@

ws3 Websocket: server_denies_protocol

+ + diff --git a/examples/chat_explore.jl b/examples/chat_explore.jl new file mode 100644 index 0000000..7e0b77b --- /dev/null +++ b/examples/chat_explore.jl @@ -0,0 +1,218 @@ +#= +Difference to chat-client: +This example declares global variables and duck typing. +Examine types in the REPL. +Function containers are explicitly defined with names. Although +anonymous functions may be more commonly used in the web domain, +named functions improve error message readability. +It does not rely on socket ids generated by HttpServer or javascript. +It uses duck typing in order to avoid unneccesary upstream +package requirements. +Argument 'client' (used elsewhere) is renamed to 'thisws' to empasize that WebSockets are initated by +the other end of the connection. +=# +using HttpServer +using HTTP +using WebSockets +global lastreq +global lastws +global lastdata +global lastmsg +global lasthttp +global lastws +global laste +const CLOSEAFTER = Base.Dates.Second(30) +const HTTPPORT = 8080 +const PORT_OLDTYPE = 8000 +const USERNAMES = Dict{String, WebSocket}() +const CONNECTIONS = Set{WebSocket}() + + +#= +low level functions +=# +function protectedwrite(ws, msg) + global laste + try + write(ws, msg) + catch e + laste = e + println("Interrupted write! Check global variable laste. Deleting references") + println(e) + ws ∈ CONNECTIONS && pop!(CONNECTIONS, ws) + for (discardname, wsref) in USERNAMES + if wsref == ws + pop!(USERNAMES, discardname) + break + end + end + return false + end + true +end + +function protectedread(ws) + global laste + global lastdata + data = Vector{UInt8}() + contflag = true + try + data = read(ws) + lastdata = data + catch e + laste = e + contflag = false + println("Ouch! Interrupted read. Check global variable laste") + println(e) + finally + return data, contflag + end +end + + + +function findusername(oldname, msg) + newname = msg[length("userName:")+1:end] + if newname == oldname || newname == "" || haskey(USERNAMES, newname) + return oldname, false + else + oldname != "" && pop!(USERNAMES, oldname) + return newname, true + end +end + + +function distributemsg(msgout, not_to_ws) + foreach(CONNECTIONS) do ws + if ws != not_to_ws + protectedwrite(ws, msgout) + end + end + nothing +end + +#= +Functions for old type i.e. HttpServer based connections +=# +function wsfunc_oldtype(req, thisws) + global lastreq + global lastws + global lastdata + global lastmsg + global laste + lastreq = req + lastws = thisws + push!(CONNECTIONS, thisws) + contflag = true + t0 = now() + data = Vector{UInt8}() + msg = "" + username = "" + changedname = false + while now()-t0 < CLOSEAFTER && contflag + data, contflag = protectedread(thisws) + if contflag + msg = String(data) + lastmsg = msg + println("Received: $msg") + if startswith(msg, "userName:") + username, changedname = findusername(username, msg) + else + changedname = false + end + if changedname + println("Tell everybody about $username") + distributemsg(username * " enters chat", thisws) + USERNAMES[username] = thisws + else + if username == "" + println("Discarded message, set a unique username!") + if !protectedwrite(thisws, "Discarded message, set a unique username!") + contflag = false + end + else + distributemsg(username * ": " * msg, thisws) + end + if contflag + contflag = !startswith(msg, "exit") + contflag || println("Received exit message. Closing.") + end + end + end + end + close(thisws) + exitusername = username == "" ? "unknown" : username + println("Closing connection with " * exitusername) + distributemsg(exitusername * " has left", thisws) + username != "" && pop!(USERNAMES, username) + thisws ∈ CONNECTIONS && pop!(CONNECTIONS, thisws) + nothing +end +# Just for easy REPL inspection, we'll declare the handler object explicitly. +# With handler we mean an instance of a structure with at least one function reference. +handler_ws_oldtype = WebSocketHandler(wsfunc_oldtype) +# explicit http server handlers +httpfunc_oldtype(req, res) = readstring(Pkg.dir("WebSockets","examples","chat_explore.html")) |> Response +handler_http_oldtype = HttpHandler(httpfunc_oldtype) +# define both in one server. We could call this a handler, too, since it's just a +# bigger function structure. Or we may call it an object. +server_def_oldtype = Server(handler_http_oldtype, handler_ws_oldtype ) + +#= + Now we'll run an external program which starts + the necessary tasks on Julia. + We can run this async, which is actually considered + bad pracice and leads to more bad connections. + For debugging and building programs, though, it's gold to run this async. +=# +listentask = @async run(server_def_oldtype, PORT_OLDTYPE) + +#= + + With listentask, we can for example throw InterrupException at it to close down. + With recent versions of Julia, we could bind a channel to it as well. + If we did, errors thrown in the task would have an outlet for its frustracctions. +=# + +println("Chat server listening on $PORT_OLDTYPE") + + + +# TODO finish the new way on another simultaneous port. +#task_httpserver = @async run(server, PORT_OLDTYPE) + +# global http +# global connections +# if WebSockets.is_upgrade(http.message) +# WebSockets.upgrade(http) do ws +# while ws.state == WebSockets.CONNECTED +# msg = String(read(ws)) +# println(msg) # Write the received message to the REPL +# protectedwrite(ws,msg) +# end +# end +# end +#end +#function wsh(ws) +# global connections +# global lastws +#end +# +#function httph(http) +# global lasthttp +# = HttpServerHandler() do req::Request, res::Response +# onepage = readstring(Pkg.dir("WebSockets","examples","chat-client.html")) +# Response(onepage) +#end +## Start the echo server +#@async HTTP.listen("127.0.0.1", UInt16(HTTPPORT)) do http +# if WebSockets.is_upgrade(http.message) +# WebSockets.upgrade(http) do ws +# while ws.state == WebSockets.CONNECTED +# msg = String(read(ws)) +# println(msg) # Write the received message to the REPL +# protectedwrite(ws,msg) +# end +# end +# end +#end From 054e1ce58af3b9d2505dffdd61ca16d26e501d48 Mon Sep 17 00:00:00 2001 From: hustf Date: Sun, 18 Feb 2018 23:23:44 +0100 Subject: [PATCH 06/35] modified: appveyor.yml - also test change_dependencies branch. Remove when merging to master. --- appveyor.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 75c64a0..845944a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,6 +6,7 @@ environment: # HttpCommon not building yet - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" branches: only: + - change_dependencies - master - /release-.*/ @@ -33,7 +34,7 @@ build_script: - IF EXIST .git\shallow (git fetch --unshallow) - ps: | Write-Host "This is PowerShell - STDERR output will be red. Redirect! " - $env:PATH+=";C:\projects\julia\bin\" + $env:PATH+=";C:\projects\julia\bin\" julia -e 'versioninfo();redirect_stderr(STDOUT);println(STDOUT, \"stdout\"); println(STDERR, \"stderr\")' julia -e 'redirect_stderr(STDOUT);Pkg.init()' julia --depwarn=no -e 'redirect_stderr(STDOUT);Pkg.add(\"HttpServer\");using HttpServer;' @@ -42,4 +43,4 @@ build_script: test_script: - ps: | Write-Host "This is PS" - julia -e 'redirect_stderr(STDOUT);info(\"Local websockets directory:\", Pkg.dir(\"WebSockets\"));Pkg.test(\"WebSockets\")' \ No newline at end of file + julia -e 'redirect_stderr(STDOUT);info(\"Local websockets directory:\", Pkg.dir(\"WebSockets\"));Pkg.test(\"WebSockets\")' From 4f40a50b9bfaff79ab738546945e961641a00ae1 Mon Sep 17 00:00:00 2001 From: hustf Date: Sun, 18 Feb 2018 23:38:00 +0100 Subject: [PATCH 07/35] modified: README.md - temporary badges for CI tests on appveyor. --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2ff5110..7210e85 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,13 @@ WebSockets.jl [![WebSockets](http://pkg.julialang.org/badges/WebSockets_0.6.svg)](http://pkg.julialang.org/?pkg=WebSockets&ver=0.6) +Temporary badges: +[![Build status](https://ci.appveyor.com/api/projects/status/sx6i51rjc9ajjdh8?svg=true)](https://ci.appveyor.com/project/hustf/websockets-jl-nfuiv) + +[![Build status](https://ci.appveyor.com/api/projects/status/sx6i51rjc9ajjdh8/branch/master?svg=true)](https://ci.appveyor.com/project/hustf/websockets-jl-nfuiv/branch/master) + +[![Build status](https://ci.appveyor.com/api/projects/status/sx6i51rjc9ajjdh8/branch/master?svg=true)](https://ci.appveyor.com/project/hustf/websockets-jl-nfuiv/branch/change_dependencies) + This is an implementation of the WebSockets protocol in Julia for both server-side and client-side applications. This package works with either [HttpServer.jl](https://github.com/JuliaWeb/HttpServer.jl) or [HTTP.jl](https://github.com/JuliaWeb/HttpServer.jl), which is what you use to set up a server that accepts HTTP(S) connections. @@ -94,7 +101,7 @@ WebSockets.jl must be used with either HttpServer.jl or HTTP.jl, but neither is julia> Pkg.add("HttpServer") ~~~ -or +or ~~~julia julia> Pkg.add("HTTP") From 9b01fe63d94d5ddf42554116143748087483ec91 Mon Sep 17 00:00:00 2001 From: Eric Forgy Date: Wed, 21 Feb 2018 00:14:16 +0800 Subject: [PATCH 08/35] Http.jl (#92) * Update README * Graceful close from client + run HTTP server HttpServer server simulataneously. * Test ping, pong. Temp tests. * Add try, catch and close Indent WIP * Fix String mask issue * Minor: Fix an indent * Update WebSockets.jl Add a comment to remove the `copy` for v0.7. --- src/HTTP.jl | 14 ++++++++-- src/HttpServer.jl | 34 ++++++++++++------------- src/WebSockets.jl | 16 ++++++------ test/runtests.jl | 65 +++++++++++++++-------------------------------- 4 files changed, 57 insertions(+), 72 deletions(-) diff --git a/src/HTTP.jl b/src/HTTP.jl index f713c42..470c891 100644 --- a/src/HTTP.jl +++ b/src/HTTP.jl @@ -29,7 +29,12 @@ function open(f::Function, url; binary=false, verbose=false, kw...) end io = HTTP.ConnectionPool.getrawstream(http) - f(WebSocket(io,false)) + ws = WebSocket(io,false) + try + f(ws) + finally + close(ws) + end end end @@ -49,7 +54,12 @@ function upgrade(f::Function, http::HTTP.Stream; binary=false) HTTP.startwrite(http) io = HTTP.ConnectionPool.getrawstream(http) - f(WebSocket(io, true)) + ws = WebSocket(io, true) + try + f(ws) + finally + close(ws) + end end function check_upgrade(http) diff --git a/src/HttpServer.jl b/src/HttpServer.jl index 6d8523f..440c32c 100644 --- a/src/HttpServer.jl +++ b/src/HttpServer.jl @@ -25,24 +25,24 @@ function websocket_handshake(request,client) Base.write(client.sock, HttpServer.Response(400)) return false end - resp_key = generate_websocket_key(key) + resp_key = generate_websocket_key(key) - response = HttpServer.Response(101) - response.headers["Upgrade"] = "websocket" - response.headers["Connection"] = "Upgrade" - response.headers["Sec-WebSocket-Accept"] = resp_key - - if haskey(request.headers, "Sec-WebSocket-Protocol") - if hasprotocol(request.headers["Sec-WebSocket-Protocol"]) - response.headers["Sec-WebSocket-Protocol"] = request.headers["Sec-WebSocket-Protocol"] - else - Base.write(client.sock, HttpServer.Response(400)) - return false - end - end - - Base.write(client.sock, response) - return true + response = HttpServer.Response(101) + response.headers["Upgrade"] = "websocket" + response.headers["Connection"] = "Upgrade" + response.headers["Sec-WebSocket-Accept"] = resp_key + + if haskey(request.headers, "Sec-WebSocket-Protocol") + if hasprotocol(request.headers["Sec-WebSocket-Protocol"]) + response.headers["Sec-WebSocket-Protocol"] = request.headers["Sec-WebSocket-Protocol"] + else + Base.write(client.sock, HttpServer.Response(400)) + return false + end + end + + Base.write(client.sock, response) + return true end """ Implement the WebSocketInterface, for compatilibility with HttpServer.""" diff --git a/src/WebSockets.jl b/src/WebSockets.jl index 95b21d8..4e8aeab 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -136,7 +136,7 @@ end write_fragment(io, islast, opcode, hasmask, data::Array{UInt8}) Write the raw frame to a bufffer """ -function write_fragment(io::IO, islast::Bool, opcode, hasmask::Bool, data::Array{UInt8}) +function write_fragment(io::IO, islast::Bool, opcode, hasmask::Bool, data::Vector{UInt8}) l = length(data) b1::UInt8 = (islast ? 0b1000_0000 : 0b0000_0000) | opcode @@ -159,10 +159,10 @@ function write_fragment(io::IO, islast::Bool, opcode, hasmask::Bool, data::Array end """ Write without interruptions""" -function locked_write(io::IO, islast::Bool, opcode, hasmask, data) +function locked_write(io::IO, islast::Bool, opcode, hasmask::Bool, data::Vector{UInt8}) isa(io, TCPSock) && lock(io.lock) try - write_fragment(io, islast, opcode, hasmask, Vector{UInt8}(data)) + write_fragment(io, islast, opcode, hasmask, data) finally if isa(io, TCPSock) flush(io) @@ -173,7 +173,7 @@ end """ Write text data; will be sent as one frame.""" function Base.write(ws::WebSocket,data::String) - locked_write(ws.socket, true, OPCODE_TEXT, !ws.server, data) + locked_write(ws.socket, true, OPCODE_TEXT, !ws.server, copy(Vector{UInt8}(data))) # Remove this `copy` after v0.7! end """ Write binary data; will be sent as one frame.""" @@ -182,14 +182,14 @@ function Base.write(ws::WebSocket, data::Array{UInt8}) end -function write_ping(io::IO, hasmask, data = "") +function write_ping(io::IO, hasmask, data = UInt8[]) locked_write(io, true, OPCODE_PING, hasmask, data) end """ Send a ping message, optionally with data.""" send_ping(ws, data...) = write_ping(ws.socket, !ws.server, data...) -function write_pong(io::IO, hasmask, data = "") +function write_pong(io::IO, hasmask, data = UInt8[]) locked_write(io, true, OPCODE_PONG, hasmask, data) end """ Send a pong message, optionally with data.""" @@ -202,7 +202,7 @@ Send a close message. function Base.close(ws::WebSocket) if isopen(ws) ws.state = CLOSING - locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, "") + locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, UInt8[]) # Wait till the other end responds with an OPCODE_CLOSE. This process is # complicated by potential blocking reads on the WebSocket in other Tasks @@ -289,7 +289,7 @@ function handle_control_frame(ws::WebSocket,wsf::WebSocketFragment) if wsf.opcode == OPCODE_CLOSE info("$(ws.server ? "Server" : "Client") received OPCODE_CLOSE") ws.state = CLOSED - locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, "") + locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, UInt8[]) elseif wsf.opcode == OPCODE_PING info("$(ws.server ? "Server" : "Client") received OPCODE_PING") send_pong(ws,wsf.data) diff --git a/test/runtests.jl b/test/runtests.jl index c0afee6..075bdb9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,25 +5,9 @@ using Base.Test @testset "HTTP" begin -info("Testing ws...") -WebSockets.open("ws://echo.websocket.org") do ws - write(ws, "Foo") - @test String(read(ws)) == "Foo" - - close(ws) -end -sleep(1) - -info("Testing wss...") -WebSockets.open("wss://echo.websocket.org") do ws - write(ws, "Foo") - @test String(read(ws)) == "Foo" - - close(ws) -end -sleep(1) - port_HTTP = 8000 +port_HttpServer = 8081 + info("Start HTTP server on port $(port_HTTP)") @async HTTP.listen("127.0.0.1",UInt16(port_HTTP)) do http if WebSockets.is_upgrade(http.message) @@ -36,7 +20,6 @@ info("Start HTTP server on port $(port_HTTP)") end end -port_HttpServer = 8081 info("Start HttpServer on port $(port_HttpServer)") wsh = WebSocketHandler() do req,ws while !eof(ws) @@ -49,32 +32,24 @@ server = Server(wsh) sleep(2) -info("Testing local HTTP server...") -WebSockets.open("ws://127.0.0.1:$(port_HTTP)") do ws - write(ws, "Foo") - @test String(read(ws)) == "Foo" - - write(ws, "Bar") - @test String(read(ws)) == "Bar" - - send_ping(ws) - read(ws) - - close(ws) -end - -info("Testing local HttpServer...") -WebSockets.open("ws://127.0.0.1:$(port_HttpServer)") do ws - write(ws, "Foo") - @test String(read(ws)) == "Foo" - - write(ws, "Bar") - @test String(read(ws)) == "Bar" - - send_ping(ws) - read(ws) - - close(ws) +servers = [ + ("ws", "ws://echo.websocket.org"), + ("wss", "wss://echo.websocket.org"), + ("HTTP", "ws://127.0.0.1:$(port_HTTP)"), + ("HttpServer", "ws://127.0.0.1:$(port_HttpServer)")] + +for (s, url) in servers + info("Testing local $(s) server at $(url)...") + WebSockets.open(url) do ws + write(ws, "Foo") + @test String(read(ws)) == "Foo" + + write(ws, "Bar") + @test String(read(ws)) == "Bar" + + send_ping(ws) + read(ws) + end end end # testset \ No newline at end of file From 03380cd01cad224d71f7ff4e089bb056df271674 Mon Sep 17 00:00:00 2001 From: hustf Date: Wed, 21 Feb 2018 20:43:58 +0100 Subject: [PATCH 09/35] modified: examples/chat_explore.jl - extended with HTTP server and a modifified upgrade function (get errors) --- examples/chat_explore.jl | 188 ++++++++++++++++++++++++++------------- 1 file changed, 126 insertions(+), 62 deletions(-) diff --git a/examples/chat_explore.jl b/examples/chat_explore.jl index 7e0b77b..3aeec0f 100644 --- a/examples/chat_explore.jl +++ b/examples/chat_explore.jl @@ -1,26 +1,36 @@ #= Difference to chat-client: -This example declares global variables and duck typing. -Examine types in the REPL. +This example declares global variables and use duck typing on +them. Their types will change. +The aim is that you can examine types in the REPL while running the +example. The aim is NOT that you can expect clean exits. We don't +release all the references after you close connections. + Function containers are explicitly defined with names. Although anonymous functions may be more commonly used in the web domain, -named functions improve error message readability. -It does not rely on socket ids generated by HttpServer or javascript. -It uses duck typing in order to avoid unneccesary upstream -package requirements. -Argument 'client' (used elsewhere) is renamed to 'thisws' to empasize that WebSockets are initated by -the other end of the connection. +named functions may improve error message readability. + +This example does not rely on socket ids generated by HttpServer or +by the browser. =# + + + +# Globals, where used in functions will change the type +global lastreq = 0 +global lastws= 0 +global lastdata= 0 +global lastmsg= 0 +global lasthttp= 0 +global lastws= 0 +global laste= 0 +global lasthttp= 0 + using HttpServer using HTTP using WebSockets -global lastreq -global lastws -global lastdata -global lastmsg -global lasthttp -global lastws -global laste + + const CLOSEAFTER = Base.Dates.Second(30) const HTTPPORT = 8080 const PORT_OLDTYPE = 8000 @@ -29,7 +39,52 @@ const CONNECTIONS = Set{WebSocket}() #= -low level functions +Redefinition of functions defined in WebSockets/ HTTP.jl. +To be removed after a choice of structure vs function solution, +ref. issue #91 20-21/2-18 +=# + +function upgrade(f::Function, http::HTTP.Stream; binary=false) + + check_upgrade(http) + if !HTTP.hasheader(http, "Sec-WebSocket-Version", "13") + throw(WebSocketError(0, "Expected \"Sec-WebSocket-Version: 13\"!\n$(http.message)")) + end + + HTTP.setstatus(http, 101) + HTTP.setheader(http, "Upgrade" => "websocket") + HTTP.setheader(http, "Connection" => "Upgrade") + key = HTTP.header(http, "Sec-WebSocket-Key") + HTTP.setheader(http, "Sec-WebSocket-Accept" => generate_websocket_key(key)) + + HTTP.startwrite(http) + + io = HTTP.ConnectionPool.getrawstream(http) + ws = WebSocket(io, true) + try + if applicable(f, http.Request, ws, binary) + f(http.Request, ws, binary) + elseif applicable(f, http.Request, ws) + f(http.Request, ws) + else + f(ws) + end + catch e + println(e) + laste = e + # Accept, continue or throw depending on e. Throw unknown types. + finally + close(ws) + end +end + + + + + + +#= +low level functions, works on old and new type. =# function protectedwrite(ws, msg) global laste @@ -91,16 +146,11 @@ function distributemsg(msgout, not_to_ws) nothing end -#= -Functions for old type i.e. HttpServer based connections -=# -function wsfunc_oldtype(req, thisws) - global lastreq +# This function would ideally be common to both types. Julia would just compile two versions depending on the exact type. +function wsfunc(thisws) global lastws global lastdata global lastmsg - global laste - lastreq = req lastws = thisws push!(CONNECTIONS, thisws) contflag = true @@ -148,9 +198,30 @@ function wsfunc_oldtype(req, thisws) thisws ∈ CONNECTIONS && pop!(CONNECTIONS, thisws) nothing end + + + +#= +Functions for old type i.e. HttpServer based connections. +This function is called after handshake, and after +subprotocol is checked against a list of user supported subprotocols. +=# +function gatekeeper_oldtype(req, ws) + global lastreq + global lastws + lastreq = req + lastreq = req + # Here we can pick between functions + # based on e.g. + # if haskey(req.headers,"Sec-WebSocket-Protocol") + # + wsfunc(ws) +end + + # Just for easy REPL inspection, we'll declare the handler object explicitly. # With handler we mean an instance of a structure with at least one function reference. -handler_ws_oldtype = WebSocketHandler(wsfunc_oldtype) +handler_ws_oldtype = WebSocketHandler(gatekeeper_oldtype) # explicit http server handlers httpfunc_oldtype(req, res) = readstring(Pkg.dir("WebSockets","examples","chat_explore.html")) |> Response handler_http_oldtype = HttpHandler(httpfunc_oldtype) @@ -176,43 +247,36 @@ listentask = @async run(server_def_oldtype, PORT_OLDTYPE) println("Chat server listening on $PORT_OLDTYPE") +#= + +Now open another port using HTTP instead of HttpServer +We'll start by defining the input functions for HTTP's listen method + +=# + + +function gatekeeper_newtype(req::HTTP.Request, ws) + global lastreq + lastreq = req + # Inspect http.Request to pick the right function. + wsfunc(ws) +end + +httpfunc_newtype(req::HTTP.Request) = readstring(Pkg.dir("WebSockets","examples","chat_explore.html")) |> HTTP.Response + +function server_def_newtype(http) + global lasthttp + lasthttp = http + if WebSockets.is_upgrade(http.message) + WebSockets.upgrade(gatekeeper_newtype, http) + else + HTTP.Servers.handle_request(httpfunc_newtype, http) + end +end + + -# TODO finish the new way on another simultaneous port. -#task_httpserver = @async run(server, PORT_OLDTYPE) - -# global http -# global connections -# if WebSockets.is_upgrade(http.message) -# WebSockets.upgrade(http) do ws -# while ws.state == WebSockets.CONNECTED -# msg = String(read(ws)) -# println(msg) # Write the received message to the REPL -# protectedwrite(ws,msg) -# end -# end -# end -#end -#function wsh(ws) -# global connections -# global lastws -#end -# -#function httph(http) -# global lasthttp -# = HttpServerHandler() do req::Request, res::Response -# onepage = readstring(Pkg.dir("WebSockets","examples","chat-client.html")) -# Response(onepage) -#end -## Start the echo server -#@async HTTP.listen("127.0.0.1", UInt16(HTTPPORT)) do http -# if WebSockets.is_upgrade(http.message) -# WebSockets.upgrade(http) do ws -# while ws.state == WebSockets.CONNECTED -# msg = String(read(ws)) -# println(msg) # Write the received message to the REPL -# protectedwrite(ws,msg) -# end -# end -# end -#end +info("Start HTTP server on port $(HTTPPORT)") +litas_newtype = @async HTTP.listen(server_def_newtype, "127.0.0.1", UInt16(HTTPPORT)) +nothing \ No newline at end of file From bafb17567136b2111d73e97a681b42d363e1189a Mon Sep 17 00:00:00 2001 From: Eric Forgy Date: Thu, 22 Feb 2018 03:53:26 +0800 Subject: [PATCH 10/35] Only copy String bytes if masked (#93) --- src/WebSockets.jl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/WebSockets.jl b/src/WebSockets.jl index 4e8aeab..12c1c0a 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -154,7 +154,12 @@ function write_fragment(io::IO, islast::Bool, opcode, hasmask::Bool, data::Vecto else error("Attempted to send too much data for one websocket fragment\n") end - hasmask && write(io,mask!(data)) + if hasmask + if opcode == OPCODE_TEXT + data = copy(data) # Avoid masking Strings bytes in place + end + write(io,mask!(data)) + end write(io, data) end @@ -173,7 +178,7 @@ end """ Write text data; will be sent as one frame.""" function Base.write(ws::WebSocket,data::String) - locked_write(ws.socket, true, OPCODE_TEXT, !ws.server, copy(Vector{UInt8}(data))) # Remove this `copy` after v0.7! + locked_write(ws.socket, true, OPCODE_TEXT, !ws.server, Vector{UInt8}(data)) # Vector{UInt8}(String) will give a warning in v0.7. end """ Write binary data; will be sent as one frame.""" From 7be02aa9576b1a82e0b61dafa6e33e481c72f80a Mon Sep 17 00:00:00 2001 From: hustf Date: Thu, 22 Feb 2018 18:27:13 +0100 Subject: [PATCH 11/35] modified: examples/chat_explore.jl - changed interface to gatekeeper, import functions where necessary. --- examples/chat_explore.jl | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/examples/chat_explore.jl b/examples/chat_explore.jl index 3aeec0f..1ec07ef 100644 --- a/examples/chat_explore.jl +++ b/examples/chat_explore.jl @@ -18,7 +18,10 @@ by the browser. # Globals, where used in functions will change the type global lastreq = 0 +global lastreqheaders = 0 + global lastws= 0 +global lastwsHTTP = 0 global lastdata= 0 global lastmsg= 0 global lasthttp= 0 @@ -30,7 +33,6 @@ using HttpServer using HTTP using WebSockets - const CLOSEAFTER = Base.Dates.Second(30) const HTTPPORT = 8080 const PORT_OLDTYPE = 8000 @@ -43,29 +45,30 @@ Redefinition of functions defined in WebSockets/ HTTP.jl. To be removed after a choice of structure vs function solution, ref. issue #91 20-21/2-18 =# - +import WebSockets.upgrade +import WebSockets.check_upgrade +import WebSockets.generate_websocket_key function upgrade(f::Function, http::HTTP.Stream; binary=false) - + global lasthttp + lasthttp = http check_upgrade(http) if !HTTP.hasheader(http, "Sec-WebSocket-Version", "13") throw(WebSocketError(0, "Expected \"Sec-WebSocket-Version: 13\"!\n$(http.message)")) end - HTTP.setstatus(http, 101) HTTP.setheader(http, "Upgrade" => "websocket") HTTP.setheader(http, "Connection" => "Upgrade") key = HTTP.header(http, "Sec-WebSocket-Key") HTTP.setheader(http, "Sec-WebSocket-Accept" => generate_websocket_key(key)) - HTTP.startwrite(http) - io = HTTP.ConnectionPool.getrawstream(http) ws = WebSocket(io, true) + lastws = ws try - if applicable(f, http.Request, ws, binary) - f(http.Request, ws, binary) - elseif applicable(f, http.Request, ws) - f(http.Request, ws) + if applicable(f, http.message.headers, ws, binary) + f(http.message.headers, ws, binary) + elseif applicable(f, http.message.headers, ws) + f(http.message.headers, ws) else f(ws) end @@ -78,11 +81,6 @@ function upgrade(f::Function, http::HTTP.Stream; binary=false) end end - - - - - #= low level functions, works on old and new type. =# @@ -255,10 +253,12 @@ We'll start by defining the input functions for HTTP's listen method =# -function gatekeeper_newtype(req::HTTP.Request, ws) - global lastreq - lastreq = req - # Inspect http.Request to pick the right function. +function gatekeeper_newtype(reqheaders, ws) + global lastreqheaders + lastreqheaders = reqheaders + global lastwsHTTP + lastwsHTTP = ws + # Inspect header Sec-WebSocket-Protocol to pick the right function. wsfunc(ws) end From d0c98a3dbc2da8458acc336b05ba0269c6008ee7 Mon Sep 17 00:00:00 2001 From: hustf Date: Sun, 25 Feb 2018 22:01:25 +0100 Subject: [PATCH 12/35] modified: examples/chat_explore.html modified: examples/chat_explore.jl --- examples/chat_explore.html | 219 +++++++++++++++++++++---------------- examples/chat_explore.jl | 160 ++++++++++++++------------- 2 files changed, 209 insertions(+), 170 deletions(-) diff --git a/examples/chat_explore.html b/examples/chat_explore.html index 15727f7..f2f4811 100644 --- a/examples/chat_explore.html +++ b/examples/chat_explore.html @@ -3,122 +3,149 @@ Websockets client - -
-
-
-

Select a username

-
- + +
+

Select a username

+ + - -
-
-
- +
+
+
+
+ -
-
+ + diff --git a/examples/chat_explore.jl b/examples/chat_explore.jl index 1ec07ef..465d866 100644 --- a/examples/chat_explore.jl +++ b/examples/chat_explore.jl @@ -10,8 +10,6 @@ Function containers are explicitly defined with names. Although anonymous functions may be more commonly used in the web domain, named functions may improve error message readability. -This example does not rely on socket ids generated by HttpServer or -by the browser. =# @@ -19,7 +17,6 @@ by the browser. # Globals, where used in functions will change the type global lastreq = 0 global lastreqheaders = 0 - global lastws= 0 global lastwsHTTP = 0 global lastdata= 0 @@ -32,13 +29,11 @@ global lasthttp= 0 using HttpServer using HTTP using WebSockets - -const CLOSEAFTER = Base.Dates.Second(30) +const CLOSEAFTER = Base.Dates.Second(1800) const HTTPPORT = 8080 const PORT_OLDTYPE = 8000 const USERNAMES = Dict{String, WebSocket}() -const CONNECTIONS = Set{WebSocket}() - +const WEBSOCKETS = Dict{WebSocket, Int}() #= Redefinition of functions defined in WebSockets/ HTTP.jl. @@ -50,6 +45,8 @@ import WebSockets.check_upgrade import WebSockets.generate_websocket_key function upgrade(f::Function, http::HTTP.Stream; binary=false) global lasthttp + global laste + global lastwsHTTP lasthttp = http check_upgrade(http) if !HTTP.hasheader(http, "Sec-WebSocket-Version", "13") @@ -63,7 +60,7 @@ function upgrade(f::Function, http::HTTP.Stream; binary=false) HTTP.startwrite(http) io = HTTP.ConnectionPool.getrawstream(http) ws = WebSocket(io, true) - lastws = ws + lastwsHTTP = ws try if applicable(f, http.message.headers, ws, binary) f(http.message.headers, ws, binary) @@ -73,9 +70,8 @@ function upgrade(f::Function, http::HTTP.Stream; binary=false) f(ws) end catch e - println(e) + warn("chat_explore.upgrade: " , e) laste = e - # Accept, continue or throw depending on e. Throw unknown types. finally close(ws) end @@ -84,21 +80,41 @@ end #= low level functions, works on old and new type. =# +function removereferences(ws) + ws in keys(WEBSOCKETS) && pop!(WEBSOCKETS, ws) + for (discardname, wsref) in USERNAMES + if wsref === ws + pop!(USERNAMES, discardname) + break + end + end + nothing +end + +function process_error(id, e) + if typeof(e) == InterruptException + info(id, "Received exit order.") + elseif typeof(e) == ArgumentError + info(id, typeof(e), "\t", e.msg) + elseif typeof(e) == ErrorException + info(id, typeof(e), "\t", e.msg) + else + if :msg in fieldnames(e) && e.msg == "Attempt to read from closed WebSocket" + warn(id, typeof(e), "\t", e.msg) + else + warn(id, e, "\nStacktrace:", stacktrace(true)) + end + end +end + function protectedwrite(ws, msg) global laste try write(ws, msg) catch e laste = e - println("Interrupted write! Check global variable laste. Deleting references") - println(e) - ws ∈ CONNECTIONS && pop!(CONNECTIONS, ws) - for (discardname, wsref) in USERNAMES - if wsref == ws - pop!(USERNAMES, discardname) - break - end - end + process_error("chat_explore.protectedwrite: ", e) + removereferences(ws) return false end true @@ -115,8 +131,7 @@ function protectedread(ws) catch e laste = e contflag = false - println("Ouch! Interrupted read. Check global variable laste") - println(e) + process_error("chat_explore.protectedread: ", e) finally return data, contflag end @@ -124,33 +139,30 @@ end -function findusername(oldname, msg) - newname = msg[length("userName:")+1:end] - if newname == oldname || newname == "" || haskey(USERNAMES, newname) - return oldname, false - else - oldname != "" && pop!(USERNAMES, oldname) - return newname, true - end +function findusername(msg, ws) + !startswith(msg, "userName:") && return "" + newname = msg[length("userName:") + 1:end] + newname =="" && return "" + haskey(USERNAMES, newname) && return "" + push!(USERNAMES, newname => ws) + newname end function distributemsg(msgout, not_to_ws) - foreach(CONNECTIONS) do ws - if ws != not_to_ws + foreach(keys(WEBSOCKETS)) do ws + if ws !== not_to_ws protectedwrite(ws, msgout) end end nothing end -# This function would ideally be common to both types. Julia would just compile two versions depending on the exact type. function wsfunc(thisws) global lastws - global lastdata global lastmsg lastws = thisws - push!(CONNECTIONS, thisws) + push!(WEBSOCKETS, thisws => length(WEBSOCKETS) +1 ) contflag = true t0 = now() data = Vector{UInt8}() @@ -163,37 +175,32 @@ function wsfunc(thisws) msg = String(data) lastmsg = msg println("Received: $msg") - if startswith(msg, "userName:") - username, changedname = findusername(username, msg) - else - changedname = false - end - if changedname - println("Tell everybody about $username") - distributemsg(username * " enters chat", thisws) - USERNAMES[username] = thisws - else - if username == "" - println("Discarded message, set a unique username!") - if !protectedwrite(thisws, "Discarded message, set a unique username!") + if username == "" + username = findusername(msg, thisws) + if username != "" + if !protectedwrite(thisws, username) contflag = false end + println("Tell everybody about $username") + foreach(keys(WEBSOCKETS)) do ws + protectedwrite(ws, username * " enters chat") + end else - distributemsg(username * ": " * msg, thisws) - end - if contflag - contflag = !startswith(msg, "exit") - contflag || println("Received exit message. Closing.") - end + println("Username taken!") + if !protectedwrite(thisws, "Username taken!") + contflag = false + end + end + else + contflag = !startswith(msg, "exit") + contflag || println("Received exit message. Closing.") end end end - close(thisws) exitusername = username == "" ? "unknown" : username - println("Closing connection with " * exitusername) distributemsg(exitusername * " has left", thisws) - username != "" && pop!(USERNAMES, username) - thisws ∈ CONNECTIONS && pop!(CONNECTIONS, thisws) + removereferences(thisws) + # It's not this functions responsibility to close the websocket. Just to forget about it. nothing end @@ -208,7 +215,7 @@ function gatekeeper_oldtype(req, ws) global lastreq global lastws lastreq = req - lastreq = req + lastws = ws # Here we can pick between functions # based on e.g. # if haskey(req.headers,"Sec-WebSocket-Protocol") @@ -230,20 +237,15 @@ server_def_oldtype = Server(handler_http_oldtype, handler_ws_oldtype ) #= Now we'll run an external program which starts the necessary tasks on Julia. - We can run this async, which is actually considered + We can run this async, which might be considered bad pracice and leads to more bad connections. - For debugging and building programs, though, it's gold to run this async. -=# -listentask = @async run(server_def_oldtype, PORT_OLDTYPE) - -#= - - With listentask, we can for example throw InterrupException at it to close down. - With recent versions of Julia, we could bind a channel to it as well. - If we did, errors thrown in the task would have an outlet for its frustracctions. -=# + For debugging and building programs, it's gold to run this async. + You can close a server that's running in a task using + @schedule Base.throwto(listentask, InterruptException()) + =# +litas_oldtype = @schedule run(server_def_oldtype, PORT_OLDTYPE) -println("Chat server listening on $PORT_OLDTYPE") +info("Chat server listening on $PORT_OLDTYPE") #= @@ -274,9 +276,19 @@ function server_def_newtype(http) end end - - - info("Start HTTP server on port $(HTTPPORT)") -litas_newtype = @async HTTP.listen(server_def_newtype, "127.0.0.1", UInt16(HTTPPORT)) +litas_newtype = @schedule HTTP.listen(server_def_newtype, "127.0.0.1", UInt16(HTTPPORT)) + +function closefromoutside() + if isdefined(:litas_newtype) + @schedule Base.throwto(litas_newtype, InterruptException()) + end + if isdefined(:litas_oldtype) + try + @schedule Base.throwto(litas_oldtype, InterruptException()) + catch e + info("closefromoutside: ", e) + end + end +end nothing \ No newline at end of file From 61221573fa5d2bb88b3099bc318efdf5ba86ed12 Mon Sep 17 00:00:00 2001 From: hustf Date: Sun, 1 Apr 2018 20:22:21 +0200 Subject: [PATCH 13/35] mod: examples/chat-client.html mod: examples/chat_explore.html mod: examples/repl-client.html mod: test/browsertest.html mod: test/browsertest2.html - tightened html5 syntax, now enforced by Firefox mod: examples/chat_explore.jl - removed ad-hoc modification of WebSockets.upgrade mod: src/HTTP.jl - slightly quicker is_websocket_handshake, added comment mod: src/HttpServer.jl - slightly quicker is_upgrade, added comment - revised upgrade function, improved feedback on caught errors, otherwise as previously in chat_explore.jl This may be changed back again if we include more fields in the WebSocket type. --- examples/chat-client.html | 6 ++-- examples/chat_explore.html | 5 +++- examples/chat_explore.jl | 49 +++---------------------------- examples/repl-client.html | 6 ++-- src/HTTP.jl | 59 +++++++++++++++++++++++++++++--------- src/HttpServer.jl | 20 +++++++++---- test/browsertest.html | 8 ++++-- test/browsertest2.html | 6 ++-- 8 files changed, 86 insertions(+), 73 deletions(-) diff --git a/examples/chat-client.html b/examples/chat-client.html index f1847f8..c4a4607 100644 --- a/examples/chat-client.html +++ b/examples/chat-client.html @@ -1,6 +1,9 @@ - + + + + Websockets client diff --git a/examples/chat_explore.html b/examples/chat_explore.html index f2f4811..c148d2e 100644 --- a/examples/chat_explore.html +++ b/examples/chat_explore.html @@ -1,6 +1,9 @@ - + + + + Websockets client + + + + +PlotlyEChartsHeadlessChromiumDataVoyagerD3TreesEscherPagesAtomPlotlyJSMuxLibExpatGSLHTTPClientLibCURLBlinkWebSocketsHttpParserCodecsGumboNullablesBufferedStreamsAbstractTreesPrimesSHAHttpServerLibzHttpCommonMbedTLSURIParserCairoOffsetArraysBaseTestNextWinRPMFixedPointNumbersRequestsHomebrewFactCheckJSONBinDepsDataStructures \ No newline at end of file From b5017e5b3356e3924286e9e8ae1e133012f66bcf Mon Sep 17 00:00:00 2001 From: hustf Date: Mon, 16 Apr 2018 21:51:35 +0200 Subject: [PATCH 20/35] mod: README.md Image experiment mod: examples/serve_verbose/svg/ws_neighborhood.svg Smaller size --- README.md | 2 +- examples/serve_verbose/svg/ws_neighborhood.svg | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 35eb9c8..72bac52 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ This package works with either [HttpServer.jl](https://github.com/JuliaWeb/HttpS ## Temporary picture This test picture shows the package neighborhood prior to this change. Some of the dependencies are test-only dependencies. -[Dependencies and test dependencies neighborhood](examples/serve_verbose/svg/ws_neighborhood.svg) +![Dependencies and test dependencies neighborhood](examples/serve_verbose/svg/ws_neighborhood.svg) ## Using with HttpServer.jl diff --git a/examples/serve_verbose/svg/ws_neighborhood.svg b/examples/serve_verbose/svg/ws_neighborhood.svg index eded708..a5bcc14 100644 --- a/examples/serve_verbose/svg/ws_neighborhood.svg +++ b/examples/serve_verbose/svg/ws_neighborhood.svg @@ -1,4 +1,4 @@ - + Template for drawing, editing and updating node graph layouts - - + -PlotlyEChartsHeadlessChromiumDataVoyagerD3TreesEscherPagesAtomPlotlyJSMuxLibExpatGSLHTTPClientLibCURLBlinkWebSocketsHttpParserCodecsGumboNullablesBufferedStreamsAbstractTreesPrimesSHAHttpServerLibzHttpCommonMbedTLSURIParserCairoOffsetArraysBaseTestNextWinRPMFixedPointNumbersRequestsHomebrewFactCheckJSONBinDepsDataStructures \ No newline at end of file +PlotlyEChartsHeadlessChromiumDataVoyagerD3TreesEscherPagesAtomPlotlyJSMuxLibExpatGSLHTTPClientLibCURLBlinkWebSocketsHttpParserCodecsGumboNullablesBufferedStreamsAbstractTreesPrimesSHAHttpServerLibzHttpCommonMbedTLSURIParserCairoOffsetArraysBaseTestNextWinRPMFixedPointNumbersRequestsHomebrewFactCheckJSONBinDepsDataStructures \ No newline at end of file From 32c6bee3d944a23452302177344d2c1cae0f8810 Mon Sep 17 00:00:00 2001 From: hustf Date: Mon, 16 Apr 2018 22:12:55 +0200 Subject: [PATCH 21/35] modified: README.md Took away temporary network graph (github stylesheets mess with inlined svg) modified: src/HttpServer.jl Reinstated try-catch after ci failure (this works locally) --- README.md | 5 ----- src/HttpServer.jl | 6 ++++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 72bac52..7210e85 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,6 @@ This is an implementation of the WebSockets protocol in Julia for both server-si This package works with either [HttpServer.jl](https://github.com/JuliaWeb/HttpServer.jl) or [HTTP.jl](https://github.com/JuliaWeb/HttpServer.jl), which is what you use to set up a server that accepts HTTP(S) connections. -## Temporary picture -This test picture shows the package neighborhood prior to this change. -Some of the dependencies are test-only dependencies. -![Dependencies and test dependencies neighborhood](examples/serve_verbose/svg/ws_neighborhood.svg) - ## Using with HttpServer.jl As a first example, we can create a WebSockets echo server: diff --git a/src/HttpServer.jl b/src/HttpServer.jl index 8224400..0162000 100644 --- a/src/HttpServer.jl +++ b/src/HttpServer.jl @@ -96,8 +96,10 @@ function HttpServer.handle(handler::WebSocketHandler, req::HttpServer.Request, c websocket_handshake(req, client) || return sock = WebSocket(client.sock,true) handler.handle(req, sock) - if isopen(sock) - close(sock) + if isopen(sock) + try + close(sock) + end end end From 5fca921219be766095b0fa5cda08a4731901b1ef Mon Sep 17 00:00:00 2001 From: hustf Date: Sun, 13 May 2018 23:00:40 +0200 Subject: [PATCH 22/35] new file: benchmark/REQUIRE No direct effect, lists extra requirements for benchmark new file: benchmark/bce.html Browser Client Echo definition new file: benchmark/benchmark.jl Comment lines only, so far new file: benchmark/benchmark_prepare.jl Bandwidth tests, all clients new file: benchmark/favicon.ico Tab icon for bce.html new file: benchmark/functions_benchmark.jl Included in benchmark_prepare.jl new file: benchmark/functions_open_browsers.jl Improved functionality, will replace test/functions_open_browsers.jl new file: benchmark/logs/benchmark_results_readable.log Manual copy-paste from generated benchmark_results.log ...since the utf8-encoding to file is not working well. new file: benchmark/phantom.js Definitions for phantomjs browser new file: benchmark/ws_hts.jl Server for HTTP, websockets, bce.html new file: benchmark/ws_jce.jl Julia Client Echo new file: logutils/log_http.jl Included in logutils_ws, HTTP types new file: logutils/log_httpserver.jl Included in logutils_ws, HttpServer types new file: logutils/log_ws.jl Included in logutils_ws, WebSocket types new file: logutils/logutils_ws.jl Logging utilities, included in benchmark_prepare. modified: src/HTTP.jl Added help text examples. In open(), add own error handling (to be improved). upgrade() error handling is also temporary. modified: src/WebSockets.jl Non-blocking read while closing. Added a reasonable closing handshake timeout Commented out WebSocketClosedError (this must be considered more): Breaks some test code. Rewriting comments (not finished) In read(ws), ignore most easy-to-trigger errors which have to do with opening / closing / scheduling/ browser behaviour. modified: test/runtests.jl Sleep a little longer, CI is slow. --- benchmark/REQUIRE | 5 + benchmark/bce.html | 97 ++ benchmark/benchmark.jl | 83 ++ benchmark/benchmark_prepare.jl | 316 ++++++ benchmark/favicon.ico | Bin 0 -> 99678 bytes benchmark/functions_benchmark.jl | 306 ++++++ benchmark/functions_open_browsers.jl | 174 ++++ benchmark/logs/benchmark_results_readable.log | 970 ++++++++++++++++++ benchmark/phantom.js | 34 + benchmark/ws_hts.jl | 156 +++ benchmark/ws_jce.jl | 131 +++ examples/chat_explore.jl | 1 + logutils/log_http.jl | 42 + logutils/log_httpserver.jl | 85 ++ logutils/log_ws.jl | 30 + logutils/logutils_ws.jl | 398 +++++++ src/HTTP.jl | 105 +- src/WebSockets.jl | 178 +++- test/runtests.jl | 2 +- 19 files changed, 3040 insertions(+), 73 deletions(-) create mode 100644 benchmark/REQUIRE create mode 100644 benchmark/bce.html create mode 100644 benchmark/benchmark.jl create mode 100644 benchmark/benchmark_prepare.jl create mode 100644 benchmark/favicon.ico create mode 100644 benchmark/functions_benchmark.jl create mode 100644 benchmark/functions_open_browsers.jl create mode 100644 benchmark/logs/benchmark_results_readable.log create mode 100644 benchmark/phantom.js create mode 100644 benchmark/ws_hts.jl create mode 100644 benchmark/ws_jce.jl create mode 100644 logutils/log_http.jl create mode 100644 logutils/log_httpserver.jl create mode 100644 logutils/log_ws.jl create mode 100644 logutils/logutils_ws.jl diff --git a/benchmark/REQUIRE b/benchmark/REQUIRE new file mode 100644 index 0000000..e554899 --- /dev/null +++ b/benchmark/REQUIRE @@ -0,0 +1,5 @@ +# Being placed in the benchmark folder, this file has no effect on Julia. +# To run benchmarks, you may need to Pkg.add("..package below...") +HTTP +IndexedTables +UnicodePlots diff --git a/benchmark/bce.html b/benchmark/bce.html new file mode 100644 index 0000000..ff2e031 --- /dev/null +++ b/benchmark/bce.html @@ -0,0 +1,97 @@ + + + + + + + WS text + + +

BCE echoing websocket. +

+
    +
  • ws1
    • +
+

| +Initiates a websocket connection. +For every received websocket message, sends one empty message and then the received message. The websocket is closed by +the server, or when user navigates away.
+

+

+

  This is ...

+

ws1 Websocket

+ + + diff --git a/benchmark/benchmark.jl b/benchmark/benchmark.jl new file mode 100644 index 0000000..afc7b68 --- /dev/null +++ b/benchmark/benchmark.jl @@ -0,0 +1,83 @@ +#= +TERMINOLOGY + +Client The side of a connection which initiates. +Server The side of a connection which accepts. +Origin Sender +Destination Receiver +HTS HTTP server +JCE Julia Client Echoing (only compatible with HTS) +BCE Browser Client Echoing (compatible with HTS and HSS) + +Note we use 'inverse speed' below. In lack of better words, we call this speed. +This is more compatible with BenchmarkTools and more directly useful. +For some fun reasoning, refer https://www.lesswrong.com/posts/qNiH3RRTiXXyvMJRg/inverse-speed + +Note 'latency' can also be defined in other ways. Ad hoc: + +speed [ns/b] time / amount +messagesize [b] Datasize for an individual message (one or more frames) +clientlatency [ns] Time from start sending from client to having received on destination +serverlatency [ns] Time from start sending from server to having received on destination +clientspeed [ns/b] = [s/GB] + = upspeed = clientlatency / messagesize + For JCE: Client sends the message, server records its having received time + For BCE: Server sends the message, client sends back two messages of different lengths. + Clientspeed is calculated from server time records. +serverspeed [ns/b] = [s/GB] + = Downspeed = serverlatency / messagesize + For JCE: Server sends the message, client measures its having received time + For BCE: Server sends the message, client sends back two messages of different lengths. + Serverspeed is calculated from server time records. +clientbandwidth [ns/b] = [s/GB] + Average clientspeed, time per datasize +serverbandwidth [ns/b] = [s/GB] + Average serverspeed, time per datasize + +=# + +#= +----- TODO ---- + +Not used: +** HSS HttpServer server +** delay [ns] Time usefully spent between received message and start of reply +** clientRTT [ns] Round-trip-time for a message initiated by client +** serverRTT [ns] Round-trip-time for a message initiated by server +** Up2down ClientBandwidth / ServerBandwidth +** throughput datasize / (RTT - Delay) +** serversize [b] Messagesize from server +** clientsize [b] Messagesize from client + + + +Outlined benchmarks for optimizing WebSockets: + +- Clientlatency() @ Bandwidth ≈ 0, Clientmessage = 1b + Requires some tweaking of BenchMarkTools.Trial, postponed. +- Serverlatency() @ Bandwidth ≈ 0, Servermessage = 1b + Requires some tweaking of BenchMarkTools.Trial, postponed. +- Maximized bandwidth(VSIZE) @ Up2Down ≈ 1 + Not using BenchmarkTools directly +- ClientRTT(VDELAY) @ Bandwidth ≈ 0, Clientmessage = 1b +- ServerRTT(VDELAY) @ Bandwidth ≈ 0, Servermessage = 1b + The last two checks vs. possibly related issue #1608 on ZMQ + + + +Outlined benchmarks for developing an application using WebSockets (postponed): +- ClientRTT(VSIZE) @ Bandwidth[0%, 50%, 100%] + ClientRTT determines user responsiveness, e.g. mouse clicks in a browser + This requires small messages or threaded read / write (async has to wait for + the server finishing sending its message) +- Bandwidth(VUP2DOWN, VSIZE) + This depends on the network in use and operating system + [(up2down, msize) for up2down in VUP2DOWN for msize in VSIZE] +- Serverlatency(VSIZE) @ Bandwidth[0%, 50%, 100%] + The outliers (due to e.g. semi-random garbage collection) determines + choice of message size and buffers for media streams +- Clientlatency() @ 100% bandwidth @ Up2Down + The outliers (due to e.g. semi-random garbage collection) determines + choice of message size and buffers for long-running calculations which + connects to a server for distribution of results +=# diff --git a/benchmark/benchmark_prepare.jl b/benchmark/benchmark_prepare.jl new file mode 100644 index 0000000..01dc129 --- /dev/null +++ b/benchmark/benchmark_prepare.jl @@ -0,0 +1,316 @@ +# See benchmark.jl for definitions + +# This file collects data and tests functions. +# The intention is preparing input for a benchmark suite, +# but the output actually is sufficient for most purposes. +# +# Both log file, results plots and tables are printed to the same file. +# Viewing the plots in a text editor is probably possible, see UnicodePlots.jl. +# Running this file on a Windows laptop with all browsers takes 5-10 minutes + +"A vector of message sizes" +const VSIZE = reverse([i^3 * 1020 for i = 1:12]) +const SRCPATH = Base.source_dir() == nothing ? Pkg.dir("WebSockets", "benchmark") :Base.source_dir() +const LOGGINGPATH = realpath(joinpath(SRCPATH, "../logutils/")) +SRCPATH ∉ LOAD_PATH && push!(LOAD_PATH, SRCPATH) +LOGGINGPATH ∉ LOAD_PATH && push!(LOAD_PATH, LOGGINGPATH) +include(joinpath(SRCPATH, "functions_open_browsers.jl")) +include(joinpath(SRCPATH, "functions_benchmark.jl")) +# +prepareworker() +# Load modules on both processes +import HTTP +using WebSockets +using ws_jce +using UnicodePlots +import IndexedTables.table +import ws_hts: listen_hts, getws_hts +# +remotecall_fetch(ws_jce.clog, 2, "ws_jce ", :green, " is ready") +# Start async HTS server on this process and check that it is up and running +const TIMEOUT = Base.Dates.Second(20) +hts_task = start_hts(TIMEOUT) + +""" +Prepare logging for this process + + logs are written to console as well as to a file, which + is different for the other process. + Logs in other processes appear in console with a delay, hence the timestamp + is interesting. + To write log file buffer to disk immediately, call zflush(). + To drop duplicate console log, use zlog(..) instead of clog(..) +""" +const ID = "Benchmark" +const LOGFILE = "benchmark_prepare.log" +import logutils_ws: logto, clog, zlog, zflush, clog_notime +fbm = open(joinpath(SRCPATH, "logs", LOGFILE), "w") +logto(fbm) +clog(ID, "Started async HTS and prepared parallel worker") +zflush() + + +# +# Do an initial test run, HTS-JCE, brief text output +# + +const INITSIZE = 130560 +const INITN = 200 + +# Measured time interval vectors [ns] +# Time measurements are taken directly both at server and client +(testid, serverlatencies, clientlatencies) = HTS_JCE(INITN, INITSIZE) +# Calculate speeds [ns/b] and averaged speeds (bandwidths) +(serverspeeds, clientspeeds, + serverbandwidth, clientbandwidth) = serverandclientspeeds(INITSIZE, serverlatencies, clientlatencies) +# Store plots and a table in dictionaries +vars= [:serverspeeds, :clientspeeds]; +init_plots = Dict(testid => lp(vars, testid)); +init_tables = Dict(testid => table(eval.(vars)..., names = vars)); +init_serverbandwidths = Dict(testid => serverbandwidth); +init_clientbandwidths = Dict(testid => clientbandwidth); +# Sleep to avoid interspersing with worker output to REPL +sleep(2) +# Brief output to file and console +clog(testid, " Initial test run with messagesize ", INITSIZE, " bytes \n\t", + "serverbandwidth = ", :yellow, round(serverbandwidth, 4), :normal, " [ns/b] = [s/GB]\n\t", + :normal, "clientbandwidth = ", :yellow, round(clientbandwidth, 4), :normal, " [ns/b] = [s/GB]") + +# +# Continue initial test run with HTS-BCE, brief text output for each browser +# + +COUNTBROWSER.value = 0 +serverbandwidth = 0. +clientbandwidth = 0. +serverbandwidths = Vector{Float64}() +clientbandwidths = Vector{Float64}() +t1 = Vector{Int}() +t2 = Vector{Int}() +t3 = Vector{Int}() +t4 = Vector{Int}() +browser = "" +success = true +while success + # Measured time interval vectors [ns] for the next browser in line + # Time measurements are taken only at server; a message pattern + # is used to distinguish server and client performance + (browser, t1, t2, t3, t4) = HTS_BCE(INITN, INITSIZE) + if browser != "" + # Calculate speeds [ns/b] and averaged speeds (bandwidths) + (serverspeeds, clientspeeds, + serverbandwidth, clientbandwidth) = + serverandclientspeeds_indirect(INITSIZE, t1, t2, t3, t4) + # Store plots and a table in dictionaries + testid = "HTS_BCE " * browser + push!(init_plots, testid => lp(vars, testid)); + push!(init_tables, testid => table(eval.(vars)..., names = vars)); + push!(init_serverbandwidths, testid => serverbandwidth); + push!(init_clientbandwidths, testid => clientbandwidth); + # Brief output to file and console + clog(testid, " Initial test run with messagesize ", INITSIZE, " bytes \n\t", + "serverbandwidth = ", :yellow, round(serverbandwidth, 4), :normal, " [ns/b] = [s/GB]\n\t", + :normal, "clientbandwidth = ", :yellow, round(clientbandwidth, 4), :normal, " [ns/b] = [s/GB]") + else + success = false + end +end + +# +# Collect HTS_JCE bandwidths vs message size +# + +const SAMPLES = 200 +serverbandwidths = Dict{String, Vector{Float64}}() +clientbandwidths = Dict{String, Vector{Float64}}() +for msgsiz in VSIZE + # Measured time interval vectors [ns] + # Time measurements are taken directly both at server and client + (testid, serverlatencies, clientlatencies) = HTS_JCE(SAMPLES, msgsiz) + # Find averaged speed (bandwidth) scalars + (_, _, + sbw, cbw) = serverandclientspeeds(msgsiz, serverlatencies, clientlatencies) + # Assign storage + !haskey(serverbandwidths, testid) && push!(serverbandwidths, testid => Vector{Float64}()) + !haskey(clientbandwidths, testid) && push!(clientbandwidths, testid => Vector{Float64}()) + # Store bandwidths from this size + push!(serverbandwidths[testid], sbw) + push!(clientbandwidths[testid], cbw) +end + +# +# Collect HTS_BCE bandwidths +# For individual and available browsers +# This opens a lot of browser tabs; there may +# be no easy way of closing them except +# by closing Julia. internet explorer even +# opens a separate window every time. +# +for msgsiz in VSIZE + COUNTBROWSER.value = 0 + t1 = Vector{Int}() + t2 = Vector{Int}() + t3 = Vector{Int}() + t4 = Vector{Int}() + browser = "" + success = true + while success + # Measured time interval vectors [ns] for the next browser in line + # Time measurements are taken only at server; a message pattern + # is used to distinguish server and client performance + (browser, t1, t2, t3, t4) = HTS_BCE(SAMPLES, msgsiz) + testid = "HTS_BCE " * browser + if browser != "" + # Find averaged speed (bandwidth) scalars + (_, _, + sbw, cbw) = + serverandclientspeeds_indirect(msgsiz, t1, t2, t3, t4) + # Assign storage + !haskey(serverbandwidths, testid) && push!(serverbandwidths, testid => Vector{Float64}()) + !haskey(clientbandwidths, testid) && push!(clientbandwidths, testid => Vector{Float64}()) + # Store bandwidths from this size + push!(serverbandwidths[testid], sbw) + push!(clientbandwidths[testid], cbw) + else + success = false + end + end +end + + + +# +# Measurements are done. Close server and log file, open results log file. +# +ws_hts.close_hts() +clog(ID, "Closing HTS server") +const RESULTFILE = "benchmark_results.log" +clog(ID, "Results are summarized in ", joinpath(SRCPATH, "logs", RESULTFILE)) +fbmr = open(joinpath(SRCPATH, "logs", RESULTFILE), "w") +logto(fbmr) +close(fbm) + + + +# +# Find optimum message size and nominal 100% bandwidths +# Make and store plots and tables +# +test_bestserverbandwidths = Dict{String, Float64}() +test_bestclientbandwidths = Dict{String, Float64}() +test_bestserverlatencies = Dict{String, Float64}() +test_bestclientlatencies = Dict{String, Float64}() +test_plots = Dict() +test_tables = Dict() +test_latency_plots = Dict() +test_latency_tables = Dict() +serverbandwidth = Vector{Float64}() +clientbandwidth = Vector{Float64}() +serverlatency = Vector{Float64}() +clientlatency = Vector{Float64}() +bestserverbandwidth = 0. +bestclientbandwidth = 0. +bestserverlatency = 0. +bestclientlatency = 0. + +for testid in keys(serverbandwidths) + serverbandwidth = serverbandwidths[testid] + clientbandwidth = clientbandwidths[testid] + # Store the optimal bandwidths in a dictionary + bestserverbandwidth = minimum(serverbandwidth) + bestclientbandwidth = minimum(clientbandwidth) + push!(test_bestserverbandwidths, testid => bestserverbandwidth); + push!(test_bestclientbandwidths, testid => bestclientbandwidth); + # Store msgsiz-bandwidth line plots and tables in dictionaries + vars = [:serverbandwidth, :clientbandwidth] + tvars = vcat([:VSIZE], vars) + push!(test_plots, testid => lp([:VSIZE, :VSIZE], vars, testid)); + push!(test_tables, testid => table(eval.(tvars)..., names = tvars)); + # Store msgsiz-latency line plots and tables in dictionaries + serverlatency = serverbandwidth .* VSIZE + clientlatency = clientbandwidth .* VSIZE + bestserverlatency = minimum(serverlatency) + bestclientlatency = minimum(clientlatency) + push!(test_bestserverlatencies, testid => bestserverlatency); + push!(test_bestclientlatencies, testid => bestclientlatency); + vars = [:serverlatency, :clientlatency] + tvars = vcat([:VSIZE], vars) + push!(test_latency_plots, testid => lp([:VSIZE, :VSIZE], vars, testid)); + push!(test_latency_tables, testid => table(eval.(tvars)..., names = tvars)); + + # Brief output to file and console + clog_notime(testid, :normal, " Varying message size: \n\t", + "bestserverbandwidth = ", :yellow, round(bestserverbandwidth, 4), :normal, " [ns/b] = [s/GB]", + " @ size = ", VSIZE[findfirst(serverbandwidth, bestserverbandwidth)], " b\n\t", + :normal, "bestclientbandwidth = ", :yellow, round(bestclientbandwidth, 4), :normal, " [ns/b] = [s/GB]", + " @ size = ", VSIZE[findfirst(clientbandwidth, bestclientbandwidth)], " b\n\t", + "bestserverlatency = ", :yellow, Int(round(bestserverlatency)), :normal, " [ns] ", + " @ size = ", VSIZE[findfirst(serverlatency, bestserverlatency)], " b\n\t", + :normal, "bestclientlatency = ", :yellow, Int(round(bestclientlatency)), :normal, " [ns]", + " @ size = ", VSIZE[findfirst(clientlatency, bestclientlatency)], " b\n\t" + ) +end + + +# +# Full text output +# The plots are not currently readably encoded in the text file +# + +clog_notime(ID, :bold, :yellow, " Plots of all samples :init_plots [ns/b], message size ", INITSIZE, " b ", SAMPLES, " samples" ) +foreach(values(init_plots)) do pls + foreach(values(pls)) do pl + clog_notime(pl) + end +end +clog_notime(ID, :bold, :yellow, " Tables of all samples, :init_tables, message size ", INITSIZE, " b ", SAMPLES, " samples" ) +for (ke, ta) in init_tables + clog_notime(ke, "\n=> ", ta, "\n") +end + + +clog_notime(ID, :bold, :yellow, " Plots of varying size messages :test_plots [ns/b],\n\t VSIZE = ", VSIZE) +foreach(values(test_plots)) do pls + foreach(values(pls)) do pl + clog_notime(pl) + end +end + +clog_notime(ID, :bold, :yellow, " Tables of varying size messages :test_tables [ns/b]") +for (ke, ta) in test_tables + clog_notime(ke, "\n=> ", ta, "\n") +end + +clog_notime(ID, :bold, :yellow, " Plots of varying size messages :test_latency_plots [ns],\n\t VSIZE = ", VSIZE) +foreach(values(test_latency_plots)) do pls + foreach(values(pls)) do pl + clog_notime(pl) + end +end + +clog_notime(ID, :bold, :yellow, " Tables of varying size messages :test_latency_tables [ns]") +for (ke, ta) in test_latency_tables + clog_notime(ke, "\n=> ", ta, "\n") +end + +clog_notime(ID, :bold, :yellow, " Dictionary :test_bestserverlatencies [ns]") +for (ke, va) in test_bestserverlatencies + clog_notime(ke, " => \t", Int(round(va))) +end + +clog_notime(ID, :bold, :yellow, " Dictionary :test_bestclientlatencies [ns]") +for (ke, va) in test_bestclientlatencies + clog_notime(ke, " => \t", Int(round(va))) +end + +clog_notime(ID, :bold, :yellow, " Dictionary :test_bestserverbandwidths [ns/b]") +for (ke, va) in test_bestserverbandwidths + clog_notime(ke, " => \t", round(va, 4)) +end + +clog_notime(ID, :bold, :yellow, " Dictionary :test_bestclientbandwidths [ns/b]") +for (ke, va) in test_bestclientbandwidths + clog_notime(ke, " => \t", round(va, 4)) +end + +zflush() diff --git a/benchmark/favicon.ico b/benchmark/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..6d1163dd60d650c353dabe327514d25254e94979 GIT binary patch literal 99678 zcmeHQ3wRvWbsmQ#CJz$QkmM^N=waD1mbJ4hVQWVwm69eTwzTpC1f+QzmnLbG=1coE zZJNegkXBkrwq+SYT#)34Fra{;rAbOK+LdGsHkMx)4A_=sgKZ4P7=ytN+3J60_pa^^ zGqW?hGpm&}`+axs%$@r<=ltj1xsP-22*L#6BZA*Az|R(@UOGXzNf3nc^633n1>rXc zbGsw=4@?$>b#o9iJ9>Y)AXNQ)f^a!}O8=t|3&MN;3Bpw5;fF(@98Q;NzoDTa!EufZ z9360Uz|jFm2OJ%6bimO8M+Y1oaCE@Y0Y?WM9dLBO(E鍠Uz|jFm2OJ%6bimO8 zM+Y1oaCE@Yfzhu6Yi{)jcDRm(9%o(IqP2CBr5zYio@;M)+u}MFx*hy2!`~i^?82V% zb?~?D-9WOm6DvFNcJO!bclw9c=zvrIt&!ts)kg<^2Y)->(YEHf$Ql1x)!wXyEN#Uq z(HL-%GnTUo%XHdqv>OM#@blPA20o&fG8sn7^Qo zd-%Ld>U~p`HXatGrv^l6(_XiitFaw4u96uac2hqBLuK^;ZM) zJ~iA?p9j~@4ZgK*e$$Z$7AzU)ym9H?2X9!m?|~bZ99%cQ@#vbl^{3h^1M2OBuXb&F zx&P8c<(qw;srXMX34h&H5_&5hKf<%;xF|jIq9|?YbC*85Ro`vd4!!6Xz1{D{=Oecd z;o{=Z6=vn77k^b%U>4rv$CTIu^Anxb%U*iq<`sR9-g4(wJ@?2>E4Fvuuyp_Gs`_`8 z`QUy_YgO<{JsQkD%S8OqZ~HsoHOONoNp4C6#~{;5OB;vS!FGy0A_G*00AkIxm9hf?Zqw^jxuU=)Yb{R^62)x*~Cvzu3J zqjv_c1L#louk`sXX}dBBf7h(8d|-2u!)H*Go_;wVL+v_lBp$stM5&j?i7_3(*kHXN z3{6O;{`zIhG9dIlXpJJ4X3 zW@Y?L!C&--f6C+GlAb!446a7gh|;rt=nuTh%EVmK+f!$5_^eVsdiUthf0xJJvNm6* zO+cT5=8RZ>;LN>qYO?fbO!O=h@Ta*Q;4nPCM_{eC_Kwx{T`j-$H?{1baU?4j{%S^3GJZzjFNOb&)k7My1DB-x`DXP2a|OeC=MBB@ zne}HwOl{uqU79 zc2dLB{P=+8lsFpxq$VdmZ1w!WXCeDo8*FEvPfG{l`+_w)c>2s-k2L<#x&7!^EPOuQ z*i(6!8`{s|e}?)adNM@Y|7}+HXukGQ)JH9x>Yvk}KXvW6SQ zzvuXf;9t6BJHJ*SmW02eB`*p`d;2>kd+Lwdg|S|~G?(0lxdui@dqwRndNgZ28)^Je z?|NC?!#d8ZBY{7a>5{@{7#)}k__mh3Wc({@9%FK+c32N5JD;ttsyof-K!4<)^k~+4 zHq!V|W>&_<8gXph0d3M9TJoZB$b2u8JFVl}iLqYZz#q0_vAMu|^k~+4mI3%< z4lcSDWAmJ^p1xo%7b|K* zv(~c=z<>U*>@oL`vA7mac7iwNZwJtFt z=7A1bg>RyK6!w6x!zTDLBQ&eU$prj~E>}soJEjAei*=`?A0Y1+;OqdUuD?-}kyaS$ z|NjiP0fF<>CrgA~s?U|{K5f35pJ~x&Bs^2_cYC{L#qh^CT=e$5ZOq2s{CY371L*TV z51X^tx6|lJzqQ|ELXcY&|g29?*WDE!$GZJkm~3z312Zs0>l+T9~Z-WE)^^J&^FB zzPXh)0T17_{5b{o=ywGEC0UyJbeV)d(d8gEuw%@S^Eb=gc9>(E3*|AMydRX;De7Z{R#aDaUZN%!vtg?%Yre>@3lmN>M!Rob<1wIZal~JX_x#N?wqW_B)2DNPS}SUPj`zX_^bdw@_7!zJrVF&L zKy%9#wckm0O~(v>qX~bWufpqkKZ8Acv!M&OLKptM(A(AGD(Py#{QWQRPWYB6Jvo!! zAAFjTzV4Xq|5U55?s|;feuy(&eg%IybfT%v7x-tgDW~%z{N;W_^UqfCGFtJsihr$q z@>)thmAgcolHK4b%V~7^vKw6`IZd9dyq3I)T4_dOxPyP>{J;EQ{$;sAaRKmd&1n#i z<}|p6blmow2G1SY!6L8V7c#1iF{^{)9sem!#a96DCb;)?@Yd6XgV38F3Z2!K%p;h^ zFV*gzr5s?f;(qD@A~a^+g*2mc`I{;zrAkfv=yZmz*k2$_f$t86wGqGI&e9j4FK;XI!n;D;aMjX+zesmoPNux^gnw30 z{0gu-!{Y_~P8QyE-7X_|YsHziLV6kRh_igYxPBqGzWCG0V3N)>;|2e$Mpq6nInHC0 z-#p`}7N%yysqLV52cLg#(EZ0uA0Lbd{J+rX`UJ*nyLfD_S?)bx7GneP`=A3S_;P4$ zk5em&@y`x=e#EfPp}K$f zm$#|mWLFr*h5KXWH@Gk41w5ZiH@=C=8_W1(p67p={3kEH_D#ER*2%Yz^p4R$>ja7L zNcZ!xioZO+-{5|a$=;^%Uc$a1^*0#}*^Qou(~WPU^2RFudG%relRNpT&hhEgaa|oi zpW_gtBfG)9I}sk~em++5$N1}OOzyOHmn?p%rb8cKZ;Xxx_gm@4H&J)E{H6lg_`B(ajlRBp%5st||AG7#3cz(*{PV;fe;+JYVdhakg$fq%BvRK)X zZmi;;+u->YlRM4DrHW&+>6P`L`lWW@m~5V7760g3Z^UkEB;Wrn(mtk#@Q){plU?b? zD*o9`#Zx)CXGH&>)*$V|G1+{_D*nYQi$4PVW9#vmF#gB<&;7~bWLLVeia*JV=37`B zKUw+-cAU^Gyl#4a3@&37#0PxGxz_VWC{!k#k)Ty^JsSiXui!1LM5+?OYd zlU?b?GX9eqU7rX3v9*6I?Y$VrI)2}zc$>gK+l65&`Hp4$NiGW8p9^Het%I4NdTk^< zKkpl9gIu=mIZsvx#ybAdHsD?M9v~Y{;MZpa!(`(W+%;=AzwU>5|34&)kKO6UI{tDS z&>;RN!=L;q%dUMrS!@%gLmy*OlPC7hL3x8Y^&Zh~9Fxs|Jm6pMFaI!b2=O{VdwddN z$?g3y7@4w$APOQ?}F)f&3bN9Q&G&`fOIXp z(Vg%t>2zSMSH^h3pJb5JSmXjOC;4_Duc_!=dS4r?->Ut?Yg)Yh{QF+cANGC){(=!X zb3_ks<+JgGKk+Wl3kTgV@j5`)lhGEK)Pp|S_ocM;r@UyM@2aKJ$CR?p#QX7vzpMk8 zhrzy_KPh#9?#X7C_JR_&vOYxTjA(xs?Fpv3PPT+tQ;gXabNO7r9N)iYBFh7#uf&Ji1FFB!fm+xL?tmZHm*;c?_Pp}Dc%m}xUaUBcY4*sZzR_GMT za-?ufwp<5)Yp^H%aqzcJmR8iqbm!sV@8F+oJ3Lb9oc7;3optKJb+Sx%+igX84*m}Q zcD3VH=%hj}QDi7I_3UcmG09#_K=Ps75=58P)Z%Iv%i-f7i_J9MRjg0PYus-tg?{B^$nA zg&aGpmwv3>SL1D~2>iI!7r3>}S6kFs?N92QN~`&IxgP&Sp|`8WOo zzw7e$9u=ic``ywrFTg+Gk~Y5P@`m4wJP-G}OS&f;p>?It|8elz44f~t`vT`W<^_-6 zzo6;R?{92=v9o&FOSpceV_w6Xh(mHY+g?%o%g*Xfn&n9-U#s{BPmAC#g1>XbtGo2s zt!hq`w)Bb8Q?H8N&>-T!KV5pVRFBT~a{r~^={dNewR3|f*WcW-^U+)G+^Xgtu5RAF zrmF58irZRQ^EkLqsAE>}?!~d`#OJHLPL=;cDg3xg3ZGH(W%M5Hz%Jrn^oD~xZTD8y zc|g0VdaslR+JV<8k9)u2{|i2E{yJ6uq5p4^4jbiHEnc)8e1fM-?$?0!-`}|GWhGxm z?;lvOWPtczRayJjeBS(Z>imaZb$NSEsqtkr4E&#m4*U;KSG%wFZ;1BJ8<*`h%C9mW z^${qKwu+ikK5zayW&WkmqlBMPUX}5n|GVY-?+y8Ry3o&=O0=(=-*{Nbm(hFbCs7-? z=K97@@Oksssq!yMUB8j*FZfXMX5=34!2SsTT~$0?e!u@i;Q#d6x%H;reY|W|SJl0T z{=*(VU*&bG{JXtfvt;_E-V;Xn<>QFbvwhTOLi+byrH_AJNgMj}kF`|>ET*d3 zs4qi(8Y6sDoJVe2u^sj2EM#}IeV$+Udv#Xu?=BBrE=u7e^d7|fd`}|2={fX%zufQf zhBjhF@B@@w^EjgxTP&*Ip@Kg*l^ z!WkQKXLh zz9)Uh1>f*Ejd1qe-PLw|cXcHA$2WyO3ErNN`KE6d-C6pA>bHyV-I6IwuYH5!;k)BU zvl=~;Rk*W!MuLBQOYyhFQ}kP|3BTP%-*LgWTFxTPL6RM#-3mY5{Nr1eU&XfxE}H$8 z^|TggUqlbwcUr+W%Ojoq)3>ys#|QDvmb3IN9#x&!`CcD=vxj77us0;NBzf}En@(Wv8yEhsAsru>UAAOqj@;BUd>Nvw$%cs5p_&-nI15K58N}o>t!TWy@{wBXs7x%rw zXuoKu6}+dLfBMD}@lW4h)514FcpuVVu=1On>BL|Dj&p zo{dKQiDrCvsfBc4YKwP(@nyuPcOTmRb7({M85tiMl}|eHCz`JG7k?c1?Snf{-xC!T9)J(AItJi-DP+WpV8Cjqd^ql1v8R#RxPa7EU3zLBZe#*02BB2)l(>W1txF5hd=R`Q|4%92-$M_D7 z<}s4L8@~wOo7Qk6xwnpBER0t}seNtFP{Q*yRUd zUmk|r-C9xGPWudDn|fKIJd&k9f7Zs!U_*Nk+@L7+yiWUSvES~HDD{#(E$uTt;VKE; zs8@d5?7-)+r{A%pl=^+ zq2JcpA1QjDdIfx7FKE}V)XK;H-zz}lG1xetpgpci`)}GCK>K813w!~4e;2CdL1D?z z4;^U&ZKt&D*`xb>vG)Odp2r?{m$H29L-;*xwofM7!w}uCb$X4j{%U0&$;_F=E*^I+Fz1w6=YDD&tJt5EbLEi?c zonqxecfwRx)o4`Pwt$9PJ}rK4f|p z_23BY*Hgnmlr}#{dwsy`2A+0y9XYonFsN4#Xumr4xt(gStm$CslzuYwQy!4%9O%(; z?6Jmvq)q#2Itc{}Y@qy0&Eb`CoKf1WS*d`?*=yT^GF--c}8SzT3sg4)Q3Z(9Bw z?X#o(SofiQg3b)053x?KACxTpG7s1Feg=E&T0txNKe7EpbcV(ru_XLmMyH-n`-jjT zb)tPaL2ab$&>zD64lSMG^Gk((J|EiaSunFp%$xm3n%0WzcherAoS-Wowh+~@^DED- zFIL^pB=a)o_CmFGMwutOw~Bt2AMXpD)tBVdyMG0`Y2OO%*MRRCq-)r6Y`wb4W9nQQ zK5uqy7yV>k1H1Zdu*Z7;8rT?ew)uSpcg}hqcK2^!pUyehZT*7Ll8kQ${iH7>-@K-x z6CB+iu%E;plKq4SY_8@cLp!ChgZ`YL=bvB;_nsc@%swu^dFHFor`NRVL?XUyp&#^^ zu;uK-KCe@`4Wi~4MUMxo=_mUts-I@;u~8SuCNHTyI9Ai2-QXz$Ef<)681eI)OAezR zY|@jn)-$W=pHx?rNAfqK`;52AOJ^KIJEeQ3O}gkOdrZ)On)atL`V#qQr;byd*P>A` z+-mt#S)jiY_NQ+qqMM!*pNK!0S1%Um(W&*!7W!+ae*rX|%uleVigXO`iFb2??iwxn z5{27AKb4sq6c^C@m2_KeZxH0Z7dlVUN9Ta~_eIzZ~jD*`sKb0@{;Tpv2iB|N79wd8nw3XkZJ=#X-OB6?s zezkesjzuG9TT5lD(PSoU^sQgSuMcE8{fYF=c^*HubmZa4!_fgtb%4Z4^EcM`C$r9c zn(_;OdSD{?{)w==BL9-`1oC0#74av;d<0TA+#m0=cu}9_6P`FA=QogbQZ8rkU;nC% z(#vLE$nW%)B&*Q%9+F!es0_4W?xKNTt`SKa}(Y&cF zB|RMr-I_9|^QpucT?OUkp%1KEDe0p*{?^KRQSV$yD$7XuSR48#&dtMFc-{S2E8UB= z7OY2v@5(L@U8*d1)$BVaVJ&Mj*3u8*jJv(?U%}jc&poq)vGqEZ#z^^CH~67T>VEYD z>ub_ew3fdWEfkrb$}(%Oykj!TK0@m>{F)QiKlb3vxx>o!)_u=+pZ0l;IL90GJx%3z%=Ui@>puNF-^%ON zzPbT^Eh16*Sm*vVo&BlCI}1bk{Zu~b8t{HeStfsvHU7Q#&Z)^_@=8>Gp%j`2-P7bh zaxQVWm+~V$hprvq%Tr!s?d~9*S!%R=Ix`ad9dc{n58C!aH^0a7!@AXzq-)A@>Hfh5 zOJBnChgn?m6Y)=Rur4?k_3jl~zhp9}HBJ$_2ioG!BPRWZ{vRY=<7uF>5#GOgejskI zu95PI26t(A2Fk*^d-xF6P!Es~nif~9oZr7@zW?*!^G{UYfOkKx55qljUq#(#c;1cj z&!;Iae(dA?8GSU*l+`HC$NbF_%uW4UcCg6H;>%~#&aL?pT83Erz@9PRDE}-z?@wwf zDn_|`r!>#(r@60b%ccF3mlhw$Z7O=q=-NlJ&9JPG=L2<2{5mQ812K>v2q znx`7id|0yOQy#gE;tx^Rj!@krU7)&WWQihGe#_n-x_C)= z@!$l1*%{ekAKV1y{AGie_{-?|(C<9hc}Dx_$YCl&>FC*+ER_NxK{QWsIb62Q^Aznr zm?4l`{k@rn>o&z@=_2(rGHwLrhVtOk&^0>`B4YR5`!n?E?%?g~SIxU)(%O0TOVDro zF8f9krD1IYOD{^@>(MvfL3Y>Zk3LQLQCir-?jSpAI_DgA>~|BL%JlMg$D~cewt5)L zz_$5ER4#9WegFK%jVp^+engqRprG@kD6?z0PrRwmQ_?f7t*ZVzR4(iP!Y=etwel&< z?d|?PYzF@X`#PM1-R-9^;+JTqaw#pw4A*O=r*LvK9?xsMwjBMxRd9DtURHDs(afGF zN}nIhzievr%xBoUbjq?BasB#4>9Ll%2y4eXS^8^MNV{_C3q+&kPiZQ?0&CNoS^E5C zGoMtxzm@Xm^QP-bjhp@(z_~idT_kQIk literal 0 HcmV?d00001 diff --git a/benchmark/functions_benchmark.jl b/benchmark/functions_benchmark.jl new file mode 100644 index 0000000..0019189 --- /dev/null +++ b/benchmark/functions_benchmark.jl @@ -0,0 +1,306 @@ +# Included in benchmark.jl +"Adds process 2, same LOAD_PATH as process 1" +function prepareworker() + # Prepare worker + FULLLOADPATH = LOAD_PATH + if nworkers() < 2 + addprocs(1) + end + futur = @spawnat 2 for p in FULLLOADPATH + p ∉ LOAD_PATH && push!(LOAD_PATH, p) + end + # waits for process 2 to get ready, trigger errors if any + fetch(futur) +end + +"Start and wait for async hts server" +function start_hts(timeout) + hts_task = @schedule ws_hts.listen_hts() + t1 = now() + timeout + while now() < t1 + sleep(0.5) + isdefined(ws_hts.TCPREF, :x) && break + end + if now()>= t1 + msg = " did not establish server before timeout " + clog("start_hts", :red, msg, timeout) + error(msg) + end + hts_task +end + + + +""" +Make and get a reference to the server side of HTS-JCE websocket connection. +Assumes a server is already running. +""" +function get_hts_jce() + id = "get_hts_jce" + # Make the parallel worker initiate a websocket connection to this process. + # It will finish execution when either peer closes the connection. + @spawnat 2 ws_jce.echowithdelay_jce() + # async HTS will not accept the connection before we yield to it + zlog(id, "JCE will be connecting to HTS. We ask HTS for a reference...") + zflush() + hts = Union{WebSocket, String}("") + t1 = now() + TIMEOUT + while now() < t1 + yield() + hts = ws_hts.getws_hts() + isa(hts, WebSocket) && break + sleep(0.5) + end + return hts +end +""" +Make and get a reference to the server side of HTS-BCE websocket connection. +Will launch a different web browser every time. Returns a string +if all available browsers have been launched one time. + +Assumes a server is already running. +""" +function get_hts_bce() + id = "get_hts_jce" + hts = Union{WebSocket, String}("") + browser ="" + opened, browser = open_a_browser() + # Launch the next browser in line. + if opened + zlog(id, "BCE in, ", browser, " will be connecting to HTS. Getting reference...") + zflush() + hts = Union{WebSocket, String}("") + t1 = now() + TIMEOUT + while now() < t1 + yield() + hts = ws_hts.getws_hts() + isa(hts, WebSocket) && break + sleep(0.5) + end + return hts, browser + end + return hts, browser +end +""" +Collect n (samples) of serverlatency and clientlatency. +Also calculate derived sizes + serverbandwidth, clientbandwidth, serverspeeds, clientspeed +Starts a new JSE-HTS connection for every evaluation (call). +""" +function HTS_JCE(n, messagesize) + id = "HTS_JCE" + zlog(id, "Warming up, compiling") + zflush() + hts = get_hts_jce() + if !isa(hts, WebSocket) + msg = " could not get websocket reference" + clog(id, msg) + error(id * msg) + end + clog(id, hts) + # Random seeding, same for all samples + srand(1) + clog(id, "Sending ", n, " messages of ", messagesize , " random bytes") + sendtime = Int64(0) + receivereplytime = Int64(0) + sendtimes = Vector{Int64}() + receivereplytimes = Vector{Int64}() + for i = 1:n + msg = rand(0x20:0x7f, messagesize) + sendtime = time_ns() + write(hts, msg) + # throw away replies + read(hts) + receivereplytime = time_ns() + push!(receivereplytimes, Int64(receivereplytime < typemax(Int64) ? receivereplytime : 0 )) + push!(sendtimes, Int64(sendtime < typemax(Int64) ? sendtime : 0 )) + end + # + zlog(id, "Sending 'exit', JCE will send its time log and then exit.") + zflush() + write(hts, "exit") + # We deserialize JCE's time records from this sample + bs = BufferStream() + write(bs, read(hts)) + close(bs) + if nb_available(bs) > 0 + receivetimes, replytimes = deserialize(bs) + else + error("Did not receive receivetimes, replytimes") + end + # We must read from the websocket in order for it to respond to + # a closing message from JCE. It's also nice to yield to the async server + # so it can exit from it's handler. + isopen(hts) && read(hts) + yield() + serverlatencies = receivetimes - sendtimes + clientlatencies = receivereplytimes - replytimes + return id, serverlatencies, clientlatencies +end +""" +Use the next browser in line, starts a new HTS-BCE connection and collects n evaluations. +This is one sample. +Returns browser name and vectors with n rows: +# t1 Send 0, receive 0, measure time interval +# t2 Send 0, receive 0 twice, measure time interval +# t3 Send x, receive 0, measure time interval +# t4 Send x, receive 0, receive x, measure time interval +""" +function HTS_BCE(n, x) + id = "HTS_BCE" + zlog(id) + zflush() + msg = "" + (hts, browser) = get_hts_bce() + if browser == "" + msg = "Could not find and open more browser types" + elseif !isa(hts, WebSocket) + msg = " could not get ws reference from " * browser * " via HTS" + end + if msg != "" + clog(id, msg) + return "", Vector{Int}(), Vector{Int}(), Vector{Int}(), Vector{Int}() + end + clog(id, hts) + # Random seeding, same for all samples + srand(1) + clog(id, "Sending ", n, " messages of ", x , " random bytes") + st1 = UInt64(0) + st2 = UInt64(0) + rt1 = UInt64(0) + rt2 = UInt64(0) + rt3 = UInt64(0) + rt4 = UInt64(0) + + t1 = Vector{Int}() + t2 = Vector{Int}() + t3 = Vector{Int}() + t4 = Vector{Int}() + + for i = 1:n + # Send empty message, measure time after two empty replies + msg = Vector{UInt8}() + st1 = time_ns() + write(hts, msg) + read(hts) + rt1 = time_ns() + read(hts) + rt2 = time_ns() + + # Send message, measure time after one empty and one echoed replies + msg = rand(0x20:0x7f, x) + st2 = time_ns() + write(hts, msg) + # throw away replies. We expect a sequence per sent message: + # two empty messages, then the original content. + read(hts) + rt3 = time_ns() + read(hts) + rt4 = time_ns() + # Store time intervals + push!(t1, Int(rt1 - st1)) + push!(t2, Int(rt2 - st1)) + push!(t3, Int(rt3 - st2)) + push!(t4, Int(rt4 - st2)) + end + # + zlog(id, "Close websocket") + close(hts) + # Also yield to the server task so it can release it's reference. + sleep(1) + # Return browser name and measured time interval vectors [ns] + browser, t1, t2, t3, t4 +end + +" +Measured time interval vectors [ns] -> client and server speeds, bandwidth [ns/b] +x is message size [b]. +" + +function serverandclientspeeds(x, serverlatencies, clientlatencies) + serverspeeds = serverlatencies / x + clientspeeds = clientlatencies / x + n = length(serverspeeds) + serverbandwidth = sum(serverlatencies) / (n * x) + clientbandwidth = sum(clientlatencies) / (n * x) + serverspeeds, clientspeeds, serverbandwidth, clientbandwidth +end + + +" +Measured time interval vectors [ns] -> client and server speeds, bandwidth [ns/b] +x is message size [b]. +" +function serverandclientspeeds_indirect(x, t1, t2, t3, t4) + # assuming a measured time interval consists of + # t(x) = t0s + t0c + a*x + b*x + # where + # t(x) measured time at the server + # t0s initial serverlatency, for a "zero length" message + # t0c initial clientlatency, for a "zero length" message + # x message length [b] + # a marginal server speed [ns/b] + # b marginal client speed [ns/b] + # + # + # t1 = t0s + t0c Send 0, receive 0, measure t1 + # t2 = t0s + 2t0c Send 0, receive 0 twice, measure t2 + # t3 = t0s + t0c + a*x Send x, receive 0, measure t3 + # t4 = t0s + 2t0c + a*x + b*x Send x, receive 0, receive x, measure t4 + # + # hence, + t0s = 2t1 - t2 + t0c = -t1 + t2 + a = (- t1 + t3) / x + b = (t1 - t2 - t3 + t4) / x + # Drop the marginal speed info, return effective speeds + # serverspeeds = (t0s + a*x) / x + # clientspeeds = (t0s + a*x) / x + serverspeeds = t0s / x + a + clientspeeds = t0c / x + b + n = length(serverspeeds) + serverbandwidth = sum(serverspeeds) / n + clientbandwidth = sum(clientspeeds) / n + return serverspeeds, clientspeeds, serverbandwidth, clientbandwidth +end + + +## Note these shorthand function require input symbols defined at module-level +## (not in a local scope) + +"Shorthand for generating a time series lineplot in REPL" +lp(sy::Symbol) = lineplot(collect(1:length(eval(sy))), eval(sy), title = String(sy), width = displaysize(STDOUT)[2]-20, canvas = AsciiCanvas) + + +"Return multiple time series lineplots with a common title prefix" +function lp(symbs::Vector{Symbol}, titleprefix) + map(symbs) do sy + pl = lp(sy) + title!(pl, titleprefix * " " * title(pl)) + end +end + +"Shorthand for generating an x-y lineplot in REPL" +function lp(syx::Symbol, syy::Symbol) + lpl = lineplot(eval(syx), eval(syy), title = String(syy), width = displaysize(STDOUT)[2]-20, canvas = AsciiCanvas) + xlabel!(lpl, String(syx)) +end +"Return multiple x-y lineplots with a common title prefix" +function lp(syxs::Vector{Symbol}, syys::Vector{Symbol}, titleprefix) + map(zip(syxs, syys)) do pair + pl = lp(pair[1], pair[2]) + title!(pl, titleprefix * " " * title(pl)) + end +end + +"Shorthand for generating an x-y scatterplot in REPL" +function sp(syx::Symbol, syy::Symbol) + spl = scatterplot(eval(syx), eval(syy), title = String(syy), width = displaysize(STDOUT)[2]-15, canvas = DotCanvas) + xlabel!(spl, String(syx)) +end + +"Shorthand for generating a densityplot in REPL" +function dp(syx::Symbol, syy::Symbol) + dpl = densityplot(eval(syx), eval(syy), title = String(syy), color = :red, width = displaysize(STDOUT)[2]-15) + xlabel!(dpl, String(syx)) +end diff --git a/benchmark/functions_open_browsers.jl b/benchmark/functions_open_browsers.jl new file mode 100644 index 0000000..d22492d --- /dev/null +++ b/benchmark/functions_open_browsers.jl @@ -0,0 +1,174 @@ +# Included in benchmark_2.jl + +"A list of potentially available browsers, to be tried in succession if present" +const BROWSERS = ["chrome", "firefox", "iexplore", "safari", "phantomjs"] +"An complicated browser counter." +mutable struct Countbrowser;value::Int;end +(c::Countbrowser)() =COUNTBROWSER.value += 1 +"For next value: COUNTBROWSER(). For current value: COUNTBROWSER.value" +const COUNTBROWSER = Countbrowser(0) +const PORT = 8000 +const URL = "http://127.0.0.1:$PORT/bce.html" + +"Get application path for developer applications" +function fwhich(s) + fi = "" + if Sys.is_windows() + try + fi = split(readstring(`where.exe $s`), "\r\n")[1] + if !isfile(fi) + fi = "" + end + catch + fi ="" + end + else + try + fi = readchomp(`which $s`) + catch + fi ="" + end + end + fi +end +function browser_path_unix_apple(shortname) + trypath = "" + if shortname == "chrome" + if Base.Sys.is_apple() + return "Google Chrome" + else + return "google-chrome" + end + end + if shortname == "firefox" + return "firefox" + end + if shortname == "safari" + if Base.Sys.is_apple() + return "safari" + else + return "" + end + end + if shortname == "phantomjs" + return fwhich(shortname) + end + return "" +end +function browser_path_windows(shortname) + # windows accepts English paths anywhere. + # Forward slash is acceptable too. + # Searches C and D drives.... + trypath = "" + homdr = ENV["HOMEDRIVE"] + path32 = homdr * "/Program Files (x86)/" + path64 = homdr * "/Program Files/" + if shortname == "chrome" + trypath = path64 * "Chrome/Application/chrome.exe" + isfile(trypath) && return trypath + trypath = path32 * "Google/Chrome/Application/chrome.exe" + isfile(trypath) && return trypath + end + if shortname == "firefox" + trypath = path64 * "Mozilla Firefox/firefox.exe" + isfile(trypath) && return trypath + trypath = path32 * "Mozilla Firefox/firefox.exe" + isfile(trypath) && return trypath + end + if shortname == "safari" + trypath = path64 * "Safari/Safari.exe" + isfile(trypath) && return trypath + trypath = path32 * "Safari/Safari.exe" + isfile(trypath) && return trypath + end + if shortname == "iexplore" + trypath = path64 * "Internet Explorer/iexplore.exe" + isfile(trypath) && return trypath + trypath = path32 * "Internet Explorer/iexplore.exe" + isfile(trypath) && return trypath + end + if shortname == "phantomjs" + return fwhich(shortname) + end + return "" +end +"Constructs launch command" +function launch_command(shortbrowsername) + if Sys.is_windows() + pt = browser_path_windows(shortbrowsername) + else + pt = browser_path_unix_apple(shortbrowsername) + end + pt == "" && return `` + if shortbrowsername == "iexplore" + prsw = "-private" + else + prsw = "--incognito" + end + if shortbrowsername == "phantomjs" + if isdefined(:SRCPATH) + script = joinpath(SRCPATH, "phantom.js") + else + script = "phantom.js" + end + return Cmd(`$pt $script $URL`) + else + if Sys.is_windows() + return Cmd( [ pt, URL, prsw]) + else + if Sys.is_apple() + return Cmd(`open --fresh -n $URL -a $pt --args $prsw`) + elseif Sys.is_linux() || Sys.is_bsd() + return Cmd(`xdg-open $(URL) $pt`) + end + end + end +end + + +function open_testpage(shortbrowsername) + id = "open_testpage" + dmc = launch_command(shortbrowsername) + if dmc == `` + clog(id, "Could not find " * shortbrowsername) + return false + else + try + if shortbrowsername == "phantomjs" + # Run enables text output of phantom messages in the REPL. In Windows + # standalone REPL, run will freeze the main thread if not run async. + @async run(dmc) + else + spawn(dmc) + end + catch + clog(id, :red, "Failed to spawn " * shortbrowsername) + return false + end + end + return true +end + +"Try to open one browser from BROWSERS. +In some cases we expect an immediate indication +of failure, for example when the corresponding file +is not found on the system. In other cases, we will +just wait in vain for a request. In those cases, +call this function again. It remembers which browsers +were tried before. +" +function open_a_browser() + id = "open_next_browser" + if COUNTBROWSER.value > length(BROWSERS) + return false + end + success = false + b = "" + while COUNTBROWSER.value < length(BROWSERS) && !success + b = BROWSERS[COUNTBROWSER()] + clog(id, "Trying to launch browser no. ", COUNTBROWSER.value, ", ", :yellow, b) + success = open_testpage(b) + end + success && clog(id, "seems to work:", :yellow, b, :normal, " on ", URL) + success, b +end diff --git a/benchmark/logs/benchmark_results_readable.log b/benchmark/logs/benchmark_results_readable.log new file mode 100644 index 0000000..b3efea9 --- /dev/null +++ b/benchmark/logs/benchmark_results_readable.log @@ -0,0 +1,970 @@ +# Note this file is made by copy paste from the console. +# Benchmark_results.log does not have the encoding of ascii graphics right. + +HTS_BCE iexplore Varying message size: + bestserverbandwidth = 2.5996 [ns/b] = [s/GB] @ size = 220320 b + bestclientbandwidth = 2.7653 [ns/b] = [s/GB] @ size = 220320 b + bestserverlatency = 59000 [ns] @ size = 1020 b + bestclientlatency = 12614 [ns] @ size = 1020 b + +HTS_BCE chrome Varying message size: + bestserverbandwidth = 4.0413 [ns/b] = [s/GB] @ size = 65280 b + bestclientbandwidth = 3.4863 [ns/b] = [s/GB] @ size = 27540 b + bestserverlatency = 66759 [ns] @ size = 1020 b + bestclientlatency = 19428 [ns] @ size = 1020 b + +HTS_BCE phantomjs Varying message size: + bestserverbandwidth = 7.0451 [ns/b] = [s/GB] @ size = 127500 b + bestclientbandwidth = 1.5451 [ns/b] = [s/GB] @ size = 27540 b + bestserverlatency = 118169 [ns] @ size = 1020 b + bestclientlatency = 2208 [ns] @ size = 1020 b + +HTS_BCE firefox Varying message size: + bestserverbandwidth = 4.5881 [ns/b] = [s/GB] @ size = 127500 b + bestclientbandwidth = 4.9592 [ns/b] = [s/GB] @ size = 127500 b + bestserverlatency = 193333 [ns] @ size = 1020 b + bestclientlatency = 21735 [ns] @ size = 1020 b + +HTS_JCE Varying message size: + bestserverbandwidth = 1.3878 [ns/b] = [s/GB] @ size = 220320 b + bestclientbandwidth = 3.8825 [ns/b] = [s/GB] @ size = 220320 b + bestserverlatency = 61522 [ns] @ size = 8160 b + bestclientlatency = 75159 [ns] @ size = 1020 b + +Benchmark Plots of all samples :init_plots [ns/b], message size 130560 b 200 samples + HTS_BCE iexplore serverspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 70 │ . │ + │ | │ + │ | │ + │ | │ + │ | │ + │ [ │ + │ [ │ + │ .\ │ + │ || │ + │ || │ + │ || │ + │ || │ + │ || │ + │ . . ..... ..|l . .. .. ..._ ..,_. .. .. ....._.__. .. .│ + 0 │\-/'\-`"\/"/"\/`""`'""""/`' """"``'"""""`'"\-/`"` """`' `""""`'"""`'"` ` '"""""│ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE iexplore clientspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 7 │ │ + │ , │ + │ l │ + │ ] │ + │ ] | │ + │ ] [ │ + │ ] . | | l | [ │ + │ O,., [ . k..l/a.J . | .,\. , M ,. │ + │ N//\ A[UU,, |N\]||\N O_..\/_N. .ulP/v./,[k. r\. N,,.l^, , \.,.u,.│ + │ ,,| `l \ | ."N/\`|n/-`Y|/` ""Y|| '"" \n\_J '' \O `\| Y\N`Y`" "Y`""``/''\'│ + │'"\` |^_/,.|r/ " || "" "` │ + │ `"` | │ + │ | │ + │ ` │ + 1 │ │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE chrome serverspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 90 │ , │ + │ | │ + │ | │ + │ | │ + │ | │ + │ ] │ + │ ] │ + │ || │ + │ || │ + │ || │ + │ || │ + │ || │ + │, || │ + │\_. ..A.._/_...._ .. __/ .rv.,u_/.... .... . ,./___ || .._... ,..._.\. . r.│ + 0 │ '"`"`'`` "` `"""'/ "` " '`"`"/`""""'"` " ""`'"`" '`""T""` "`""""""`"│ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE chrome clientspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 200 │ │ + │ │ + │ │ + │ │ + │ . │ + │ | │ + │ | │ + │ | ,, │ + │ [ || │ + │ .\ || │ + │ || || │ + │ || || │ + │ || || │ + │ || /\ │ + 0 │\-------f\------.r-v.----------/\----.r--v.._r--------\-----.v-\r---------.-n\r-│ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE phantomjs serverspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 50 │ . │ + │ | │ + │ | │ + │ | │ + │ | │ + │ ] │ + │ /, │ + │ || . │ + │ || | │ + │, || /, │ + │\r, , || . |/\ . │ + │ \f|.. , u. . d , . . . |O. /L. . |\`l., M .\ │ + │ " '/"^V\/'\^``/`..|D| .... .k ..N./,. M ......, . .|"\-` \n/""/ "/``"""G │ + │ '`'"`''""`""`"`"`'`T"""`""`\"/"'"`' `\/"│ + 0 │ │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE phantomjs clientspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 60 │ │ + │ │ + │ , │ + │ | │ + │ | │ + │ | │ + │ | │ + │ [ │ + │ ,\ │ + │ || │ + │ || │ + │ || │ + │ || │ + │ || │ + 0 │/"`--------------------------------------------------------------/"^-^--`-f\----│ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE firefox serverspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 40 │ . │ + │ | │ + │ | │ + │ | . │ + │ [ l │ + │ | [ ] │ + │ [ ,\ ] │ + │ [ || ] │ + │, N , , || ] │ + │| N, [ l . |l /,. , │ + │ \|l, .l ,] A \| |, ||.||] .., . [ │ + │ '\ .. /. ...|| .. . /N.N.Ok |\ . .|"W||^, . /`\ ,,\\ ...^.│ + │ ^.|_.."\/lJ \f,\^/T V`|\ ._/,|"G|/ |r-fl./|r/r_/// '` `\../\./ ""/|l |\||J│ + │ ` '` ' """ "` ' "`` "` \` │ + 0 │ │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE firefox clientspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 70 │ , │ + │ . | | │ + │ | | | │ + │ l | [ | │ + │ ] | [ | │ + │ ] [ [ [ │ + │ ] [ [ [ │ + │ ] || [ [ │ + │ ] || [ [ │ + │ ] || k [ │ + │ N || N N │ + │ N || N N │ + │ . N || . N . , N │ + │,\/-v______-n__Nr_^.vu_\.-\..___.rfl_^J\^.ru_./.___r-v"rf\\^_._-`.^-f\v-V__\rr._│ + 0 │ \ ` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_JCE serverspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 200 │ │ + │ │ + │ │ + │ │ + │ │ + │ . │ + │ | │ + │ | |│ + │ [ l│ + │ [ ]│ + │ [ ]│ + │ M ]│ + │ N O│ + │ N N│ + 0 │._________N____________________________________________________________________N│ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_JCE clientspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 400 │ │ + │, │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\ │ + │| . . │ + │| l | │ + │| J M │ + 0 │|______________N________________N_______________________________________________│ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + +Benchmark Tables of all samples, :init_tables, message size 130560 b 200 samples +HTS_BCE iexplore +=> Table with 200 rows, 2 columns: +serverspeeds clientspeeds +────────────────────────── +4.5221 2.87694 +2.86018 2.97191 +2.91326 3.16742 +2.69259 3.00821 +2.60321 2.98866 +3.5026 2.86576 +3.92436 3.19256 +5.22318 2.77639 +3.5417 6.38512 +⋮ +4.51652 3.60595 +4.75952 3.38529 +4.23441 3.55008 +4.24837 3.10039 +3.67857 3.51377 +4.67292 3.45232 +5.12263 3.38529 +4.17016 3.40484 + +HTS_BCE chrome +=> Table with 200 rows, 2 columns: +serverspeeds clientspeeds +────────────────────────── +15.3204 10.6307 +7.13368 4.42155 +6.51081 6.62254 +6.39072 5.79856 +6.02202 5.11145 +5.6589 5.63935 +5.30977 5.05558 +5.45501 5.55556 +5.63097 7.03313 +⋮ +4.95503 4.85448 +5.02767 4.42155 +4.23719 4.17575 +9.28162 5.23715 +8.24537 5.15335 +8.6336 4.98856 +6.38513 4.79862 +5.52763 4.96621 + +HTS_BCE phantomjs +=> Table with 200 rows, 2 columns: +serverspeeds clientspeeds +────────────────────────── +18.0577 2.57806 +13.354 3.30149 +13.7953 3.46908 +14.8763 1.97754 +15.4824 3.41602 +11.7647 2.04179 +9.77321 2.62835 +12.2312 2.12279 +10.3626 2.15072 +⋮ +6.0332 1.50551 +5.77342 1.50271 +5.23994 1.49992 +5.77063 1.85744 +6.24547 1.64795 +5.77622 1.5865 +5.96337 1.58371 +6.08068 1.81834 + +HTS_BCE firefox +=> Table with 200 rows, 2 columns: +serverspeeds clientspeeds +────────────────────────── +17.1806 5.455 +14.6305 5.27904 +12.5859 8.54982 +10.7732 6.09464 +10.748 6.25664 +26.2248 2.20938 +15.0467 10.2899 +14.2702 7.28172 +9.98549 6.32367 +⋮ +6.13375 6.03598 +8.64757 6.37954 +8.9185 6.96889 +14.8176 5.66729 +7.46607 4.86566 +5.42708 5.41311 +5.85164 5.26229 +8.92688 5.63377 + +HTS_JCE +=> Table with 200 rows, 2 columns: +serverspeeds clientspeeds +────────────────────────── +2.69538 359.706 +2.89091 5.9047 +2.8518 6.44378 +2.36579 4.90196 +2.1172 4.32099 +1.94962 4.46623 +2.03899 4.67852 +2.6479 4.28748 +2.08648 4.02213 +⋮ +1.4664 3.69812 +1.38261 4.11988 +1.94682 4.8098 +2.01665 4.14223 +1.86861 3.90481 +1.80996 8.29005 +105.274 5.29859 +2.92442 4.42434 + +Benchmark Plots of varying size messages :test_plots [ns/b], + VSIZE = [1762560, 1357620, 1020000, 743580, 522240, 349860, 220320, 127500, 65280, 27540, 8160, 1020] + HTS_BCE iexplore serverbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 60 │, │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\ │ + │|_. │ + 0 │ '"""""`/""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE iexplore clientbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 13 │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\_---.__________r-----------------------------------------------------* │ + 2 │ │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome serverbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 70 │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │l │ + │|_ .____________-----------._________________________________________. │ + 0 │ ""` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome clientbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 20 │. │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| .____* │ + │| .f-._ _________r----""""` │ + │l ._ ./` "`-----.___----------/""""""""""""" │ + │"/` "` │ + │ │ + 0 │ │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs serverbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 200 │ │ + │ │ + │ │ + │ │ + │ │ + │ │ + │, │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │. │ + 0 │"-----------------/""""""`--------------------------------------------* │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs clientbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 2.5 │ ._-------/"""""""""` │ + │ __-/"` │ + │ \ _.-/" │ + │ |\ _-/"""""""""" │ + │ ||. _-" │ + │| | \ .__. _-" │ + │| | ". .` """""" │ + │|,` ". /` │ + │|| \ ./ │ + │|| `` │ + │|| │ + │|| │ + │|| │ + │|| │ + 1.5 │"` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox serverbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 200 │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\ │ + 0 │ \-------/"""""""""""""""\--------------------------------------------* │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox clientbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 30 │ │ + │ │ + │ │ + │ │ + │, │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| .____________..__________________________________________r-----* │ + │"-^---"` │ + │ │ + 0 │ │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE serverbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 90 │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + 0 │"----______________________________________________________________---* │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE clientbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 80 │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │'u_ │ + 0 │ """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + +Benchmark Tables of varying size messages :test_tables [ns/b] +HTS_BCE iexplore +=> Table with 12 rows, 3 columns: +VSIZE serverbandwidth clientbandwidth +───────────────────────────────────────── +1762560 2.84794 3.02725 +1357620 2.89765 3.01044 +1020000 2.72513 3.1075 +743580 3.20976 3.04866 +522240 2.73883 3.18919 +349860 3.09709 2.87533 +220320 2.59964 2.76531 +127500 3.07385 3.01227 +65280 3.77368 3.03525 +27540 5.23605 2.88944 +8160 10.9192 3.6188 +1020 57.8435 12.3667 + +HTS_BCE chrome +=> Table with 12 rows, 3 columns: +VSIZE serverbandwidth clientbandwidth +───────────────────────────────────────── +1762560 5.55213 7.15743 +1357620 5.48115 5.58792 +1020000 5.30875 5.11574 +743580 6.20273 4.77186 +522240 6.3483 4.34657 +349860 6.12922 4.77051 +220320 5.10403 6.23671 +127500 4.80703 3.85924 +65280 4.04126 4.13268 +27540 6.02253 3.4863 +8160 15.4351 5.59254 +1020 65.4499 19.047 + +HTS_BCE phantomjs +=> Table with 12 rows, 3 columns: +VSIZE serverbandwidth clientbandwidth +───────────────────────────────────────── +1762560 7.89614 2.49179 +1357620 8.07148 2.46927 +1020000 8.08981 2.29689 +743580 8.54885 2.27637 +522240 9.19892 2.07951 +349860 8.15506 2.11337 +220320 8.02959 1.85921 +127500 7.04513 2.05976 +65280 7.61785 2.34252 +27540 8.88562 1.54509 +8160 18.3139 1.57713 +1020 115.852 2.16479 + +HTS_BCE firefox +=> Table with 12 rows, 3 columns: +VSIZE serverbandwidth clientbandwidth +───────────────────────────────────────── +1762560 7.8439 6.88982 +1357620 7.41197 6.29824 +1020000 8.48551 6.14324 +743580 8.63577 6.31355 +522240 9.17379 6.67839 +349860 9.59405 6.32544 +220320 8.73156 6.46279 +127500 4.58813 4.95919 +65280 6.10207 5.34758 +27540 10.8064 5.09502 +8160 29.9181 5.77555 +1020 189.542 21.3083 + +HTS_JCE +=> Table with 12 rows, 3 columns: +VSIZE serverbandwidth clientbandwidth +───────────────────────────────────────── +1762560 2.0474 4.43299 +1357620 1.81932 4.27325 +1020000 1.96443 4.2789 +743580 1.76729 4.11628 +522240 1.61051 4.09457 +349860 1.61742 4.16369 +220320 1.38784 3.88246 +127500 1.96355 4.36213 +65280 3.23359 5.50702 +27540 3.43824 7.21109 +8160 7.53947 11.2128 +1020 84.4146 73.6853 + +Benchmark Plots of varying size messages :test_latency_plots [ns], + VSIZE = [1762560, 1357620, 1020000, 743580, 522240, 349860, 220320, 127500, 65280, 27540, 8160, 1020] + HTS_BCE iexplore serverlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 6000000 │ │ + │ │ + │ ._r* │ + │ ._r-/"` │ + │ ._r-/"` │ + │ ..-/"` │ + │ ._-/"` │ + │ _r-"` │ + │ ____---/""" │ + │ .r/" │ + │ ..-"` │ + │ .__--/` │ + │ _r/""` │ + │ .__-/" │ + 0 │_--""` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE iexplore clientlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 6000000 │ │ + │ ._/ │ + │ ._r-"` │ + │ ._-/"` │ + │ __-/"` │ + │ __--"" │ + │ __--"" │ + │ ._-/" │ + │ ._-/"` │ + │ __-/"` │ + │ __--"" │ + │ _r/" │ + │ __-/" │ + │ ._.-/" │ + 0 │_.-/"` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome serverlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 10000000 │ ._-/ │ + │ ._r-"` │ + │ _.-/` │ + │ ._-/" │ + │ _r-"` │ + │ ._-/" │ + │ ._.-"` │ + │ .__.--/""` │ + │ ._r/"` │ + │ _.-"` │ + │ _r/" │ + │ _r/" │ + │ ._-" │ + │ ._-/` │ + 0 │__--"` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome clientlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ │ + │ ._* │ + │ ..-"` │ + │ _r/"` │ + │ ._-/" │ + │ __r--"` │ + │ .__--/"" │ + │ .__.--/""` │ + │ .__.--f""` │ + │ .______r----/""` │ + 0 │____.-/"` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs serverlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ ._.* │ + │ __r--""` │ + │ .__--/"" │ + │ __r-/"` │ + │ ._.--"" │ + │ .__r-/""` │ + │ __.--""` │ + │ _.-/"" │ + │ __-/" │ + │ _u-/"" │ + 0 │__r--"" │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs clientlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 5000000 │ │ + │ .. │ + │ ._--"` │ + │ ._-/"` │ + │ _.-/"` │ + │ ._-/" │ + │ _r-"` │ + │ ._-/" │ + │ __-/"` │ + │ ._r-"" │ + │ ._r/"` │ + │ __r-"` │ + │ ._.--"" │ + │ __r-"` │ + 0 │__--/"" │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox serverlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ ._/ │ + │ ._.-/"` │ + │ _.-/"` │ + │ .___r---"" │ + │ ._.---""""` │ + │ __r-/"` │ + │ __r-/"" │ + │ __r-/"" │ + │ ._--"" │ + │ .r/"` │ + 0 │___r--"` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox clientlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ │ + │ . │ + │ __--"" │ + │ .__--"" │ + │ __.-/"` │ + │ __.--/"" │ + │ .__r--/"" │ + │ .___---"""` │ + │ ._.--/""` │ + │ .__---""` │ + 0 │___.--/"` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE serverlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 4000000 │ │ + │ ..* │ + │ ._-/` │ + │ _-/` │ + │ _r/" │ + │ __.-" │ + │ .__--/"" │ + │ ._--""` │ + │ ..-/` │ + │ _.-"` │ + │ ._-/" │ + │ .__-/"` │ + │ ._r-/"` │ + │ .__.-/"` │ + 0 │_-""""` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE clientlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 8000000 │ ._-* │ + │ ._-/` │ + │ _.-"` │ + │ ._r/" │ + │ ._r-"` │ + │ ._r-"` │ + │ _r-"` │ + │ ._-/" │ + │ _r-"` │ + │ _.-/" │ + │ ._.-/" │ + │ ._r-"` │ + │ ._-/"` │ + │ __--"` │ + 0 │_-/"" │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + +Benchmark Tables of varying size messages :test_latency_tables [ns] +HTS_BCE iexplore +=> Table with 12 rows, 3 columns: +VSIZE serverlatency clientlatency +───────────────────────────────────── +1762560 5.01966e6 5.33571e6 +1357620 3.93391e6 4.08703e6 +1020000 2.77963e6 3.16965e6 +743580 2.38672e6 2.26692e6 +522240 1.43032e6 1.66552e6 +349860 1.08355e6 1.00596e6 +220320 5.72753e5 6.09253e5 +127500 3.91916e5 3.84064e5 +65280 2.46346e5 1.98141e5 +27540 1.44201e5 79575.2 +8160 89100.4 29529.4 +1020 59000.4 12614.0 + +HTS_BCE chrome +=> Table with 12 rows, 3 columns: +VSIZE serverlatency clientlatency +───────────────────────────────────── +1762560 9.78596e6 1.26154e7 +1357620 7.44132e6 7.58628e6 +1020000 5.41492e6 5.21805e6 +743580 4.61223e6 3.54826e6 +522240 3.31534e6 2.26995e6 +349860 2.14437e6 1.66901e6 +220320 1.12452e6 1.37407e6 +127500 6.12896e5 4.92053e5 +65280 2.63813e5 2.69781e5 +27540 1.6586e5 96012.8 +8160 1.25951e5 45635.1 +1020 66758.9 19428.0 + +HTS_BCE phantomjs +=> Table with 12 rows, 3 columns: +VSIZE serverlatency clientlatency +───────────────────────────────────── +1762560 1.39174e7 4.39194e6 +1357620 1.0958e7 3.35233e6 +1020000 8.2516e6 2.34283e6 +743580 6.35676e6 1.69266e6 +522240 4.80404e6 1.086e6 +349860 2.85313e6 7.39385e5 +220320 1.76908e6 4.09622e5 +127500 8.98254e5 2.62619e5 +65280 4.97293e5 152920.0 +27540 244710.0 42551.8 +8160 149441.0 12869.3 +1020 1.18169e5 2208.09 + +HTS_BCE firefox +=> Table with 12 rows, 3 columns: +VSIZE serverlatency clientlatency +───────────────────────────────────── +1762560 1.38254e7 1.21437e7 +1357620 1.00626e7 8.55061e6 +1020000 8.65522e6 6.26611e6 +743580 6.42138e6 4.69463e6 +522240 4.79092e6 3.48772e6 +349860 3.35657e6 2.21302e6 +220320 1.92374e6 1.42388e6 +127500 5.84986e5 6.32297e5 +65280 3.98343e5 3.4909e5 +27540 2.97608e5 1.40317e5 +8160 2.44132e5 47128.5 +1020 1.93333e5 21734.5 + +HTS_JCE +=> Table with 12 rows, 3 columns: +VSIZE serverlatency clientlatency +───────────────────────────────────── +1762560 3.60866e6 7.81341e6 +1357620 2.46994e6 5.80145e6 +1020000 2.00372e6 4.36448e6 +743580 1.31412e6 3.06078e6 +522240 841072.0 2.13835e6 +349860 5.65872e5 1.45671e6 +220320 305769.0 8.55384e5 +127500 2.50353e5 5.56171e5 +65280 211089.0 359498.0 +27540 94689.1 1.98593e5 +8160 61522.1 91496.4 +1020 86102.9 75159.0 + +Benchmark Dictionary :test_bestserverlatencies [ns] +HTS_BCE iexplore => 59000 +HTS_BCE chrome => 66759 +HTS_BCE phantomjs => 118169 +HTS_BCE firefox => 193333 +HTS_JCE => 61522 +Benchmark Dictionary :test_bestclientlatencies [ns] +HTS_BCE iexplore => 12614 +HTS_BCE chrome => 19428 +HTS_BCE phantomjs => 2208 +HTS_BCE firefox => 21735 +HTS_JCE => 75159 +Benchmark Dictionary :test_bestserverbandwidths [ns/b] +HTS_BCE iexplore => 2.5996 +HTS_BCE chrome => 4.0413 +HTS_BCE phantomjs => 7.0451 +HTS_BCE firefox => 4.5881 +HTS_JCE => 1.3878 +Benchmark Dictionary :test_bestclientbandwidths [ns/b] +HTS_BCE iexplore => 2.7653 +HTS_BCE chrome => 3.4863 +HTS_BCE phantomjs => 1.5451 +HTS_BCE firefox => 4.9592 +HTS_JCE => 3.8825 diff --git a/benchmark/phantom.js b/benchmark/phantom.js new file mode 100644 index 0000000..09aaf58 --- /dev/null +++ b/benchmark/phantom.js @@ -0,0 +1,34 @@ +// Note that the console log output does not appear in Julia REPL when +// this is called using spawn. For debugging, use "run". These functions +// are run in a shell outside the web page. +var page = require('webpage').create(), + system = require('system'), + t, address; + +if (system.args.length === 1) { + console.log('Phantomjs.js: too few arguments. Need phantom.js '); + phantom.exit(); +} +console.log('PhantomJS: The default user agent is ' + page.settings.userAgent); +page.settings.userAgent = 'PhantomJS'; +console.log('PhantomJS: User agent set to ' + page.settings.userAgent); + + +t = Date.now(); +address = system.args[1]; +page.open(address, function(status) { + if (status !== 'success') { + console.log('FAIL to load the address'); + } else { + t = Date.now() - t; + console.log('PhantomJS loading ' + system.args[1]); + console.log('PhantomJS loading time ' + t + ' msec'); + window.setTimeout( (function() { + page.render("phantomjs.png"); + console.log("PhantomJS saved render, exits after 30s") + phantom.exit() + }), + 30000) + } + } +); diff --git a/benchmark/ws_hts.jl b/benchmark/ws_hts.jl new file mode 100644 index 0000000..7ce3b15 --- /dev/null +++ b/benchmark/ws_hts.jl @@ -0,0 +1,156 @@ +# Submodule Julia HTTP Server +# HTTP and WebSockets need to be loaded in the calling context. +# LOAD_PATH must include logutils +# Intended for accepting echoing clients, such as ws_jce.jl, and +# running echo tests with that client. +# The server stays open until close_hts or the websocket is closed. +module ws_hts +using ..HTTP +import HTTP.Header +using ..WebSockets +# We want to log to a separate file, so +# we use our own instance of logutils_ws here. +import logutils_ws: logto, clog, zlog, zflush +const SRCPATH = Base.source_dir() == nothing ? Pkg.dir("WebSockets", "benchmark") :Base.source_dir() +const SERVEFILE = "bce.html" +const PORT = 8000 +const SERVER = "127.0.0.1" +const WSMAXTIME = Base.Dates.Second(600) +const WEBSOCKET = Vector{WebSockets.WebSocket}() +const TCPREF = Ref{HTTP.Sockets.TCPServer}() +"Run asyncronously or in separate process" +function listen_hts() + id = "listen_hts" + try + clog(id,"listen_hts starts on ", SERVER, ":", PORT) + zflush() + HTTP.listen(SERVER, UInt16(PORT), tcpref = TCPREF) do http + if WebSockets.is_upgrade(http.message) + acceptholdws(http) + clog(id, "Websocket closed, server stays open until ws_hts.close_hts()") + else + HTTP.Servers.handle_request(handlerequest, http) + end + end + catch err + clog(id, :red, err) + clog_notime.(catch_stacktrace()[1:4]) + zflush() + end +end + +"Accepts an incoming connection, upgrades to websocket, +and waits for timeout or a closed socket. +Also stops the server from accepting more connections on exit." +function acceptholdws(http) + id = "ws_hts.acceptholdws" + zlog(id);zflush() + # If the ugrade is successful, just hold the reference and thread + # of execution. Other tasks may do useful things with it. + WebSockets.upgrade(http) do ws + push!(WEBSOCKET, ws) + zlog(id, ws);zflush() + t1 = now() + WSMAXTIME + while isopen(ws) && now() < t1 + yield() + end + length(WEBSOCKET) > 0 && pop!(WEBSOCKET) + zlog(id, " exiting");zflush() + end +end +"Returns a websocket or a string" +function getws_hts() + id = "getws_hts" + if length(WEBSOCKET) > 0 + if isopen(WEBSOCKET[1]) + zlog(id, " return reference") + return WEBSOCKET[1] + else + msg = " Websocket is referred but not open. Acceptholdws might not have been scheduled after is was closed." + clog(id, msg) + zflush() + return msg + end + else + if !isdefined(TCPREF, :x) + msg = " No server running yet, run ws_hts.listen_hts() or wait" + else + msg = " No websocket has connected yet, run ws_hce.echowithdelay_jce() or wait" + end + zlog(id, msg) + return id * msg + end +end + + + + +"HTTP request -> HTTP response." +function handlerequest(request::HTTP.Request) + id = "handlerequest" + zlog(id, request) + response = responseskeleton(request) + try + if request.method == "GET" + response = resp_HTTP(request.target, response) + elseif request.method == "HEAD" + response = resp_HTTP(request.target, response) + response.body = Array{UInt8,1}() + else + response = HTTP.Response(501, "Not implemented method: $(request.method), fix $id") + end + end + zlog(id, response) + response +end + + +""" +Tell browser about the methods this server supports. +""" +function responseskeleton(request::HTTP.Request) + r = HTTP.Response() + HTTP.Messages.appendheader(r, Header("Allow" => "GET,HEAD")) + HTTP.Messages.appendheader(r, Header("Connection" => "close")) + r # +end + +"request.target -> HTTP.Response , building on a skeleton response" +function resp_HTTP(resource::String, resp::HTTP.Response) + id = "resp_HTTP" + if resource == "/favicon.ico" + s = read(joinpath(SRCPATH, "favicon.ico")) + push!(resp.headers, "Content-Type" => "image/x-icon") + else + s = read(joinpath(SRCPATH, SERVEFILE)) + push!(resp.headers, "Content-Type" => "text/html") + end + resp.body = s + push!(resp.headers, "Content-Length" => string(length(s))) + resp.status = 200 + resp +end + +"Close the websocket, stop the server. TODO improve" +function close_hts() + clog("ws_hts.close_hts") + length(WEBSOCKET) >0 && isopen(WEBSOCKET[1]) && close(WEBSOCKET[1]) && sleep(0.5) + isassigned(TCPREF) && close(TCPREF.x) +end + +end # module +""" +For debugging: + +import HTTP +using WebSockets +const SRCPATH = Base.source_dir() == nothing ? Pkg.dir("WebSockets", "benchmark") :Base.source_dir() +const LOGGINGPATH = realpath(joinpath(SRCPATH, "../logutils/")) +# for finding local modules +SRCPATH ∉ LOAD_PATH && push!(LOAD_PATH, SRCPATH) +LOGGINGPATH ∉ LOAD_PATH && push!(LOAD_PATH, LOGGINGPATH) +import ws_hts.listen_hts +tas = @schedule ws_hts.listen_hts() +sleep(5) +hts = ws_hts.getws_hts() +""" \ No newline at end of file diff --git a/benchmark/ws_jce.jl b/benchmark/ws_jce.jl new file mode 100644 index 0000000..f4b193b --- /dev/null +++ b/benchmark/ws_jce.jl @@ -0,0 +1,131 @@ +__precompile__(false) +""" +Submodule Julia Client Echo +Intended for running in its own worker process. +HTTP and WebSockets need to be loaded in the calling context. +LOAD_PATH must include the directory logutils_ws. +See comment at the end of this file for debugging code. +""" +module ws_jce +using ..HTTP +using ..WebSockets +# We want to log to a separate file, and so use our own +# instance of logutils_ws in this process +import logutils_ws: logto, clog, zlog, zflush +const SRCPATH = Base.source_dir() == nothing ? Pkg.dir("WebSockets", "benchmark") :Base.source_dir() +const LOGFILE = "ws_jce.log" + +const PORT = 8000 +const SERVER = "ws://127.0.0.1:$(PORT)" +const CLOSEAFTER = Base.Dates.Second(30) + +""" +Opens a client, echoes with optional delay, an integer in milliseconds. +Stores time records for received messages and before sending messages. +Specify delay in milliseconds by sending a message on the websocket: + send(ws_jce, "delay|15") +Echoes any message except "exit" and "delay" + +At exit or after CLOSEAFTER, sends one message containing two vectors of +timestamps [ns].AbstractTrees +""" +function echowithdelay_jce() + # This will be run in a worker process. Even so, individual console log + # output will be redirected to process 1 and prefixed + # with a "From worker 2". + # It will also be interspersed with process 1 output, + # sometimes before a line is finished printing. + # We use :green to distinguish more easily. + id = "echowithdelay_jce" + f = open(joinpath(SRCPATH, "logs", LOGFILE), "w") + try + logto(f) + clog(id, :green, "Open client on ", SERVER, "\nclient side handler ", _jce) + zflush() + WebSockets.open(_jce, SERVER) + zlog(id, :green, " Websocket closed, control returned.") + catch err + clog(id, :red, err) + clog_notime.(catch_stacktrace()[1:4]) + zflush() + finally + clog(id, :green, " Closing log ", LOGFILE) + zflush() + logto(Base.DevNullStream()) + close(f) + end +end +" +Handler for client websocket, defined by echowithdelay +" +function _jce(ws) + id = "_jce" + clog(id, :green, ws) + zflush() + receivetimes = Vector{Int64}() + replytime = Int64(0) + replytimes = Vector{Int64}() + msg = Vector{UInt8}() + delay = 0 # Integer milliseconds + t1 = now() + CLOSEAFTER + while isopen(ws) && now() < t1 + # read, record receive time + msg = read(ws) + ti = time_ns() + push!(receivetimes, Int64(ti < typemax(Int64) ? ti : 0 )) + # break out when receiving 'exit' + length(msg) == 4 && msg == Vector{UInt8}("exit") && break + # react to delay instruction + if length(msg) < 16 && msg[1:6] == Vector{UInt8}("delay=") + delay = parse(Int, String(msg[7:end])) + clog(id, :green, " Changing delay to ", delay, " ms") + zflush() + end + sleep(delay / 1000) + # record send time, echo + replytime = time_ns() + write(ws, msg) + # clean record of instruction message + if length(msg) > 16 && msg[1:6] != Vector{UInt8}("delay=") + push!(replytimes, Int64(replytime < typemax(Int64) ? replytime : 0 )) + elseif msg[1:6] != Vector{UInt8}("delay=") + push!(replytimes, Int64(replytime < typemax(Int64) ? replytime : 0 )) + end + end + if length(receivetimes) > 1 + # Don't include receive time for the "exit" message. Reply time was not recorded + pop!(receivetimes) + if length(receivetimes) > 1 + # Send one message with serialized receive and reply times. + buf = IOBuffer() + serialize(buf, (receivetimes, replytimes)) + if isopen(ws) + write(ws, take!(buf)) + zlog(id, :green, " Sent serialized receive and reply times.") + zflush() + end + end + end + clog(id, :green, " Exit, close websocket.") + zflush() + # Exiting this function starts a closing handshake + # from this side (HTTP.jl:39). The other side must read in + # in order for this to proceed smoothly. + nothing +end +end # module + +#= +For debugging in a separate terminal: + +using HTTP +using WebSockets +const SRCPATH = Base.source_dir() == nothing ? Pkg.dir("WebSockets", "benchmark") :Base.source_dir() +const LOGGINGPATH = realpath(joinpath(SRCPATH, "../logutils/")) +# for finding local modules +SRCPATH ∉ LOAD_PATH && push!(LOAD_PATH, SRCPATH) +LOGGINGPATH ∉ LOAD_PATH && push!(LOAD_PATH, LOGGINGPATH) + +import ws_jce.echowithdelay_jce +ws_jce.echowithdelay_jce() +=# \ No newline at end of file diff --git a/examples/chat_explore.jl b/examples/chat_explore.jl index 66d979f..0838a70 100644 --- a/examples/chat_explore.jl +++ b/examples/chat_explore.jl @@ -238,6 +238,7 @@ end info("Start HTTP server on port $(HTTPPORT)") litas_newtype = @schedule HTTP.listen(server_def_newtype, "127.0.0.1", UInt16(HTTPPORT)) +# TODO replace with improvement from 'close websocket'. function closefromoutside() if isdefined(:litas_newtype) @schedule Base.throwto(litas_newtype, InterruptException()) diff --git a/logutils/log_http.jl b/logutils/log_http.jl new file mode 100644 index 0000000..1ecf05c --- /dev/null +++ b/logutils/log_http.jl @@ -0,0 +1,42 @@ +#= +Included in logutils xxx.jl if HTTP is loaded +=# + +"HTTP.Response already has a show method, we're not overwriting that. +This metod is called only when logging to an Abstractdevice. The default +show method does not print binary data well as per now." +function _show(d::AbstractDevice, response::HTTP.Messages.Response) + _log(d, :green, "Response status: ", :bold, response.status," ") + response.status > 0 && _log(d, HTTP.Messages.STATUS_MESSAGES[response.status], " ") + if !isempty(response.headers) + _log(d, :green, " Headers: ", :bold, length(response.headers)) + _log(d, :green, "\n", response.headers) + end + if isdefined(response, :cookies) + if !isempty(response.cookies) + _log(d, " Cookies: ", :bold, length(response.cookies)) + _log(d, "\n", response.cookies) + end + end + if !isempty(response.body) + _log(d, "\t", DataDispatch(response.body, HTTP.header(response, "content-type", ""))) + end + nothing +end + + +"HTTP.Request already has a show method, we're not overwriting that. +This metod is called only when logging to an Abstractdevice" +function _show(d::AbstractDevice, request::HTTP.Messages.Request) + _log(d, :normal, :light_yellow, "Request ", :normal) + _log(d, :bold, request.method, " ", :cyan, request.target, "\n", :normal) + if !isempty(request.body) + _log(d, "\t", DataDispatch(request.body, HTTP.header(request, "content-type", ""))) + end + if !isempty(request.headers) + _log(d, "\t", :cyan, " Headers: ", length(request.headers)) + _log(d, :cyan, "\n", request.headers) + end + nothing +end + diff --git a/logutils/log_httpserver.jl b/logutils/log_httpserver.jl new file mode 100644 index 0000000..5b6444b --- /dev/null +++ b/logutils/log_httpserver.jl @@ -0,0 +1,85 @@ +import HttpServer.Client +import HttpServer.HttpHandler +import HttpServer.Server +import HttpServer.ClientParser +import HttpServer.HttpParser +import HttpCommon.Cookie +import HttpCommon.STATUS_CODES +import URIParser.URI +export printstartinfo +show(io::IO, client::Client) = directto_abstractdevice(io, client) +function _show(d::AbstractDevice, client::Client) + _log(d, typeof(client), " id ", :bold, client.id, :normal) + _log(d, " ", client.sock, "\n") + _log(d, "\t\t\t", client.parser) + nothing +end + +# TODO is this good enough? +function show(io::IO, p::ClientParser) + print(io, "HttpServer.ClientParser.HttpParser.libhttp-parser: v", HttpParser.version()) + if p.parser.http_major > 0 + print(io, " HTTP/", p.parser.http_major, ".", p.parser.http_minor) + end + nothing +end + +"Response already has a decent show method, we're not overwriting that. +This metod is called only when logging to an Abstractdevice" +function _show(d::AbstractDevice, response::HttpServer.Response) + _log(d, :green, "Response status: ", :bold, response.status," ") + _log(d, get(STATUS_CODES, response.status, "--"), " ") + if !isempty(response.headers) + _log(d, :green, " Headers: ", :bold, response.headers.count) + _log(d, :green, "\n", response.headers) + end + if !isempty(response.cookies) + _log(d, " Cookies: ", :bold, length(response.cookies)) + _log(d, "\n", response.cookies) + end + if !isempty(response.data) + _log(d, "\t", DataDispatch(response.data, get(response.headers, "Content-Type","---"))) + end + nothing +end + +"Request already has a decent show method, we're not overwriting that. +This metod is called only when logging to an Abstractdevice" +function _show(d::AbstractDevice, request::HttpServer.Request) + _log(d, :normal, :light_yellow, "Request ", :normal) + _log(d, :bold, request.method, " ", :cyan, request.resource, "\n", :normal) + if !isempty(request.data) + _log(d, "\t", DataDispatch(request.data, get(request.headers, "Content-Type", "text/html; charset=utf-8"))) + end + if request.uri != URI("") + _log(d, :cyan, "\tUri:", :bold, _string(request.uri), :normal, "\n\t") + end + if !isempty(request.headers) + _log(d, "\t", :cyan, " .headers: ", request.headers.count) + _log(d, :cyan, "\n", request.headers) + end + nothing +end + +"HttpServer does not define a show method for its server type. Defining this is not piracy." +show(io::IO, server::Server) = directto_abstractdevice(io, server) +function _show(d::AbstractDevice, server::Server) + _log(d, :bold , :green, typeof(server), "(\n", :normal) + server.http != nothing && _log(d, "\t", server.http) + server.websock != nothing && _log(d, "\t", server.websock) + _log(d, :bold, :green, ")") + nothing +end + +"HttpServer does not define a show method for its HttpHandler type. Defining this is not piracy." +show(io::IO, httphandler::HttpHandler) = directto_abstractdevice(io, httphandler) +function _show(d::AbstractDevice, httphandler::HttpHandler) + _log(d, :bold, Base.info_color(), typeof(httphandler), "( " , :normal) + _log(d, ".handle: ", :blue, :bold, httphandler.handle, :normal, "\n") + _log(d, "\t\t\t.events:\n") + _log(d, httphandler.events) + _log(d, "\t\t\t", ".socket:\t", httphandler.sock, :bold, Base.info_color(), ")\n") + nothing +end + +nothing \ No newline at end of file diff --git a/logutils/log_ws.jl b/logutils/log_ws.jl new file mode 100644 index 0000000..956536b --- /dev/null +++ b/logutils/log_ws.jl @@ -0,0 +1,30 @@ +#= +Included in logutils.jl +=# + +import WebSockets.WebSocket # todo remove when including in WebSockets itself. +show(io::IO, ws::WebSocket) = directto_abstractdevice(io, ws) +function _show(d::AbstractDevice, ws::WebSocket{T}) where T + _log(d, "WebSocket{", T, "}(") + _log(d, ws.server ? "server, " : "client, ") + _log(d, ws.socket, " ") + showcompact(d.s, ws.state) + _log(d, ")") +# size = ws.socket.buffer.size +# if size == 0 +# _log(d, size, " b)") +# elseif size < 1000 +# _log(d, :yellow, size, " b)") +# else +# _log(d, :bold, :yellow, div(size, 1000), " kB)") +# end + nothing +end +@require HttpServer import WebSockets.WebSocketHandler +@require HttpServer show(io::IO, wsh::WebSocketHandler) = directto_abstractdevice(io, wsh) +@require HttpServer function _show(d::AbstractDevice, wsh::WebSocketHandler) + _log(d, typeof(wsh), "( " , :blue, :bold, wsh.handle, :normal, ")") + nothing + end + +nothing \ No newline at end of file diff --git a/logutils/logutils_ws.jl b/logutils/logutils_ws.jl new file mode 100644 index 0000000..7e3437a --- /dev/null +++ b/logutils/logutils_ws.jl @@ -0,0 +1,398 @@ +__precompile__() +""" +Specialized logging for testing and examples in WebSockets. + +To avoid type piracy, defines _show for types where specialized show methods exist, +and falls back to 'show' where they don't. + +When logging to a file, it's beneficial to keep a file open during the whole logging sesssion, +since opening and closing files is so slow it may affect the sequence of things. + +This module dispatches on an ad-hoc type AbstractDevice, instead of putting +additional data on an IOContext method like 'show' does. When AbstracDevice points to +Base.NullDevice(), the input arguments are processed before the call is made, but +that is all. + +In Julia 0.7, current logging functionality is replaced with macros, which +can be even faster. Macros can also retrieve the current function's name without using stacktraces. +With this module, each function defines id = "thisfunction". + +Methods in this file have no external dependencies. Methods with dependencies are +loaded from separate files with @require. This adds to loading time. +""" +module logutils_ws +using Requires +import Base.text_colors +import Base.color_normal +import Base.text_colors +import Base.show +@require HttpServer include("log_httpserver.jl") +@require HTTP include("log_http.jl") +@require WebSockets include("log_ws.jl") +export clog +export clog_notime +export zlog +export zlog_notime +export logto +export loggingto +export directto_abstractdevice +export AbstractDevice +export DataDispatch +export zflush + +const ColorDevices = Union{Base.TTY, IOBuffer, IOContext, Base.PipeEndpoint} +const BlackWDevices = Union{IOStream, IOBuffer} # and endpoint... +const LogDevices = Union{ColorDevices, BlackWDevices, Base.DevNullStream} +abstract type AbstractDevice{T} end +struct NullDevice <: AbstractDevice{NullDevice} + s::Base.DevNullStream + end +struct ColorDevice{S<:ColorDevices}<:AbstractDevice{ColorDevice} + s::S + end +struct BlackWDevice{S<:Union{IOStream, IOBuffer}}<:AbstractDevice{BlackWDevice} + s::S + end +mutable struct LogDevice + s::AbstractDevice + end +_devicecategory(::AbstractDevice{S}) where S = S +const CURDEVICE = LogDevice(NullDevice(Base.DevNullStream())) +struct DataDispatch + data::Array{UInt8,1} + contenttype::String +end +""" +Redirect coming zlog calls to a stream. Default is no logging. +clog calls will duplicate to STDOUT. +""" +logto(io::ColorDevices) = CURDEVICE.s = ColorDevice(io) +logto(io::BlackWDevices) = CURDEVICE.s = BlackWDevice(io) +logto(io::Base.DevNullStream) = CURDEVICE.s = NullDevice(io) + +""" +Returns the current logging stream +""" +loggingto() = CURDEVICE.s.s + +""" +Log to (default) nothing, or streams as given in CURDEVICE. +First argument is expected to be function id. +""" +function zlog(vars::Vararg) + _zlog(CURDEVICE.s, vars...) + nothing +end +""" +Log to (default) NullDevice, or the device given in CURDEVICE. +Falls back to Base.showdefault when no special methods are defined. +""" +function zlog_notime(vars::Vararg) + _zlog_notime(CURDEVICE.s, vars...) + nothing +end +""" +Log to the given device, but also to STDOUT if that's not the given device. +""" +function clog(vars::Vararg) + _zlog(CURDEVICE.s, vars...) + _devicecategory(CURDEVICE.s) != ColorDevice && _zlog(ColorDevice(STDOUT), vars...) + nothing +end +""" +Log to the given device, but also to STDOUT if that's not the given device. +""" +function clog_notime(vars::Vararg) + _zlog_notime(CURDEVICE.s, vars...) + _devicecategory(CURDEVICE.s) != ColorDevice && _zlog_notime(ColorDevice(STDOUT), vars...) + nothing +end +""" +Flushing file write buffers, only has an effect on streams. +We do not flush automatically after every log, because that +has a relatively dramatic effect on logging speeds. +""" +zflush() = isa(CURDEVICE.s.s, IOStream) && flush(CURDEVICE.s.s) + + +## Below are internal functions starting with _ +_zlog(::NullDevice, ::Vararg) = nothing +_zlog_notime(::NullDevice, ::Vararg) = nothing +"First argument padded and emphasized. +End the original argument list with a :normal and a new line argument." +function _zlog(d::AbstractDevice, vars::Vararg{Any,N}) where N + if N == 1 + _log(d, :normal, _tg(), " ", :bold, :cyan, vars[1], :normal, "\n") + else + lid = 26 + pads = repeat(" ", max(0, lid - length(_string(vars[1])))) #rpad don't work with color codes + _log(d, :normal, _tg(), " ", :bold, :cyan, vars[1], :normal, pads, vars[2:end]..., :normal, "\n") + end + nothing +end +"zlog, but no time stamp and no different format first argument." +function _zlog_notime(d::AbstractDevice, vars::Vararg{Any,N}) where N + # End the original argument list with a :normal and a new line argument. + _log(d, :bold, :cyan, vars[1:end]..., :normal, "\n") + nothing +end +"Write to blackwdevices, inapplicable formatting neglegted. No linefeed." +function _log(bwd::BlackWDevice, vars::Vararg) + _show.(bwd, vars) + nothing +end + +"Write colordevices. Does not reset color codes after, no linefeed." +function _log(cd::ColorDevice, vars::Vararg) + buf = ColorDevice(IOBuffer()) + _show.(buf, vars) + write(cd.s, take!(buf.s)) + nothing +end +"Write directly to colordevice/ IOBuffers" +function _log(cd::ColorDevice{Base.AbstractIOBuffer{Array{UInt8,1}}}, vars::Vararg) + _show.(cd, vars) + nothing +end + +"Return a string by emulating a bwdevice, where color codes are not output" +function _string(vars::Vararg) + buf = BlackWDevice(IOBuffer()) + _show.(buf, vars) + buf.s |> take! |> String +end + + + +"_show takes just device and one other argument. +It falls back to normal show for nondefined _show methods." +_show(d::AbstractDevice, var) = show(d.s, var) +_show(d::AbstractDevice, s::AbstractString) = write(d.s, s);nothing + +function _show(d::ColorDevice, sy::Symbol) + co = get(text_colors, sy, :NA) + if co != :NA + write(d.s, co) + else + write(d.s, sy) + end + nothing +end +function _show(d::ColorDevices, sy::Symbol) + co = get(text_colors, sy, :NA) + if co != :NA + write(d.s, co) + else + write(d.s, sy) + end + nothing +end + +function _show(d::BlackWDevice, sy::Symbol) + co = get(text_colors, sy, :NA) + if co == :NA + write(d.s, ":", sy) + end + nothing +end + +function _show(d::AbstractDevice, ex::Exception) + _log(d, Base.warn_color(), typeof(ex), "\n") + showerror(d.s, ex, []) + _log(d, :normal, "\n") +end +function _show(d::AbstractDevice, err::ErrorException) + _log(d, Base.error_color(), typeof(err), "\n") + showerror(d.s, err, []) + _log(d, :normal, "\n") +end +function _show(d::AbstractDevice, err::Base.UVError) + _log(d, Base.error_color(), typeof(err), "\n") + showerror(d.s, err, []) + _log(d, :normal, "\n") +end + + + +"Print dict, no heading, three pairs per line, truncate end to fit" +function _show(d::AbstractDevice, di::Dict) + linelength = displaysize(STDOUT)[2] + indent = 8 + npa = 3 + plen = div(linelength - indent, npa) + pairs = collect(di) + lpa = length(pairs) + _log(d, :bold, :blue) + for i = 1:npa:lpa + write(d.s, " "^indent) + write(d.s, _pairpad(pairs[i], plen)) + i+1 <= lpa && write(d.s, _pairpad(pairs[i + 1], plen)) + i+2 <= lpa && write(d.s, _pairpad(pairs[i + 2], plen)) + write(d.s, "\n") + end + _log(d, :normal) + nothing +end +"Print array of pairs, no heading, three pairs per line, truncate end to fit" +function _show(d::AbstractDevice, pairs::Vector{Pair{SubString{String},SubString{String}}}) + linelength = 95 + indent = 8 + npa = 3 + plen = div(linelength - indent, npa) + lpa = length(pairs) + _log(d, :bold, :blue) + for i = 1:npa:lpa + write(d.s, " "^indent) + write(d.s, _pairpad(pairs[i], plen)) + i+1 <= lpa && write(d.s, _pairpad(pairs[i + 1], plen)) + i+2 <= lpa && write(d.s, _pairpad(pairs[i + 2], plen)) + write(d.s, "\n") + end + _log(d, :normal) + nothing +end +"Print dict, no heading, two pairs per line, truncate end to fit" +function _show(d::AbstractDevice, di::Dict{String, Function}) + linelength = 95 + indent = 8 + npa = 2 + plen = div(linelength - indent, npa) + pairs = collect(di) + lpa = length(pairs) + _log(d, :bold, :blue) + for i = 1:npa:lpa + write(d.s, " "^indent) + write(d.s, _pairpad(pairs[i], plen)) + i+1 <= lpa && write(d.s, _pairpad(pairs[i + 1], plen)) + write(d.s, "\n") + end + _log(d, :normal) + nothing +end + + +_pairpad(pa::Pair, plen::Int) = Base.cpad(_limlen(_string(pa), plen) , plen ) +_string(pa::Pair) = _string(pa[1]) * " => " * _string(pa[2]) +function _show(d::AbstractDevice, f::Function) + mt = typeof(f).name.mt + fnam = splitdir(string(mt.defs.func.file))[2] + write(d.s, string(f) * " at " * fnam * ":" + * string(mt.defs.func.line)) + nothing +end + +"Type info not printed here as it is assumed the type is given by the context." +function _show(d::ColorDevice, stream::Base.LibuvStream) + _log(d, "(", :bold, _uv_status(stream)..., :normal) + nba = Base.nb_available(stream.buffer) + nba > 0 && print(d.s, ", ", Base.nb_available(stream.buffer)," bytes waiting") + print(d.s, ")") + nothing +end +function _uv_status(x) + s = x.status + if x.handle == Base.C_NULL + if s == Base.StatusClosed + return :red, "✘" #"closed" + elseif s == Base.StatusUninit + return :red, "null" + end + return :red, "invalid status" + elseif s == Base.StatusUninit + return :yellow, "uninit" + elseif s == Base.StatusInit + return :yellow, "init" + elseif s == Base.StatusConnecting + return :yellow, "connecting" + elseif s == Base.StatusOpen + return :green, "✓" # "open" + elseif s == Base.StatusActive + return :green, "active" + elseif s == Base.StatusPaused + return :red, "paused" + elseif s == Base.StatusClosing + return :red, "closing" + elseif s == Base.StatusClosed + return :red, "✘" #"closed" + elseif s == Base.StatusEOF + return :yellow, "eof" + end + return :red, "invalid status" +end + + +function _show(d::AbstractDevice, serv::Base.LibuvServer) + _log(d, typeof(serv), "(", :bold, _uv_status(serv)..., :normal, ")") + nothing +end + + +"Data as a truncated string" +function _show(d::AbstractDevice, datadispatch::DataDispatch) + _showdata(d, datadispatch.data, datadispatch.contenttype) + write(d.s, "\n") + nothing +end + + +function _showdata(d::AbstractDevice, data::Array{UInt8,1}, contenttype::String) + if ismatch(r"(text|script|html|xml|julia|java)", lowercase(contenttype)) + _log(d, :green, "\Data length: ", length(data), " ", :bold, :blue) + s = data |> String |> _limlen + write(d.s, replace(s, r"\s+", " ")) + else + _log(d, :green, "\Data length: ", length(data), " ", :blue) + write(d.s, data |> _limlen) + end + nothing +end + +"Truncates for logging" +_limlen(data::AbstractString) = _limlen(data, 74) +function _limlen(data::AbstractString, linelength::Int) + le = length(data) + if le < linelength + return normalize_string(string(data), stripcc = true) + else + adds = " … " + addlen = length(adds) + truncat = 2 * div(linelength, 3) + tail = linelength - truncat - addlen - 1 + truncstring = String(data)[1:truncat] * adds * String(data)[end-tail:end] + return normalize_string(truncstring, stripcc = true) + end +end +function _limlen(data::Union{Vector{UInt8}, Vector{Float64}}) + le = length(data) + maxlen = 12 # elements, not characters + if le < maxlen + return string(data) + else + adds = " ..... " + addlen = 2 + truncat = 2 * div(maxlen, 3) + tail = maxlen - truncat - addlen - 1 + return string(data[1:truncat])[1:end-1] * adds * string(data[end-tail:end])[7:end] + end +end + + +"Time group. show() converts to string only when necessary." +_tg() = Dates.Time(now()) + + + +"For use in show(io::IO, obj) methods. Hook into this logger's dispatch mechanism." +function directto_abstractdevice(io::IO, obj) + if isa(io, ColorDevices) + buf = ColorDevice(IOBuffer()) + else + buf = BlackWDevice(IOBuffer()) + end + _show(buf, obj) + write(io, take!(buf.s)) + nothing +end + +nothing +end # module \ No newline at end of file diff --git a/src/HTTP.jl b/src/HTTP.jl index 3ec974e..74c7525 100644 --- a/src/HTTP.jl +++ b/src/HTTP.jl @@ -1,9 +1,27 @@ info("Loading HTTP methods...") +""" +Initiate a websocket connection to server defined by url. If the server accepts +the connection and the upgrade to websocket, f is called with an open client type websocket. + +e.g. say hello, close and leave +```julia +import HTTP +using WebSockets +WebSockets.open("ws://127.0.0.1:8000") do ws + write(ws, "Hello") + println("that's it") +end; +``` +If a server is listening and accepts, "Hello" is sent (as a Vector{UInt8}). + +On exit, a closing handshake is started. If the server is not currently reading +(which is a blocking function), this side will reset the underlying connection (ECONNRESET) +after a reasonable amount of time and continue execution. +""" function open(f::Function, url; verbose=false, optionalProtocol = "", kw...) key = base64encode(rand(UInt8, 16)) - headers = [ "Upgrade" => "websocket", "Connection" => "Upgrade", @@ -14,44 +32,81 @@ function open(f::Function, url; verbose=false, optionalProtocol = "", kw...) push!(headers, "Sec-WebSocket-Protocol" => optionalProtocol ) end - HTTP.open("GET", url, headers; - reuse_limit=0, verbose=verbose ? 2 : 0, kw...) do http + try + HTTP.open("GET", url, headers; + reuse_limit=0, verbose=verbose ? 2 : 0, kw...) do http - HTTP.startread(http) + HTTP.startread(http) - status = http.message.status - if status != 101 - return - end + status = http.message.status + if status != 101 + return + end - check_upgrade(http) + check_upgrade(http) - if HTTP.header(http, "Sec-WebSocket-Accept") != generate_websocket_key(key) - throw(WebSocketError(0, "Invalid Sec-WebSocket-Accept\n" * - "$(http.message)")) - end + if HTTP.header(http, "Sec-WebSocket-Accept") != generate_websocket_key(key) + throw(WebSocketError(0, "Invalid Sec-WebSocket-Accept\n" * + "$(http.message)")) + end - io = HTTP.ConnectionPool.getrawstream(http) - ws = WebSocket(io,false) - try - f(ws) - finally - close(ws) + io = HTTP.ConnectionPool.getrawstream(http) + ws = WebSocket(io,false) + try + f(ws) + finally + close(ws) + end + end + catch err + if typeof(err) == Base.UVError + warn(STDERR, err) + else + rethrow(err) end end end """ -Responds to a WebSocket handshake request. +Used as part of a server definition. Call this if +is_upgrade(http.message) returns true. + +Responds to a WebSocket handshake request. If the connection is acceptable, sends status code 101 and headers according to RFC 6455, then calls user's handler function f with the connection wrapped in -a WebSocket instance with the open socket as one of the fields. +a WebSocket instance. + +f(ws) is called with the websocket and no client info +f(headers, ws) also receives a dictionary of request headers for added security measures + +On exit from f, a closing handshake is started. If the client is not currently reading +(which is a blocking function), this side will reset the underlying connection (ECONNRESET) +after a reasonable amount of time and continue execution. -Closes the websocket after returning from f. +If the upgrade is not accepted, responds to client with '400'. -Otherwise responds with '400' and returns nothing. -Calls user's handler function f upon a successful upgrade. +e.g. server with local error handling. Combine with WebSocket.open example. +```julia +import HTTP +using WebSockets + +badgatekeeper(reqdict, ws) = sqrt(-2) +handlerequest(req) = HTTP.Response(501) + +try + HTTP.listen("127.0.0.1", UInt16(8000)) do http + if WebSockets.is_upgrade(http.message) + WebSockets.upgrade(badgatekeeper, http) + else + HTTP.Servers.handle_request(handlerequest, http) + end + end +catch err + showerror(err) + println.(catch_stacktrace()[1:4]) +end +``` """ function upgrade(f::Function, http::HTTP.Stream) # Double check the request. is_upgrade should already have been called by user. @@ -96,7 +151,7 @@ function upgrade(f::Function, http::HTTP.Stream) f(ws) end catch err - warn(STDERR, "WebSockets.HTTP.upgrade: Caught unhandled error while calling argument function f, the handler / gatekeeper:\n\t") + warn("WebSockets.HTTP.upgrade: Caught unhandled error while calling argument function f, the handler / gatekeeper:\n\t") mt = typeof(f).name.mt fnam = splitdir(string(mt.defs.func.file))[2] print_with_color(:yellow, STDERR, "f = ", string(f) * " at " * fnam * ":" * string(mt.defs.func.line) * "\nERROR:\t") diff --git a/src/WebSockets.jl b/src/WebSockets.jl index 87fcf07..50e6e94 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -21,8 +21,7 @@ an http server. module WebSockets import MbedTLS: digest, MD_SHA1 -using Requires - +using Requires export WebSocket, write, read, @@ -31,15 +30,17 @@ export WebSocket, send_pong const TCPSock = Base.TCPSocket +"A reasonable amount of time" +const TIMEOUT_CLOSEHANDSHAKE = 10.0 @enum ReadyState CONNECTED=0x1 CLOSING=0x2 CLOSED=0x3 """ Buffer writes to socket till flush (sock)""" -init_socket(sock) = Base.buffer_writes(sock) +init_socket(sock) = Base.buffer_writes(sock) -struct WebSocketClosedError <: Exception end -Base.showerror(io::IO, e::WebSocketClosedError) = print(io, "Error: client disconnected") +#struct WebSocketClosedError <: Exception end +#Base.showerror(io::IO, e::WebSocketClosedError) = print(io, "Error: client disconnected") struct WebSocketError <: Exception status::Int16 @@ -110,9 +111,9 @@ Add to supported SUBProtocols through e.g. ``` WebSockets.addsubproto("special-protocol") WebSockets.addsubproto("json") -``` -In the general websocket handler function, specialize -further by checking +``` +In the general websocket handler function, specialize +further by checking # Example ``` if get(wsrequest.headers, "Sec-WebSocket-Protocol", "") = "special-protocol" @@ -122,7 +123,7 @@ else end ``` """ -const SUBProtocols= Array{String,1}() +const SUBProtocols= Array{String,1}() "Used in handshake. See SUBProtocols" hasprotocol(s::AbstractString) = in(s, SUBProtocols) @@ -132,7 +133,7 @@ function addsubproto(name) push!(SUBProtocols, string(name)) return true end -""" +""" write_fragment(io, islast, opcode, hasmask, data::Array{UInt8}) Write the raw frame to a bufffer """ @@ -200,30 +201,41 @@ end """ Send a pong message, optionally with data.""" send_pong(ws, data...) = write_pong(ws.socket, !ws.server, data...) -""" +""" close(ws::WebSocket) -Send a close message. +Send an OPCODE_CLOSE frame, wait for the same response while dropping +further received data. If not, consider as closed after waiting a +reasonable amount of time, TIMEOUT_CLOSEHANDSHAKE. """ function Base.close(ws::WebSocket) if isopen(ws) ws.state = CLOSING locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, UInt8[]) - # Wait till the other end responds with an OPCODE_CLOSE. This process is + # Wait till the peer responds with an OPCODE_CLOSE while discarding any + # trailing bytes received. + # + # We have no guarantee that the peer is actually reading our OPCODE_CLOSE + # frame. If not, the peer's state will not change, and we will not receive + # an aknowledgment of closing. We use a nonblocking read and give up + # after TIMEOUT_CLOSEHANDSHAKE + # + # This process is # complicated by potential blocking reads on the WebSocket in other Tasks # which may receive the response control frame. Synchronization of who is # responsible for closing the underlying socket is done using the # WebSocket's state. When this side initiates closing the connection it is # responsible for cleaning up, when the other side initiates the close the - # read method is + # read method is. # # The exception handling is necessary as read_frame will error when the # OPCODE_CLOSE control frame is received by a potentially blocking read in # another Task + # try - while isopen(ws) - wsf = read_frame(ws) - # ALERT: stuff might get lost in ether here + t1 = time() + TIMEOUT_CLOSEHANDSHAKE + while isopen(ws) && time() < t1 + wsf = readframe_nonblocking(ws) if is_control_frame(wsf) && (wsf.opcode == OPCODE_CLOSE) ws.state = CLOSED end @@ -231,8 +243,14 @@ function Base.close(ws::WebSocket) if isopen(ws.socket) close(ws.socket) end - catch exception - !isa(exception, EOFError) && rethrow(exception) + catch err + # Typical 'errors' received while closing down are neglected. + errtyp = typeof(err) + errtyp != InterruptException && + errtyp != Base.UVError && + errtyp != Base.BoundsError && + errtyp != Base.EOFError && + rethrow(err) end else ws.state = CLOSED @@ -307,21 +325,14 @@ end """ Read a frame: turn bytes from the websocket into a WebSocketFragment.""" function read_frame(ws::WebSocket) - ab = read(ws.socket,2) - #= - TODO error handling decision... + # Try to read two bytes. There is no guarantee that two bytes are actually allocated. + ab = read(ws.socket, 2) + #= Browsers will seldom close in the middle of writing to a socket, but other clients often do, and the stacktraces can be very long. - ab is often not assigned. - We could check for and throw a WebSocketError: - isassigned(ab) && throw(WebSocketError(0, "Socket closed while reading")) - This is often triggered: - BoundsError: attempt to access 0-element Array{UInt8,1} at index [1] - ...and sometimes also: - BoundsError: attempt to access 0-element Array{UInt8,1} at index [2] - If the last message is thrown here, ab has been assigned but only partly - read. - =# + ab can be assigned, but of length 1. An enclosing try..catch in the calling function + seems to + =# a = ab[1] fin = a & 0b1000_0000 >>> 7 # If fin, then is final fragment rsv1 = a & 0b0100_0000 # If not 0, fail. @@ -351,32 +362,105 @@ end """ read(ws::WebSocket) +Typical use: + String(read(ws)) Read one non-control message from a WebSocket. Any control messages that are -read will be handled by the handle_control_frame function. This function will -not return until a full non-control message has been read. If the other side -doesn't ever complete its message, this function will never return. Only the -data (contents/body/payload) of the message will be returned from this -function. +read will be handled by the handle_control_frame function. +Only the data (contents/body/payload) of the message will be returned as a +Vector{UInt8}. + +This function will not return until a full non-control message has been read. If the other side +doesn't ever complete its message, this function will never return. +If the socket is closed without following protocol, or this function receives +an InterruptException, it : + - writes a warning to STDERR. This can be redirected using logging(..) + - returns an empty array and closes the socket from this side. """ function Base.read(ws::WebSocket) + # TODO add a try..catch...finally with + # error(WebSocketClosedError()) + # which also ensures Array{UInt8,1}()) is returned instead of Void() . + # EOF will fail conversion to String, but so will String(Void()) + # if !isopen(ws) error("Attempt to read from closed WebSocket") end - frame = read_frame(ws) + # Previous versions would retrun any of the errors below or return Void(), + # which would result in the errors below during conversion to String() + # This try..catch adds roughly 20 ns to each read(ws) + try + frame = read_frame(ws) + # Handle control (non-data) messages. + if is_control_frame(frame) + # Don't return control frames; they're not interesting to users. + handle_control_frame(ws,frame) + return frame.data + end - # Handle control (non-data) messages. - if is_control_frame(frame) - # Don't return control frames; they're not interesting to users. - handle_control_frame(ws,frame) + # Handle data message that uses multiple fragments. + if !frame.is_last + return vcat(frame.data, read(ws)) + end return frame.data + catch err + if typeof(err) == InterruptException + # This exception originates from this side. Follow protocol so no errors are triggered on the other side. + close(ws) + warn(STDERR, " While reading websocket, received InterruptException") + elseif typeof(err) == Base.UVError + warn(STDERR, " While reading websocket, received UVError, prefix:", err.prefix, ", code:", err.code) + elseif typeof(err) == Base.BoundsError + warn(STDERR, " While reading websocket, received incomplete frame") + elseif typeof(err) == Base.EOFError + warn(STDERR, " While reading, received EOFError") + else + # Unknown cause, give up continued execution. + # If this actually happens in a multiple fragment message, the accumulated + # stacktrace could be very long since read(ws) is iterative. + ws.state = CLOSED + if isopen(ws.socket) + close(ws.socket) + end + rethrow(err) + end + ws.state = CLOSED + if isopen(ws.socket) + close(ws.socket) + end end + return Vector{UInt8}() +end - # Handle data message that uses multiple fragments. - if !frame.is_last - return vcat(frame.data, read(ws)) +""" +For the closing handshake, we won't wait indefinitely for non-responsive clients. +Returns a throwaway frame if the socket happens to be empty +""" +function readframe_nonblocking(ws) + chnl= Channel{WebSocketFragment}(1) + # Read, output put to Channel for type stability + function _readinterruptable(c::Channel{WebSocketFragment}) + try + put!(chnl, read_frame(ws)) + catch + # Output a dummy frame that is not a control frame. + put!(chnl, WebSocketFragment(false, false, false, false, + UInt8(0), false, UInt64(0), + Vector{UInt8}([0x0,0x0,0x0,0x0]), + Vector{UInt8}())) + end end - - return frame.data + # Start reading as a task. Will not return if there is nothing to read + rt = @schedule _readinterruptable(chnl) + bind(chnl, rt) + yield() + # Define a task for throwing interrupt exception to the (possibly blocked) read task. + # We don't start this task because it would never return + killta = @task try;Base.throwto(rt, InterruptException());catch;end + # We start the killing task. When it is scheduled the second time, + # we pass an InterruptException through the scheduler. + try;schedule(killta, InterruptException(), error = false);catch;end + # We now have content on chnl, and no additional tasks. + take!(chnl) end """ diff --git a/test/runtests.jl b/test/runtests.jl index 075bdb9..c21327f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -30,7 +30,7 @@ end server = Server(wsh) @async run(server,port_HttpServer) -sleep(2) +sleep(4) servers = [ ("ws", "ws://echo.websocket.org"), From 404b3b6e56a67343c49031b37714bcd714e53e54 Mon Sep 17 00:00:00 2001 From: hustf Date: Mon, 14 May 2018 21:32:15 +0200 Subject: [PATCH 23/35] modified: benchmark/functions_benchmark.jl Improved explanations, dropped dead code densityplot modified: benchmark/ws_hts.jl Added testing for pre-existing sockets. modified: benchmark/ws_jce.jl Added explanation for currently unused 'delay' code modified: examples/chat_explore.jl Added reference to alternative close-down method --- benchmark/functions_benchmark.jl | 76 +++++++++++++++----------------- benchmark/ws_hts.jl | 10 +++++ benchmark/ws_jce.jl | 20 +++++---- examples/chat_explore.jl | 12 ++++- 4 files changed, 68 insertions(+), 50 deletions(-) diff --git a/benchmark/functions_benchmark.jl b/benchmark/functions_benchmark.jl index 0019189..91324fd 100644 --- a/benchmark/functions_benchmark.jl +++ b/benchmark/functions_benchmark.jl @@ -32,7 +32,7 @@ end """ -Make and get a reference to the server side of HTS-JCE websocket connection. +Make connection and get a reference to the HTS side of HTS-JCE connection. Assumes a server is already running. """ function get_hts_jce() @@ -65,7 +65,7 @@ function get_hts_bce() hts = Union{WebSocket, String}("") browser ="" opened, browser = open_a_browser() - # Launch the next browser in line. + # Launch the next browser in line. if opened zlog(id, "BCE in, ", browser, " will be connecting to HTS. Getting reference...") zflush() @@ -82,17 +82,18 @@ function get_hts_bce() return hts, browser end """ -Collect n (samples) of serverlatency and clientlatency. -Also calculate derived sizes - serverbandwidth, clientbandwidth, serverspeeds, clientspeed -Starts a new JSE-HTS connection for every evaluation (call). +Send n messages of length messagesize HTS-JCE and back. + -> id, serverlatencies, clientlatencies +Starts and closes a new HTS-JCE connection every function call. +In the terminology of BenchmarkTools, a call is a sample +consisting of n evaluations. """ function HTS_JCE(n, messagesize) id = "HTS_JCE" zlog(id, "Warming up, compiling") zflush() hts = get_hts_jce() - if !isa(hts, WebSocket) + if !isa(hts, WebSocket) msg = " could not get websocket reference" clog(id, msg) error(id * msg) @@ -130,7 +131,7 @@ function HTS_JCE(n, messagesize) end # We must read from the websocket in order for it to respond to # a closing message from JCE. It's also nice to yield to the async server - # so it can exit from it's handler. + # so it can exit from it's handler and release the websocket reference. isopen(hts) && read(hts) yield() serverlatencies = receivetimes - sendtimes @@ -138,8 +139,8 @@ function HTS_JCE(n, messagesize) return id, serverlatencies, clientlatencies end """ -Use the next browser in line, starts a new HTS-BCE connection and collects n evaluations. -This is one sample. +Use the next browser in line, start a new HTS-BCE connection and collect n evaluations. +This is one sample. Returns browser name and vectors with n rows: # t1 Send 0, receive 0, measure time interval # t2 Send 0, receive 0 twice, measure time interval @@ -152,9 +153,9 @@ function HTS_BCE(n, x) zflush() msg = "" (hts, browser) = get_hts_bce() - if browser == "" + if browser == "" msg = "Could not find and open more browser types" - elseif !isa(hts, WebSocket) + elseif !isa(hts, WebSocket) msg = " could not get ws reference from " * browser * " via HTS" end if msg != "" @@ -213,16 +214,16 @@ function HTS_BCE(n, x) end " -Measured time interval vectors [ns] -> client and server speeds, bandwidth [ns/b] -x is message size [b]. +Constant message size [b], measured time interval vectors [ns] + -> server and client speeds, server and client bandwidth [ns/b] " -function serverandclientspeeds(x, serverlatencies, clientlatencies) - serverspeeds = serverlatencies / x - clientspeeds = clientlatencies / x +function serverandclientspeeds(messagesize, serverlatencies, clientlatencies) + serverspeeds = serverlatencies / messagesize + clientspeeds = clientlatencies / messagesize n = length(serverspeeds) - serverbandwidth = sum(serverlatencies) / (n * x) - clientbandwidth = sum(clientlatencies) / (n * x) + serverbandwidth = sum(serverlatencies) / (n * messagesize) + clientbandwidth = sum(clientlatencies) / (n * messagesize) serverspeeds, clientspeeds, serverbandwidth, clientbandwidth end @@ -234,20 +235,20 @@ x is message size [b]. function serverandclientspeeds_indirect(x, t1, t2, t3, t4) # assuming a measured time interval consists of # t(x) = t0s + t0c + a*x + b*x - # where + # where # t(x) measured time at the server - # t0s initial serverlatency, for a "zero length" message + # t0s initial serverlatency, for a "zero length" message # t0c initial clientlatency, for a "zero length" message # x message length [b] # a marginal server speed [ns/b] # b marginal client speed [ns/b] # - # + # # t1 = t0s + t0c Send 0, receive 0, measure t1 # t2 = t0s + 2t0c Send 0, receive 0 twice, measure t2 # t3 = t0s + t0c + a*x Send x, receive 0, measure t3 # t4 = t0s + 2t0c + a*x + b*x Send x, receive 0, receive x, measure t4 - # + # # hence, t0s = 2t1 - t2 t0c = -t1 + t2 @@ -259,20 +260,20 @@ function serverandclientspeeds_indirect(x, t1, t2, t3, t4) serverspeeds = t0s / x + a clientspeeds = t0c / x + b n = length(serverspeeds) - serverbandwidth = sum(serverspeeds) / n + serverbandwidth = sum(serverspeeds) / n clientbandwidth = sum(clientspeeds) / n return serverspeeds, clientspeeds, serverbandwidth, clientbandwidth end -## Note these shorthand function require input symbols defined at module-level -## (not in a local scope) +## Note that the shorthand plot functions below require input symbols +## that are defined at module-level (not in a local scope) -"Shorthand for generating a time series lineplot in REPL" +"Generate a time series lineplot" lp(sy::Symbol) = lineplot(collect(1:length(eval(sy))), eval(sy), title = String(sy), width = displaysize(STDOUT)[2]-20, canvas = AsciiCanvas) -"Return multiple time series lineplots with a common title prefix" +"Generate a vector of time series lineplots with a common title prefix" function lp(symbs::Vector{Symbol}, titleprefix) map(symbs) do sy pl = lp(sy) @@ -280,12 +281,13 @@ function lp(symbs::Vector{Symbol}, titleprefix) end end -"Shorthand for generating an x-y lineplot in REPL" -function lp(syx::Symbol, syy::Symbol) +"Generate an x-y lineplot in REPL" +function lp(syx::Symbol, syy::Symbol) lpl = lineplot(eval(syx), eval(syy), title = String(syy), width = displaysize(STDOUT)[2]-20, canvas = AsciiCanvas) xlabel!(lpl, String(syx)) end -"Return multiple x-y lineplots with a common title prefix" + +"Generate a vector of x-y lineplots with a common title prefix" function lp(syxs::Vector{Symbol}, syys::Vector{Symbol}, titleprefix) map(zip(syxs, syys)) do pair pl = lp(pair[1], pair[2]) @@ -293,14 +295,8 @@ function lp(syxs::Vector{Symbol}, syys::Vector{Symbol}, titleprefix) end end -"Shorthand for generating an x-y scatterplot in REPL" -function sp(syx::Symbol, syy::Symbol) +"Generate an x-y scatterplot" +function sp(syx::Symbol, syy::Symbol) spl = scatterplot(eval(syx), eval(syy), title = String(syy), width = displaysize(STDOUT)[2]-15, canvas = DotCanvas) xlabel!(spl, String(syx)) -end - -"Shorthand for generating a densityplot in REPL" -function dp(syx::Symbol, syy::Symbol) - dpl = densityplot(eval(syx), eval(syy), title = String(syy), color = :red, width = displaysize(STDOUT)[2]-15) - xlabel!(dpl, String(syx)) -end +end \ No newline at end of file diff --git a/benchmark/ws_hts.jl b/benchmark/ws_hts.jl index 7ce3b15..c949303 100644 --- a/benchmark/ws_hts.jl +++ b/benchmark/ws_hts.jl @@ -48,6 +48,16 @@ function acceptholdws(http) # If the ugrade is successful, just hold the reference and thread # of execution. Other tasks may do useful things with it. WebSockets.upgrade(http) do ws + if length(WEBSOCKET) > 0 + # unexpected behaviour. + if isclosed(WEBSOCKET[1]) + pop!(WEBSOCKET) + else + msg = " A websocket is already open. Not accepting the attempt at opening more." + clog(id, :red, msg);zflush() + return + end + end push!(WEBSOCKET, ws) zlog(id, ws);zflush() t1 = now() + WSMAXTIME diff --git a/benchmark/ws_jce.jl b/benchmark/ws_jce.jl index f4b193b..52632cd 100644 --- a/benchmark/ws_jce.jl +++ b/benchmark/ws_jce.jl @@ -20,14 +20,20 @@ const SERVER = "ws://127.0.0.1:$(PORT)" const CLOSEAFTER = Base.Dates.Second(30) """ -Opens a client, echoes with optional delay, an integer in milliseconds. +Opens a client, echoes with an optional delay, an integer in milliseconds. Stores time records for received messages and before sending messages. -Specify delay in milliseconds by sending a message on the websocket: +Specify the delay in milliseconds by sending a message on the websocket: send(ws_jce, "delay|15") -Echoes any message except "exit" and "delay" +Echoes any message except "exit" and "delay". -At exit or after CLOSEAFTER, sends one message containing two vectors of -timestamps [ns].AbstractTrees +Delays to reading, in the websocket use situation, would be caused by usefully spent +calculation time between reads. However, they may be interpreted by the underlying protocol +as transmission problems and cause large slowdowns. Hence the interest in testing +with delays. A countermeasure for optimizing speed might be to run a websocket +reading function in a parallel, not asyncronous process, putting messages on an internal queue. + +At exit or after CLOSEAFTER, this function sends one message containing two vectors of +timestamps [ns]. """ function echowithdelay_jce() # This will be run in a worker process. Even so, individual console log @@ -56,7 +62,7 @@ function echowithdelay_jce() end end " -Handler for client websocket, defined by echowithdelay +Handler for client websocket, defined by echowithdelay_jce " function _jce(ws) id = "_jce" @@ -109,8 +115,6 @@ function _jce(ws) clog(id, :green, " Exit, close websocket.") zflush() # Exiting this function starts a closing handshake - # from this side (HTTP.jl:39). The other side must read in - # in order for this to proceed smoothly. nothing end end # module diff --git a/examples/chat_explore.jl b/examples/chat_explore.jl index 0838a70..68f63f7 100644 --- a/examples/chat_explore.jl +++ b/examples/chat_explore.jl @@ -12,7 +12,7 @@ named functions may improve error message readability. =# - +# TODO fix errors and style # Globals, where used in functions will change the type global lastreq = 0 @@ -238,8 +238,14 @@ end info("Start HTTP server on port $(HTTPPORT)") litas_newtype = @schedule HTTP.listen(server_def_newtype, "127.0.0.1", UInt16(HTTPPORT)) -# TODO replace with improvement from 'close websocket'. +""" +This stops the servers using InterruptExceptions. +""" function closefromoutside() + # Throwing exceptions can be slow. This function also + # starts a task which seems to not exit and free up + # its memory properly. HTTP.listen offers an alternative + # method. See HTTP.listen > tcpref if isdefined(:litas_newtype) @schedule Base.throwto(litas_newtype, InterruptException()) end @@ -251,4 +257,6 @@ function closefromoutside() end end end + + nothing \ No newline at end of file From 8edb4597f84d5efbe8cf1d0b2fc5b6d5ca03ad2f Mon Sep 17 00:00:00 2001 From: hustf Date: Thu, 24 May 2018 23:30:58 +0200 Subject: [PATCH 24/35] Do not return control frames like ping. Recurse to the next frame like in master. modified: src/WebSockets.jl Reintroduce recursion on control frames:376. Returning control codes would crash String(msg). Multi-frame messages are also possible. Add WebSocketClosedError messages :47, for underlying exceptions. Propagate unrecognized errors. Remove INFO message when closing. Readguarded() informs better. Added functions: readguarded return tuple: data and success indication incomplete messages are always empty writeguarded return success indication subprotocol(request) common for Httpserver and HTTP target(request) common for Httpserver and HTTP origin(request) common for Httpserver and HTTP modified: src/HTTP.jl make upgrade(ws) call websocket handler with full request. Origin can now be determined, as per recommendations. call showerror with catch_stacktrace(). backtrace was less interesting. add method target(request) add method origin(request) add method subprotocol(request) add ServerWS WebsocketHandler serve(::ServerWS, etc..) The intententions is user code brevity, access to HTTP keyword options, more similar interface with HttpServer, can save the user from distinguishing Stream and Message. modified: src/HttpServer.jl add method target(request) add method origin(request) add method subprotocol(request) improve inline doc for WebSocketHandler modified: examples/chat_explore.jl demonstrate new functions, improve readability modified: benchmark/functions_benchmark.jl capture WebSocketClosedError with readguarded(ws) modified: logutils/log_httpserver.jl explicitly import Request and Response from HttpServer modified: test/functions_server.jl rephrase inline docs, use subprotocol(request) modified: test/runtests.jl not rely on pong returning read(ws) call --- benchmark/functions_benchmark.jl | 4 +- examples/chat_explore.jl | 315 +++++++++++-------------------- logutils/log_httpserver.jl | 4 +- src/HTTP.jl | 102 +++++++++- src/HttpServer.jl | 26 ++- src/WebSockets.jl | 289 ++++++++++++++++++++-------- test/functions_server.jl | 40 ++-- test/runtests.jl | 41 ++-- 8 files changed, 486 insertions(+), 335 deletions(-) diff --git a/benchmark/functions_benchmark.jl b/benchmark/functions_benchmark.jl index 91324fd..1354b50 100644 --- a/benchmark/functions_benchmark.jl +++ b/benchmark/functions_benchmark.jl @@ -111,7 +111,7 @@ function HTS_JCE(n, messagesize) sendtime = time_ns() write(hts, msg) # throw away replies - read(hts) + readguarded(hts) receivereplytime = time_ns() push!(receivereplytimes, Int64(receivereplytime < typemax(Int64) ? receivereplytime : 0 )) push!(sendtimes, Int64(sendtime < typemax(Int64) ? sendtime : 0 )) @@ -132,7 +132,7 @@ function HTS_JCE(n, messagesize) # We must read from the websocket in order for it to respond to # a closing message from JCE. It's also nice to yield to the async server # so it can exit from it's handler and release the websocket reference. - isopen(hts) && read(hts) + isopen(hts) && readguarded(hts) yield() serverlatencies = receivetimes - sendtimes clientlatencies = receivereplytimes - replytimes diff --git a/examples/chat_explore.jl b/examples/chat_explore.jl index 68f63f7..ea91935 100644 --- a/examples/chat_explore.jl +++ b/examples/chat_explore.jl @@ -1,46 +1,89 @@ #= -Difference to chat-client: -This example declares global variables and use duck typing on -them. Their types will change. -The aim is that you can examine types in the REPL while running the -example. The aim is NOT that you can expect clean exits. We don't -release all the references after you close connections. -Function containers are explicitly defined with names. Although -anonymous functions may be more commonly used in the web domain, -named functions may improve error message readability. +A chat application using both HttpServer and HTTP to do +the same thing: Start a new task for each browser (tab) that connects. -=# +To use: + - include("chat_explore.jl") in REPL + - start a browser on address 127.0.0.1:8000, and another on 127.0.0.1:8080 + - inspect global variables starting with 'last' while the chat is running asyncronously + +To call in from other devices, figure out your IP address on the network and change the 'gatekeeper' code. + +Note that type of 'lastreq' changes depending on whether the last call was made through HttpServer or HTTP. -# TODO fix errors and style +Functions used as arguments are explicitly defined with names instead of anonymous functions (do..end constructs). +This may improve debugging readability at the cost of increased verbosity. -# Globals, where used in functions will change the type +=# global lastreq = 0 -global lastreqheadersdict = 0 global lastws= 0 -global lastwsHTTP = 0 -global lastdata= 0 global lastmsg= 0 -global lasthttp= 0 global lastws= 0 -global laste= 0 -global lasthttp= 0 using HttpServer using HTTP using WebSockets const CLOSEAFTER = Base.Dates.Second(1800) const HTTPPORT = 8080 -const PORT_OLDTYPE = 8000 +const HTTPSERVERPORT = 8000 +const URL = "127.0.0.1" const USERNAMES = Dict{String, WebSocket}() +const HTMLSTRING = readstring(Pkg.dir("WebSockets","examples","chat_explore.html")); + +# If we are to access a websocket from outside +# it's websocket handler function, we need some kind of +# mutable container for storing references: const WEBSOCKETS = Dict{WebSocket, Int}() +""" +Called by 'gatekeeper', this function stays active while the +particular websocket is open. The argument is an open websocket. +Other instances of the function run in other tasks. The tasks +are generated by either HTTP or HttpServer. +""" +function usews(thisws) + global lastws = thisws + push!(WEBSOCKETS, thisws => length(WEBSOCKETS) +1 ) + t1 = now() + CLOSEAFTER + username = "" + while now() < t1 + # This next call waits for a message to + # appear on the socket. If there is none, + # this task yields to other tasks. + data, success = readguarded(thisws) + !success && break + global lastmsg = msg = String(data) + print("Received: $msg ") + if username == "" + username = approvedusername(msg, thisws) + if username != "" + println("from new user $username ") + !writeguarded(thisws, username) && break + println("Tell everybody about $username") + foreach(keys(WEBSOCKETS)) do ws + writeguarded(ws, username * " enters chat") + end + else + println(", username taken!") + !writeguarded(thisws, "Username taken!") && break + end + else + println("from $username ") + distributemsg(msg, thisws) + startswith(msg, "exit") && break + end + end + exitmsg = username == "" ? "unknown" : username * " has left" + distributemsg(exitmsg, thisws) + println(exitmsg) + # No need to close the websocket. Just clean up external references: + removereferences(thisws) + nothing +end -#= -low level functions, works on old and new type. -=# function removereferences(ws) - ws in keys(WEBSOCKETS) && pop!(WEBSOCKETS, ws) + haskey(WEBSOCKETS, ws) && pop!(WEBSOCKETS, ws) for (discardname, wsref) in USERNAMES if wsref === ws pop!(USERNAMES, discardname) @@ -50,55 +93,8 @@ function removereferences(ws) nothing end -function process_error(id, e) - if typeof(e) == InterruptException - info(id, "Received exit order.") - elseif typeof(e) == ArgumentError - info(id, typeof(e), "\t", e.msg) - elseif typeof(e) == ErrorException - info(id, typeof(e), "\t", e.msg) - else - if :msg in fieldnames(e) && e.msg == "Attempt to read from closed WebSocket" - warn(id, typeof(e), "\t", e.msg) - else - warn(id, e, "\nStacktrace:", stacktrace(true)) - end - end -end - -function protectedwrite(ws, msg) - global laste - try - write(ws, msg) - catch e - laste = e - process_error("chat_explore.protectedwrite: ", e) - removereferences(ws) - return false - end - true -end - -function protectedread(ws) - global laste - global lastdata - data = Vector{UInt8}() - contflag = true - try - data = read(ws) - lastdata = data - catch e - laste = e - contflag = false - process_error("chat_explore.protectedread: ", e) - finally - return data, contflag - end -end - - -function findusername(msg, ws) +function approvedusername(msg, ws) !startswith(msg, "userName:") && return "" newname = msg[length("userName:") + 1:end] newname =="" && return "" @@ -111,152 +107,63 @@ end function distributemsg(msgout, not_to_ws) foreach(keys(WEBSOCKETS)) do ws if ws !== not_to_ws - protectedwrite(ws, msgout) + writeguarded(ws, msgout) end end nothing end -function wsfunc(thisws) - global lastws - global lastmsg - lastws = thisws - push!(WEBSOCKETS, thisws => length(WEBSOCKETS) +1 ) - contflag = true - t0 = now() - data = Vector{UInt8}() - msg = "" - username = "" - changedname = false - while now()-t0 < CLOSEAFTER && contflag - data, contflag = protectedread(thisws) - if contflag - msg = String(data) - lastmsg = msg - println("Received: $msg") - if username == "" - username = findusername(msg, thisws) - if username != "" - if !protectedwrite(thisws, username) - contflag = false - end - println("Tell everybody about $username") - foreach(keys(WEBSOCKETS)) do ws - protectedwrite(ws, username * " enters chat") - end - else - println("Username taken!") - if !protectedwrite(thisws, "Username taken!") - contflag = false - end - end - else - contflag = !startswith(msg, "exit") - contflag || println("Received exit message. Closing.") - end - end - end - exitusername = username == "" ? "unknown" : username - distributemsg(exitusername * " has left", thisws) - removereferences(thisws) - # It's not this functions responsibility to close the websocket. Just to forget about it. - nothing -end - - - -#= -Functions for old type i.e. HttpServer based connections. -This function is called after handshake, and after -subprotocol is checked against a list of user supported subprotocols. -=# -function gatekeeper_oldtype(req, ws) - global lastreq - global lastws - lastreq = req - lastws = ws - # Here we can pick between functions - # based on e.g. - # if haskey(req.headers,"Sec-WebSocket-Protocol") - # - wsfunc(ws) -end - - -# Just for easy REPL inspection, we'll declare the handler object explicitly. -# With handler we mean an instance of a structure with at least one function reference. -handler_ws_oldtype = WebSocketHandler(gatekeeper_oldtype) -# explicit http server handlers -httpfunc_oldtype(req, res) = readstring(Pkg.dir("WebSockets","examples","chat_explore.html")) |> Response -handler_http_oldtype = HttpHandler(httpfunc_oldtype) -# define both in one server. We could call this a handler, too, since it's just a -# bigger function structure. Or we may call it an object. -server_def_oldtype = Server(handler_http_oldtype, handler_ws_oldtype ) - -#= - Now we'll run an external program which starts - the necessary tasks on Julia. - We can run this async, which might be considered - bad pracice and leads to more bad connections. - For debugging and building programs, it's gold to run this async. - You can close a server that's running in a task using - @schedule Base.throwto(listentask, InterruptException()) - =# -litas_oldtype = @schedule run(server_def_oldtype, PORT_OLDTYPE) - -info("Chat server listening on $PORT_OLDTYPE") -#= - -Now open another port using HTTP instead of HttpServer -We'll start by defining the input functions for HTTP's listen method - -=# - - -function gatekeeper_newtype(reqheadersdict, ws) - global lastreqheadersdict - lastreqheadersdict = reqheadersdict - global lastwsHTTP - lastwsHTTP = ws - # Inspect header Sec-WebSocket-Protocol to pick the right function. - wsfunc(ws) -end - -httpfunc_newtype(req::HTTP.Request) = readstring(Pkg.dir("WebSockets","examples","chat_explore.html")) |> HTTP.Response +""" +`Server => gatekeeper(Request, WebSocket) => usews(WebSocket)` -function server_def_newtype(http) - global lasthttp - lasthttp = http - if WebSockets.is_upgrade(http.message) - WebSockets.upgrade(gatekeeper_newtype, http) +The gatekeeper makes it a little harder to connect with +malicious code. It inspects the request that was upgraded +to a a websocket. +""" +function gatekeeper(req, ws) + global lastreq = req + global lastws = ws + orig = WebSockets.origin(req) + if startswith(orig, "http://localhost") || startswith(orig, "http://127.0.0.1") + usews(ws) else - HTTP.Servers.handle_request(httpfunc_newtype, http) + warn("Unauthorized websocket connection, $orig not approved by gatekeeper") end + nothing end -info("Start HTTP server on port $(HTTPPORT)") -litas_newtype = @schedule HTTP.listen(server_def_newtype, "127.0.0.1", UInt16(HTTPPORT)) +"Request to response. Response is the predefined HTML page with some javascript" +req2resp(req::HttpServer.Request, resp) = HTMLSTRING |> Response +req2resp(req::HTTP.Request) = HTMLSTRING |> HTTP.Response + +# Both server definitions need two function wrappers; one handler function for page requests, +# one for opening websockets (which the javascript in the HTML page will try to do) +server_httpserver = Server(HttpHandler(req2resp), WebSocketHandler(gatekeeper)) +server_HTTP = WebSockets.ServerWS(HTTP.HandlerFunction(req2resp), WebSockets.WebsocketHandler(gatekeeper)) + +# Start the HTTP server asyncronously, and stop it later +litas_HTTP = @schedule WebSockets.serve(server_HTTP, URL, HTTPPORT, false) +@schedule begin + println("HTTP server listening on $URL:$HTTPPORT for $CLOSEAFTER") + sleep(CLOSEAFTER.value) + println("Time out, closing down $HTTPPORT") + Base.throwto(litas_HTTP, InterruptException()) +end -""" -This stops the servers using InterruptExceptions. -""" -function closefromoutside() - # Throwing exceptions can be slow. This function also - # starts a task which seems to not exit and free up - # its memory properly. HTTP.listen offers an alternative - # method. See HTTP.listen > tcpref - if isdefined(:litas_newtype) - @schedule Base.throwto(litas_newtype, InterruptException()) - end - if isdefined(:litas_oldtype) - try - @schedule Base.throwto(litas_oldtype, InterruptException()) - catch e - info("closefromoutside: ", e) - end - end +# Start the HttpServer asyncronously, stop it later +litas_httpserver = @schedule run(server_httpserver, HTTPSERVERPORT) +@schedule begin + println("HttpServer listening on $URL:$HTTPSERVERPORT for $CLOSEAFTER") + sleep(CLOSEAFTER.value + 2) + println("Time out, closing down $HTTPSERVERPORT") + Base.throwto(litas_httpserver, InterruptException()) end +# Note that stopping the HttpServer in a while will send an error messages to the +# console. We could get rid of the messages by binding the task to a Channel. +# However, we can't get rid of ECONNRESET messages in that way. This is +# because the errors are triggered in tasks generated by litas_httpserver again, +# and those aren't channeled anywhere. nothing \ No newline at end of file diff --git a/logutils/log_httpserver.jl b/logutils/log_httpserver.jl index 5b6444b..f0af9a9 100644 --- a/logutils/log_httpserver.jl +++ b/logutils/log_httpserver.jl @@ -3,6 +3,8 @@ import HttpServer.HttpHandler import HttpServer.Server import HttpServer.ClientParser import HttpServer.HttpParser +import HttpServer.Request # Not sure if necessary, but otherwise dispatching on HttpCommon.Request can fail. +import HttpServer.Response import HttpCommon.Cookie import HttpCommon.STATUS_CODES import URIParser.URI @@ -45,7 +47,7 @@ end "Request already has a decent show method, we're not overwriting that. This metod is called only when logging to an Abstractdevice" -function _show(d::AbstractDevice, request::HttpServer.Request) +function _show(d::AbstractDevice, request::Request) _log(d, :normal, :light_yellow, "Request ", :normal) _log(d, :bold, request.method, " ", :cyan, request.resource, "\n", :normal) if !isempty(request.data) diff --git a/src/HTTP.jl b/src/HTTP.jl index 74c7525..ddd2434 100644 --- a/src/HTTP.jl +++ b/src/HTTP.jl @@ -145,8 +145,8 @@ function upgrade(f::Function, http::HTTP.Stream) io = HTTP.ConnectionPool.getrawstream(http) ws = WebSocket(io, true) try - if applicable(f, Dict(http.message.headers), ws) - f(Dict(http.message.headers), ws) + if applicable(f, http.message, ws) + f(http.message, ws) else f(ws) end @@ -155,7 +155,7 @@ function upgrade(f::Function, http::HTTP.Stream) mt = typeof(f).name.mt fnam = splitdir(string(mt.defs.func.file))[2] print_with_color(:yellow, STDERR, "f = ", string(f) * " at " * fnam * ":" * string(mt.defs.func.line) * "\nERROR:\t") - showerror(STDERR, err, backtrace()) + showerror(STDERR, err, catch_stacktrace()) finally close(ws) end @@ -168,7 +168,7 @@ function check_upgrade(http) throw(WebSocketError(0, "Check upgrade: Expected \"Upgrade => websocket\"!\n$(http.message)")) end if !(HTTP.hasheader(http, "Connection", "upgrade") || HTTP.hasheader(http, "Connection", "keep-alive, upgrade")) - throw(WebSocketError(0, "Check upgrade: Expected \"Connection => upgrade or Connection => keep alive, upgrad\"!\n$(http.message)")) + throw(WebSocketError(0, "Check upgrade: Expected \"Connection => upgrade or Connection => keep alive, upgrade\"!\n$(http.message)")) end end @@ -189,4 +189,98 @@ function is_upgrade(r::HTTP.Message) end return false end +# Inline docs in 'WebSockets.jl' +target(req::HTTP.Messages.Request) = req.resource +subprotocol(req::HTTP.Messages.Request) = HTTP.header(req, "Sec-WebSocket-Protocol") +origin(req::HTTP.Messages.Request) = HTTP.header(req, "Origin") +""" +WebsocketHandler(f::Function) <: HTTP.Handler + +A simple Function-wrapper for HTTP. +The provided argument should be one of the forms + `f(WebSocket) => nothing` + `f(HTTP.Request, WebSocket) => nothing` +The latter form is intended for gatekeeping, ref. RFC 6455 section 10.1 + +f accepts a `WebSocket` and does interesting things with it, like reading, writing and exiting when finished. + +Take note of the very similar WebSocketHandler (capital 'S'), which is a subtype of HttpServer, an alternative +to HTTP. +""" +struct WebsocketHandler{F <: Function} <: HTTP.Handler + func::F # func(ws) or func(request, ws) +end + + +""" +A ServerWS is a variant of HTTP.Server +which aims to hook into some of the higher-level functionality. +It is also possible to hook into the lower-level functionality +using `listen`. +""" +mutable struct ServerWS{T <: HTTP.Servers.Scheme, H <: HTTP.Handler, W <: WebsocketHandler} + handler::H + wshandler::W + logger::IO + in::Channel{Any} + out::Channel{Any} + options::HTTP.ServerOptions + + ServerWS{T, H, W}(handler::H, wshandler::W, logger::IO = HTTP.compat_stdout(), ch=Channel(1), ch2=Channel(1), + options=HTTP.ServerOptions()) where {T, H, W} = + new{T, H, W}(handler, wshandler, logger, ch, ch2, options) +end + + + +ServerWS(h::Function, w::Function, l::IO=HTTP.compat_stdout(); + cert::String="", key::String="", args...) = ServerWS(HTTP.HandlerFunction(h), WebsocketHandler(w), l; + cert=cert, key=key, args...) +function ServerWS(handler::H, + wshandler::W, + logger::IO = HTTP.compat_stdout(); + cert::String = "", + key::String = "", + args...) where {H <: HTTP.Handler, W <: WebsocketHandler} + if cert != "" && key != "" + serverws = ServerWS{HTTP.Servers.https, H, W}(handler, wshandler, logger, Channel(1), Channel(1), HTTP.ServerOptions(; sslconfig=HTTP.MbedTLS.SSLConfig(cert, key), args...)) + else + serverws = ServerWS{HTTP.Servers.http, H, W}(handler, wshandler, logger, Channel(1), Channel(1), HTTP.ServerOptions(; args...)) + end + return serverws +end + + +function serve(server::ServerWS{T, H, W}, host, port, verbose) where {T, H, W} + + tcpserver = Ref{HTTP.Sockets.TCPServer}() + + @async begin + while !isassigned(tcpserver) + sleep(1) + end + while true + val = take!(server.in) + val == HTTP.Servers.KILL && close(tcpserver[]) + end + end + + HTTP.listen(host, port; + tcpref=tcpserver, + ssl=(T == HTTP.Servers.https), + sslconfig = server.options.sslconfig, + verbose = verbose, + tcpisvalid = server.options.ratelimit > 0 ? HTTP.Servers.check_rate_limit : + (tcp; kw...) -> true, + ratelimits = Dict{IPAddr, HTTP.Servers.RateLimit}(), + ratelimit = server.options.ratelimit) do stream::HTTP.Stream + + if is_upgrade(stream.message) + upgrade(server.wshandler.func, stream) + else + HTTP.Servers.handle_request(server.handler.func, stream) + end + end + return +end \ No newline at end of file diff --git a/src/HttpServer.jl b/src/HttpServer.jl index 0162000..8000b5d 100644 --- a/src/HttpServer.jl +++ b/src/HttpServer.jl @@ -70,20 +70,32 @@ function websocket_handshake(request, client) response.headers["Connection"] = "Upgrade" response.headers["Sec-WebSocket-Accept"] = resp_key # TODO move this part further up, similar to in HTTP.jl - if haskey(request.headers, "Sec-WebSocket-Protocol") + if haskey(request.headers, "Sec-WebSocket-Protocol") if hasprotocol(request.headers["Sec-WebSocket-Protocol"]) response.headers["Sec-WebSocket-Protocol"] = request.headers["Sec-WebSocket-Protocol"] else Base.write(client.sock, HttpServer.Response(400)) return false end - end - + end + Base.write(client.sock, response) return true end -""" Implement the WebSocketInterface, for compatilibility with HttpServer.""" +""" +WebSocketHandler(f::Function) <: HttpServer.WebSocketInterface + +A simple Function-wrapper for HttpServer. + +The provided argument should be of the form + `f(Request, WebSocket) => nothing` + +Request is intended for gatekeeping, ref. RFC 6455 section 10.1. +WebSocket is for reading, writing and exiting when finished. + +Take note of the very similar WebsocketHandler (no capital 'S'), which is a subtype of HTTP. +""" struct WebSocketHandler <: HttpServer.WebSocketInterface handle::Function end @@ -116,4 +128,8 @@ function HttpServer.is_websocket_handshake(handler::WebSocketHandler, req::HttpS end end return false -end \ No newline at end of file +end +# Inline docs in WebSockets.jl +target(req::HttpServer.Request) = req.resource +subprotocol(req::HttpServer.Request) = get(req.headers, "Sec-WebSocket-Protocol", "") +origin(req::HttpServer.Request) = get(req.headers, "Origin", "") \ No newline at end of file diff --git a/src/WebSockets.jl b/src/WebSockets.jl index 50e6e94..4a6b0aa 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -23,9 +23,14 @@ module WebSockets import MbedTLS: digest, MD_SHA1 using Requires export WebSocket, + readguarded, + writeguarded, write, read, close, + subprotocol, + target, + origin, send_ping, send_pong @@ -39,8 +44,9 @@ const TIMEOUT_CLOSEHANDSHAKE = 10.0 init_socket(sock) = Base.buffer_writes(sock) -#struct WebSocketClosedError <: Exception end -#Base.showerror(io::IO, e::WebSocketClosedError) = print(io, "Error: client disconnected") +struct WebSocketClosedError <: Exception + message::String +end struct WebSocketError <: Exception status::Int16 @@ -106,33 +112,14 @@ const OPCODE_PONG = 0xA """ Handshakes with subprotocols are rejected by default. -Add to supported SUBProtocols through e.g. -# Examples -``` - WebSockets.addsubproto("special-protocol") +Add to acceptable SUBProtocols through e.g. +```julia WebSockets.addsubproto("json") ``` -In the general websocket handler function, specialize -further by checking -# Example -``` -if get(wsrequest.headers, "Sec-WebSocket-Protocol", "") = "special-protocol" - specialhandler(websocket) -else - generalhandler(websocket) -end -``` +Also see function subprotocol """ const SUBProtocols= Array{String,1}() -"Used in handshake. See SUBProtocols" -hasprotocol(s::AbstractString) = in(s, SUBProtocols) - -"Used to specify handshake response. See SUBProtocols" -function addsubproto(name) - push!(SUBProtocols, string(name)) - return true -end """ write_fragment(io, islast, opcode, hasmask, data::Array{UInt8}) Write the raw frame to a bufffer @@ -203,9 +190,9 @@ send_pong(ws, data...) = write_pong(ws.socket, !ws.server, data...) """ close(ws::WebSocket) -Send an OPCODE_CLOSE frame, wait for the same response while dropping -further received data. If not, consider as closed after waiting a -reasonable amount of time, TIMEOUT_CLOSEHANDSHAKE. +Send an OPCODE_CLOSE frame, and wait for the same response or until +a reasonable amount of time, $(round(TIMEOUT_CLOSEHANDSHAKE, 1)) s, has passed. +Data received while closing is dropped. """ function Base.close(ws::WebSocket) if isopen(ws) @@ -216,7 +203,7 @@ function Base.close(ws::WebSocket) # trailing bytes received. # # We have no guarantee that the peer is actually reading our OPCODE_CLOSE - # frame. If not, the peer's state will not change, and we will not receive + # frame. If not, the peer's state will not change, and we will not receive # an aknowledgment of closing. We use a nonblocking read and give up # after TIMEOUT_CLOSEHANDSHAKE # @@ -231,7 +218,7 @@ function Base.close(ws::WebSocket) # The exception handling is necessary as read_frame will error when the # OPCODE_CLOSE control frame is received by a potentially blocking read in # another Task - # + # try t1 = time() + TIMEOUT_CLOSEHANDSHAKE while isopen(ws) && time() < t1 @@ -244,12 +231,13 @@ function Base.close(ws::WebSocket) close(ws.socket) end catch err - # Typical 'errors' received while closing down are neglected. + # Typical 'errors' received while closing down are neglected. errtyp = typeof(err) errtyp != InterruptException && errtyp != Base.UVError && errtyp != Base.BoundsError && errtyp != Base.EOFError && + errtyp != Base.ArgumentError && rethrow(err) end else @@ -258,7 +246,7 @@ function Base.close(ws::WebSocket) end """ - isopen(WebSocket)-> Bool + isopen(::WebSocket)-> Bool A WebSocket is closed if the underlying TCP socket closes, or if we send or receive a close message. """ @@ -307,19 +295,23 @@ end is_control_frame(msg::WebSocketFragment) = (msg.opcode & 0b0000_1000) > 0 """ Respond to pings, ignore pongs, respond to close.""" -function handle_control_frame(ws::WebSocket,wsf::WebSocketFragment) +function handle_control_frame(ws::WebSocket, wsf::WebSocketFragment) if wsf.opcode == OPCODE_CLOSE - info("$(ws.server ? "Server" : "Client") received OPCODE_CLOSE") + #info("ws|$(ws.server ? "server" : "client") received OPCODE_CLOSE") ws.state = CLOSED - locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, UInt8[]) + try + locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, UInt8[]) + end + # If we did not throw an error here, ArgumentError("Stream is closed or unusable") would be thrown later + throw("ws|$(ws.server ? "server" : "client") received OPCODE_CLOSE, replied with same.") elseif wsf.opcode == OPCODE_PING - info("$(ws.server ? "Server" : "Client") received OPCODE_PING") - send_pong(ws,wsf.data) + info("ws|$(ws.server ? "server" : "client") received OPCODE_PING") + send_pong(ws, wsf.data) elseif wsf.opcode == OPCODE_PONG - info("$(ws.server ? "Server" : "Client") received OPCODE_PONG") + info("ws|$(ws.server ? "server" : "client") received OPCODE_PONG") # Nothing to do here; no reply is needed for a pong message. else # %xB-F are reserved for further control frames - error("Unknown opcode $(wsf.opcode)") + error(" while handle_control_frame(ws|$(ws.server ? "server" : "client"), wsf): Unknown opcode $(wsf.opcode)") end end @@ -363,38 +355,29 @@ end """ read(ws::WebSocket) Typical use: - String(read(ws)) + msg = String(read(ws)) Read one non-control message from a WebSocket. Any control messages that are read will be handled by the handle_control_frame function. Only the data (contents/body/payload) of the message will be returned as a Vector{UInt8}. -This function will not return until a full non-control message has been read. If the other side -doesn't ever complete its message, this function will never return. -If the socket is closed without following protocol, or this function receives -an InterruptException, it : - - writes a warning to STDERR. This can be redirected using logging(..) - - returns an empty array and closes the socket from this side. +This function will not return until a full non-control message has been read. """ function Base.read(ws::WebSocket) - # TODO add a try..catch...finally with - # error(WebSocketClosedError()) - # which also ensures Array{UInt8,1}()) is returned instead of Void() . - # EOF will fail conversion to String, but so will String(Void()) - # if !isopen(ws) - error("Attempt to read from closed WebSocket") + error("Attempt to read from closed WebSocket|$(ws.server ? "server" : "client"). First isopen(ws), or use readguarded(ws)!") end - # Previous versions would retrun any of the errors below or return Void(), - # which would result in the errors below during conversion to String() - # This try..catch adds roughly 20 ns to each read(ws) try frame = read_frame(ws) # Handle control (non-data) messages. if is_control_frame(frame) # Don't return control frames; they're not interesting to users. - handle_control_frame(ws,frame) - return frame.data + handle_control_frame(ws, frame) + # Recurse to return the next data frame. +- return read(ws) + # The following line from commit 7d5fb4480e17320e0d62cfd60d650381e4fb4960 is a typo? + # It would return control frame contents to the user. + # return frame.data end # Handle data message that uses multiple fragments. @@ -403,29 +386,28 @@ function Base.read(ws::WebSocket) end return frame.data catch err - if typeof(err) == InterruptException - # This exception originates from this side. Follow protocol so no errors are triggered on the other side. - close(ws) - warn(STDERR, " While reading websocket, received InterruptException") - elseif typeof(err) == Base.UVError - warn(STDERR, " While reading websocket, received UVError, prefix:", err.prefix, ", code:", err.code) - elseif typeof(err) == Base.BoundsError - warn(STDERR, " While reading websocket, received incomplete frame") - elseif typeof(err) == Base.EOFError - warn(STDERR, " While reading, received EOFError") - else - # Unknown cause, give up continued execution. - # If this actually happens in a multiple fragment message, the accumulated - # stacktrace could be very long since read(ws) is iterative. - ws.state = CLOSED + try + errtyp = typeof(err) + if errtyp <: InterruptException + # This exception originates from this side. Follow close protocol so as not to irritate the other side. + close(ws) + throw(WebSocketClosedError(" while read(ws|$(ws.server ? "server" : "client") received local interrupt exception. Performed closing handshake.")) + elseif errtyp <: Base.UVError || + errtyp <: Base.BoundsError || + errtyp <: Base.EOFError || + errtyp <: Base.ArgumentError + throw(WebSocketClosedError(" while read(ws|$(ws.server ? "server" : "client")) $(string(err))")) + else + # Unknown cause, give up continued execution. + # If this happens in a multiple fragment message, the accumulated + # stacktrace could be very long since read(ws) is iterative. + rethrow(err) + end + finally if isopen(ws.socket) close(ws.socket) end - rethrow(err) - end - ws.state = CLOSED - if isopen(ws.socket) - close(ws.socket) + ws.state = CLOSED end end return Vector{UInt8}() @@ -455,10 +437,10 @@ function readframe_nonblocking(ws) yield() # Define a task for throwing interrupt exception to the (possibly blocked) read task. # We don't start this task because it would never return - killta = @task try;Base.throwto(rt, InterruptException());catch;end + killta = @task try;Base.throwto(rt, InterruptException());end # We start the killing task. When it is scheduled the second time, # we pass an InterruptException through the scheduler. - try;schedule(killta, InterruptException(), error = false);catch;end + try;schedule(killta, InterruptException(), error = false);end # We now have content on chnl, and no additional tasks. take!(chnl) end @@ -484,6 +466,157 @@ function mask!(data, mask=rand(UInt8, 4)) return mask end +"Used in handshake. See SUBProtocols" +hasprotocol(s::AbstractString) = in(s, SUBProtocols) + +"Used to specify acceptable subprotocols. See SUBProtocols" +function addsubproto(name) + push!(SUBProtocols, string(name)) + return true +end + + + + + + +""" +`target(request) => String` + +Convenience function for reading upgrade request target. + E.g. +```julia + function gatekeeper(req, ws) + if target(req) == "/gamepad" + @spawnat 2 gamepad(ws) + elseif target(req) == "/console" + @spawnat 3 console(ws) + ... + end + end +``` +Then, in browser javascript (or equivalent with Julia WebSockets.open( , )) +```javascript +function load(){ + var wsuri = document.URL.replace("http:", "ws:"); + ws1 = new WebSocket(wsuri + "/gamepad"); + ws2 = new WebSocket(wsuri + "/console"); + ws3 = new WebSocket(wsuri + "/graphics"); + ws4 = new WebSocket(wsuri + "/audiochat"); + ws1.onmessage = function(e){vibrate(e.data)} + } // load + +``` +""" +function target # Methods added in include files +end + +""" +`subprotocol(request) => String` + +Convenience function for reading upgrade request subprotocol. +Acceptable subprotocols need to be predefined using +addsubproto(myprotocol). No other subprotocols will pass the handshake. +E.g. +```julia +WebSockets.addsubproto("instructions") +WebSockets.addsubproto("relay_backend") +function gatekeeper(req, ws) + subpr = WebSockets.subprotocol(req) + if subpr == "instructions" + instructions(ws) + elseif subpr == "relay_backend" + relay_backend(ws) + end +end +``` + +Then, in browser javascript (or equivalent with Julia WebSockets.open( , )) +```javascript +function load(){ + var wsuri = document.URL.replace("http:", "ws:"); + ws1 = new WebSocket(wsuri, "instructions"); + ws2 = new WebSocket(wsuri, "relay_backend"); + ws1.onmessage = function(e){doinstructions(e.data)}; + ... + } // load +``` +""" +function subprotocol # Methods added in include files +end + + +""" +`origin(request) => String` +Convenience function for checking which server / port address +the client claims its code was downloaded from. +The resource path can be found with target(req). +E.g. +```julia +function gatekeeper(req, ws) + orig = WebSockets.origin(req) + if startswith(orig, "http://localhost") || startswith(orig, "http://127.0.0.1") + handlewebsocket(ws) + end +end +``` +""" +function origin # Methods added in include files +end + + +""" +`writeguarded(websocket, message) => Bool` + +Return true if write is successful, false if not. +The peer can potentially disconnect at any time, but no matter the +cause you will usually just want to exit your websocket handling function +when you can't write to it. + +""" +function writeguarded(ws, msg) + try + write(ws, msg) + catch + return false + end + true +end + +""" +`readguarded(websocket) => (Vector, Bool)` + +Return (data::Vector, true) + or + (Vector{UInt8}(), false) + +The peer can potentially disconnect at any time, but no matter the +cause you will usually just want to exit your websocket handling function +when you can't write to it. + +E.g. +```julia +while true + data, success = readguarded(websocket) + !success && break + println(String(data)) +end +``` +""" +function readguarded(ws) + data = Vector{UInt8}() + success = true + try + data = read(ws) + catch err + data = Vector{UInt8}() + success = false + finally + return data, success + end +end + + @require HTTP include("HTTP.jl") @require HttpServer include("HttpServer.jl") end # module WebSockets diff --git a/test/functions_server.jl b/test/functions_server.jl index e27ddd3..e8db467 100644 --- a/test/functions_server.jl +++ b/test/functions_server.jl @@ -2,12 +2,10 @@ include("functions_log_test.jl") include("handler_functions_events.jl") include("handler_functions_websockets_general_test.jl") include("handler_functions_websockets_subprotocol_test.jl") + """ -Although a websocket could be opened from a file on any server or just from the browser -working on the file system, we open a test http server. -The term 'handler' is relative to point of view. We make a hierachy of functions and then give -the types defined in HttpServer references to the functions. More commonly, anonymous functions -are used. +This function returns responses to http requests. +For serving the HTML javascript pages which then open WebSocket clients. """ function httphandle(request::Request, response::Response) global n_responders @@ -33,22 +31,20 @@ function httphandle(request::Request, response::Response) end """ -Inner function for WebsocketHandler. Called on opening a new websocket after -the handshake procedure is finished. -The request contains info which can be used for additional delegation or gatekeeping. -Function never exits until the websocket is closed, but calls are made asyncronously. +When a connection performs the handshake successfully, this function +is called as a separate task. Based on inspecting the request that was +accepted as an upgrade, it in turn calls one of three functions. +The function call does not return until it is time to close the websocket. """ -function websockethandle(wsrequest::Request, websocket::WebSocket) - id = "server_functions.websockethandle\t" +function gatekeeper(wsrequest::Request, websocket::WebSocket) + id = "server_functions.gatekeeper\t" clog(id, :cyan, wsrequest, "\t", :yellow, websocket, "\n") - if haskey(wsrequest.headers,"Sec-WebSocket-Protocol") - if wsrequest.headers["Sec-WebSocket-Protocol"] == "websocket-testprotocol" - ws_test_protocol(websocket) - elseif wsrequest.headers["Sec-WebSocket-Protocol"] == "websocket-test-binary" - ws_test_binary(websocket) - else - clog(id, :red, "Unknown sub protocol let through, not responding further. \n") - end + if subprotocol(wsrequest) == "websocket-testprotocol" + ws_test_protocol(websocket) + elseif subprotocol(wsrequest) == "websocket-test-binary" + ws_test_binary(websocket) + elseif subprotocol(wsrequest) != "" + clog(id, :red, "Unknown sub protocol let through, not responding further. \n") else ws_general(websocket) end @@ -59,7 +55,7 @@ end function start_ws_server_async() id = "server_functions.start_ws_server\t" - # Specify this subprotocol is to be let through to websockethandle: + # Specify this subprotocol is to be let through to gatekeeper: WebSockets.addsubproto("websocket-testprotocol") WebSockets.addsubproto("websocket-test-binary") # Tell HttpHandler which functions to spawn when something happens. @@ -70,8 +66,8 @@ function start_ws_server_async() httpha.events["close"] = ev_close httpha.events["write"] = ev_write httpha.events["reset"] = ev_reset - # Pack the websockethandle function in an interface container - wsh = WebSocketHandler(websockethandle) + # Pack the gatekeeper function in a recognizable function wrapper + wsh = WebSocketHandler(gatekeeper) server = Server(httpha, wsh ) clog(id, "Server to be started:\n", server ) servertask = @async run( server, 8080) diff --git a/test/runtests.jl b/test/runtests.jl index c21327f..497484a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,27 +5,28 @@ using Base.Test @testset "HTTP" begin -port_HTTP = 8000 -port_HttpServer = 8081 +const port_HTTP = 8000 +const port_HttpServer = 8081 info("Start HTTP server on port $(port_HTTP)") -@async HTTP.listen("127.0.0.1",UInt16(port_HTTP)) do http + +function echows(ws) + while true + data, success = readguarded(ws) + !success && break + !writeguarded(ws, data) && break + end +end + +@async HTTP.listen("127.0.0.1", UInt16(port_HTTP)) do http if WebSockets.is_upgrade(http.message) - WebSockets.upgrade(http) do ws - while !eof(ws) - data = String(read(ws)) - write(ws,data) - end - end + WebSockets.upgrade(echows, http) end end info("Start HttpServer on port $(port_HttpServer)") -wsh = WebSocketHandler() do req,ws - while !eof(ws) - msg = String(read(ws)) - write(ws, msg) - end +wsh = WebSocketHandler() do req, ws + echows(ws) end server = Server(wsh) @async run(server,port_HttpServer) @@ -39,17 +40,19 @@ servers = [ ("HttpServer", "ws://127.0.0.1:$(port_HttpServer)")] for (s, url) in servers - info("Testing local $(s) server at $(url)...") + info("Testing ws client connecting to $(s) server at $(url)...") WebSockets.open(url) do ws + print(" -Foo-") write(ws, "Foo") @test String(read(ws)) == "Foo" - + print(" -Ping-") + send_ping(ws) + println(" -Bar-") write(ws, "Bar") @test String(read(ws)) == "Bar" - - send_ping(ws) - read(ws) + sleep(1) end + sleep(1) end end # testset \ No newline at end of file From bb9af69874db4fcd8dcdb6d9402ad71ed0b4715e Mon Sep 17 00:00:00 2001 From: hustf Date: Sat, 26 May 2018 12:29:40 +0200 Subject: [PATCH 25/35] modified: src/WebSockets.jl reinstate check for present mask with error message --- src/WebSockets.jl | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/WebSockets.jl b/src/WebSockets.jl index 4a6b0aa..4bc31c9 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -122,7 +122,7 @@ const SUBProtocols= Array{String,1}() """ write_fragment(io, islast, opcode, hasmask, data::Array{UInt8}) -Write the raw frame to a bufffer +Write the raw frame to a bufffer. Websockets not servers must set 'hasmask'. """ function write_fragment(io::IO, islast::Bool, opcode, hasmask::Bool, data::Vector{UInt8}) l = length(data) @@ -144,7 +144,9 @@ function write_fragment(io::IO, islast::Bool, opcode, hasmask::Bool, data::Vecto end if hasmask if opcode == OPCODE_TEXT - data = copy(data) # Avoid masking Strings bytes in place + # Avoid masking Strings bytes in place. + # This makes client websockets slower than server websockets. + data = copy(data) end write(io,mask!(data)) end @@ -337,6 +339,12 @@ function read_frame(ws::WebSocket) mask = b & 0b1000_0000 >>> 7 hasmask = mask != 0 + if mask != ws.server + error("WebSocket reader cannot handle incoming messages without mask. " * + "See http://tools.ietf.org/html/rfc6455#section-5.3") + end + + payload_len::UInt64 = b & 0b0111_1111 if payload_len == 126 payload_len = ntoh(read(ws.socket,UInt16)) # 2 bytes @@ -373,11 +381,8 @@ function Base.read(ws::WebSocket) if is_control_frame(frame) # Don't return control frames; they're not interesting to users. handle_control_frame(ws, frame) - # Recurse to return the next data frame. + # Recurse to return the next data frame. - return read(ws) - # The following line from commit 7d5fb4480e17320e0d62cfd60d650381e4fb4960 is a typo? - # It would return control frame contents to the user. - # return frame.data end # Handle data message that uses multiple fragments. From 15adbdf6ae7c216891ea35c97b3d6f591fdc3806 Mon Sep 17 00:00:00 2001 From: hustf Date: Sat, 26 May 2018 18:51:25 +0200 Subject: [PATCH 26/35] Status codes implemented for closing handshake modified: benchmark/ws_hts.jl isclosed -> !isopen modified: src/HTTP.jl remove dead code 'browserclient'::bool modified: src/HttpServer.jl reshuffle order of handshake checks, now similar to HTTP.jl modified: src/WebSockets.jl close a socket with handshake and 1002: ProtocolError when incorrect mask received. include status code for sending close include received status codes for receiving close update inline doc introduction remove implemented TODO comments --- benchmark/ws_hts.jl | 4 +- src/HTTP.jl | 1 - src/HttpServer.jl | 26 ++++++------- src/WebSockets.jl | 91 ++++++++++++++++++++++++++++++++------------- 4 files changed, 81 insertions(+), 41 deletions(-) diff --git a/benchmark/ws_hts.jl b/benchmark/ws_hts.jl index c949303..14c4ec3 100644 --- a/benchmark/ws_hts.jl +++ b/benchmark/ws_hts.jl @@ -50,7 +50,7 @@ function acceptholdws(http) WebSockets.upgrade(http) do ws if length(WEBSOCKET) > 0 # unexpected behaviour. - if isclosed(WEBSOCKET[1]) + if !isopen(WEBSOCKET[1]) pop!(WEBSOCKET) else msg = " A websocket is already open. Not accepting the attempt at opening more." @@ -161,6 +161,6 @@ SRCPATH ∉ LOAD_PATH && push!(LOAD_PATH, SRCPATH) LOGGINGPATH ∉ LOAD_PATH && push!(LOAD_PATH, LOGGINGPATH) import ws_hts.listen_hts tas = @schedule ws_hts.listen_hts() -sleep(5) +sleep(7) hts = ws_hts.getws_hts() """ \ No newline at end of file diff --git a/src/HTTP.jl b/src/HTTP.jl index ddd2434..43cc48c 100644 --- a/src/HTTP.jl +++ b/src/HTTP.jl @@ -111,7 +111,6 @@ end function upgrade(f::Function, http::HTTP.Stream) # Double check the request. is_upgrade should already have been called by user. check_upgrade(http) - browserclient = HTTP.hasheader(http, "Origin") if !HTTP.hasheader(http, "Sec-WebSocket-Version", "13") HTTP.setheader(http, "Sec-WebSocket-Version" => "13") HTTP.setstatus(http, 400) diff --git a/src/HttpServer.jl b/src/HttpServer.jl index 8000b5d..cd23ebb 100644 --- a/src/HttpServer.jl +++ b/src/HttpServer.jl @@ -47,17 +47,26 @@ Typical headers: "Accept-Language" => "en-US,en;q=0.5" """ function websocket_handshake(request, client) - if !haskey(request.headers, "Sec-WebSocket-Key") - Base.write(client.sock, HttpServer.Response(400)) - return false - end + if get(request.headers, "Sec-WebSocket-Version", "13") != "13" response = HttpServer.Response(400) response.headers["Sec-WebSocket-Version"] = "13" Base.write(client.sock, response) return false end + if haskey(request.headers, "Sec-WebSocket-Protocol") + if hasprotocol(request.headers["Sec-WebSocket-Protocol"]) + response.headers["Sec-WebSocket-Protocol"] = request.headers["Sec-WebSocket-Protocol"] + else + Base.write(client.sock, HttpServer.Response(400)) + return false + end + end + if !haskey(request.headers, "Sec-WebSocket-Key") + Base.write(client.sock, HttpServer.Response(400)) + return false + end key = request.headers["Sec-WebSocket-Key"] if length(base64decode(key)) != 16 # Key must be 16 bytes Base.write(client.sock, HttpServer.Response(400)) @@ -69,15 +78,6 @@ function websocket_handshake(request, client) response.headers["Upgrade"] = "websocket" response.headers["Connection"] = "Upgrade" response.headers["Sec-WebSocket-Accept"] = resp_key - # TODO move this part further up, similar to in HTTP.jl - if haskey(request.headers, "Sec-WebSocket-Protocol") - if hasprotocol(request.headers["Sec-WebSocket-Protocol"]) - response.headers["Sec-WebSocket-Protocol"] = request.headers["Sec-WebSocket-Protocol"] - else - Base.write(client.sock, HttpServer.Response(400)) - return false - end - end Base.write(client.sock, response) return true diff --git a/src/WebSockets.jl b/src/WebSockets.jl index 4bc31c9..4c22327 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -1,25 +1,23 @@ __precompile__() """ WebSockets -This module implements the server side of the WebSockets protocol. Some -things would need to be added to implement a WebSockets client, such as -masking of sent frames. +This module implements the WebSockets protocol. -WebSockets expects to be used with HttpServer to provide the HttpServer -for accepting the HTTP request that begins the opening handshake. WebSockets -implements a subtype of the WebSocketInterface from HttpServer; this means -that you can create a WebSocketsHandler and pass it into the constructor for -an http server. +WebSockets relies on either packages HttpServer, HTTP or both. + +Websocket|server relies on a client initiating the connection. +Websocket|client initiate the connection, but requires HTTP. + +The other side of the connection, the peer, is typically a browser with +scripts enabled. Browsers are always the initiating, client side. But the +peer can be any program, in any language, that follows the protocol. That +includes another Julia session, parallel process or task. Future improvements: 1. Logging of refused requests and closures due to bad behavior of client. -2. Better error handling (should we always be using "error"?) -3. Unit tests with an actual client -- to automatically test the examples. -4. Send close messages with status codes. -5. Allow users to receive control messages if they want to. +2. Allow users to receive control messages if they want to. """ module WebSockets - import MbedTLS: digest, MD_SHA1 using Requires export WebSocket, @@ -53,6 +51,17 @@ struct WebSocketError <: Exception message::String end +"Status codes according to RFC 6455 7.4.1" +const codeDesc = Dict{Int, String}(1000=>"Normal", 1001=>"Going Away", + 1002=>"Protocol Error", 1003=>"Unsupported Data", + 1004=>"Reserved", 1005=>"No Status Recvd- reserved", + 1006=>"Abnormal Closure- reserved", 1007=>"Invalid frame payload data", + 1008=>"Policy Violation", 1009=>"Message too big", + 1010=>"Missing Extension", 1011=>"Internal Error", + 1012=>"Service Restart", 1013=>"Try Again Later", + 1014=>"Bad Gateway", 1015=>"TLS Handshake") + + """ A WebSocket is a wrapper over a TcpSocket. It takes care of wrapping outgoing data in a frame and unwrapping (and concatenating) incoming data. @@ -122,7 +131,7 @@ const SUBProtocols= Array{String,1}() """ write_fragment(io, islast, opcode, hasmask, data::Array{UInt8}) -Write the raw frame to a bufffer. Websockets not servers must set 'hasmask'. +Write the raw frame to a bufffer. Websocket|client must set 'hasmask'. """ function write_fragment(io::IO, islast::Bool, opcode, hasmask::Bool, data::Vector{UInt8}) l = length(data) @@ -148,7 +157,8 @@ function write_fragment(io::IO, islast::Bool, opcode, hasmask::Bool, data::Vecto # This makes client websockets slower than server websockets. data = copy(data) end - write(io,mask!(data)) + # Write the random masking key to io, also mask the data in-place + write(io, maskswitch!(data)) end write(io, data) end @@ -192,14 +202,21 @@ send_pong(ws, data...) = write_pong(ws.socket, !ws.server, data...) """ close(ws::WebSocket) + close(ws::WebSocket, statusnumber) Send an OPCODE_CLOSE frame, and wait for the same response or until a reasonable amount of time, $(round(TIMEOUT_CLOSEHANDSHAKE, 1)) s, has passed. Data received while closing is dropped. +Status codes according to RFC 6455 7.4.1 can be included, see WebSockets.codeDesc """ -function Base.close(ws::WebSocket) +function Base.close(ws::WebSocket; statusnumber = 0) if isopen(ws) ws.state = CLOSING - locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, UInt8[]) + if statusnumber == 0 + locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server) + else + statuscode = reverse(reinterpret(UInt8, [UInt16(statusnumber)])) + locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, statuscode) + end # Wait till the peer responds with an OPCODE_CLOSE while discarding any # trailing bytes received. @@ -304,8 +321,20 @@ function handle_control_frame(ws::WebSocket, wsf::WebSocketFragment) try locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, UInt8[]) end - # If we did not throw an error here, ArgumentError("Stream is closed or unusable") would be thrown later - throw("ws|$(ws.server ? "server" : "client") received OPCODE_CLOSE, replied with same.") + # Find out why the other side wanted to close. + # RFC 6455 5.5.1. If there is a status code, it's a two-byte number in network order. + if wsf.payload_len == 0 + reason = " No reason " + elseif wsf.payload_len == 2 + scode = Int(reinterpret(UInt16, reverse(wsf.data))[1]) + reason = string(scode) * ":" * get(codeDesc, scode, "") + else + scode = Int(reinterpret(UInt16, reverse(wsf.data[1:2]))[1]) + reason = string(scode) * ":" * String(wsf.data[3:end]) + end + #info(reason) + throw("ws|$(ws.server ? "server" : "client") received OPCODE_CLOSE, replied with same. " * + "Received reason: " * reason) elseif wsf.opcode == OPCODE_PING info("ws|$(ws.server ? "server" : "client") received OPCODE_PING") send_pong(ws, wsf.data) @@ -340,11 +369,14 @@ function read_frame(ws::WebSocket) hasmask = mask != 0 if mask != ws.server - error("WebSocket reader cannot handle incoming messages without mask. " * - "See http://tools.ietf.org/html/rfc6455#section-5.3") + if ws.server + msg = "WebSocket|server cannot handle incoming messages without mask. Ref. rcf6455 5.3" + else + msg = "WebSocket|client cannot handle incoming messages with mask. Ref. rcf6455 5.3" + end + throw(WebSocketError(1002, msg)) end - payload_len::UInt64 = b & 0b0111_1111 if payload_len == 126 payload_len = ntoh(read(ws.socket,UInt16)) # 2 bytes @@ -355,7 +387,7 @@ function read_frame(ws::WebSocket) maskkey = hasmask ? read(ws.socket,4) : UInt8[] data = read(ws.socket,Int(payload_len)) - hasmask && mask!(data,maskkey) + hasmask && maskswitch!(data,maskkey) return WebSocketFragment(fin,rsv1,rsv2,rsv3,opcode,mask,payload_len,maskkey,data) end @@ -393,10 +425,14 @@ function Base.read(ws::WebSocket) catch err try errtyp = typeof(err) - if errtyp <: InterruptException + if errtyp <: InterruptException # This exception originates from this side. Follow close protocol so as not to irritate the other side. close(ws) throw(WebSocketClosedError(" while read(ws|$(ws.server ? "server" : "client") received local interrupt exception. Performed closing handshake.")) + elseif errtyp <: WebSocketError + # This exception originates on the other side. Follow close protocol with reason. + close(ws, statusnumber = err.status) + throw(WebSocketClosedError(" while read(ws|$(ws.server ? "server" : "client") $(err.message) - Performed closing handshake.")) elseif errtyp <: Base.UVError || errtyp <: Base.BoundsError || errtyp <: Base.EOFError || @@ -464,7 +500,12 @@ function generate_websocket_key(key) return base64encode(digest(MD_SHA1, hashkey)) end -function mask!(data, mask=rand(UInt8, 4)) +""" +Masks or unmasks data, returns the key for reverting mask. +Calling twice with the same key restores data. +Ref. RFC 6455 5-3. +""" +function maskswitch!(data, mask = rand(UInt8, 4)) for i in 1:length(data) data[i] = data[i] ⊻ mask[((i-1) % 4)+1] end From 22525afa150bfc4a25f7c66b13a00c826cdd7672 Mon Sep 17 00:00:00 2001 From: hustf Date: Sun, 27 May 2018 19:20:01 +0200 Subject: [PATCH 27/35] modified: benchmark/logs/benchmark_results_readable.log document an overall reduction in bandwidths, 15% since last benchmark result commit. More than expected. modified: src/HttpServer.jl bugfix remaining since last commit modified: src/WebSockets.jl improve inline doc maskswitch! close: bugfix statuscode 0, use hton instead of reverse modified: test/runtests.jl reintroduce unit tests from master, now including client add close status code test add varying message length --- benchmark/logs/benchmark_results_readable.log | 1728 ++++++++--------- src/HttpServer.jl | 4 +- src/WebSockets.jl | 11 +- test/runtests.jl | 259 ++- 4 files changed, 1121 insertions(+), 881 deletions(-) diff --git a/benchmark/logs/benchmark_results_readable.log b/benchmark/logs/benchmark_results_readable.log index b3efea9..513417f 100644 --- a/benchmark/logs/benchmark_results_readable.log +++ b/benchmark/logs/benchmark_results_readable.log @@ -1,970 +1,968 @@ -# Note this file is made by copy paste from the console. -# Benchmark_results.log does not have the encoding of ascii graphics right. - HTS_BCE iexplore Varying message size: - bestserverbandwidth = 2.5996 [ns/b] = [s/GB] @ size = 220320 b - bestclientbandwidth = 2.7653 [ns/b] = [s/GB] @ size = 220320 b - bestserverlatency = 59000 [ns] @ size = 1020 b - bestclientlatency = 12614 [ns] @ size = 1020 b + bestserverbandwidth = 2.843 [ns/b] = [s/GB] @ size = 1357620 b + bestclientbandwidth = 3.0215 [ns/b] = [s/GB] @ size = 220320 b + bestserverlatency = 142514 [ns] @ size = 1020 b + bestclientlatency = 19971 [ns] @ size = 1020 b HTS_BCE chrome Varying message size: - bestserverbandwidth = 4.0413 [ns/b] = [s/GB] @ size = 65280 b - bestclientbandwidth = 3.4863 [ns/b] = [s/GB] @ size = 27540 b - bestserverlatency = 66759 [ns] @ size = 1020 b - bestclientlatency = 19428 [ns] @ size = 1020 b + bestserverbandwidth = 5.2447 [ns/b] = [s/GB] @ size = 220320 b + bestclientbandwidth = 4.4889 [ns/b] = [s/GB] @ size = 27540 b + bestserverlatency = 192733 [ns] @ size = 1020 b + bestclientlatency = 32490 [ns] @ size = 1020 b HTS_BCE phantomjs Varying message size: - bestserverbandwidth = 7.0451 [ns/b] = [s/GB] @ size = 127500 b - bestclientbandwidth = 1.5451 [ns/b] = [s/GB] @ size = 27540 b - bestserverlatency = 118169 [ns] @ size = 1020 b - bestclientlatency = 2208 [ns] @ size = 1020 b + bestserverbandwidth = 7.994 [ns/b] = [s/GB] @ size = 220320 b + bestclientbandwidth = 1.6196 [ns/b] = [s/GB] @ size = 27540 b + bestserverlatency = 266542 [ns] @ size = 8160 b + bestclientlatency = 2883 [ns] @ size = 1020 b HTS_BCE firefox Varying message size: - bestserverbandwidth = 4.5881 [ns/b] = [s/GB] @ size = 127500 b - bestclientbandwidth = 4.9592 [ns/b] = [s/GB] @ size = 127500 b - bestserverlatency = 193333 [ns] @ size = 1020 b - bestclientlatency = 21735 [ns] @ size = 1020 b + bestserverbandwidth = 6.1446 [ns/b] = [s/GB] @ size = 127500 b + bestclientbandwidth = 4.8754 [ns/b] = [s/GB] @ size = 127500 b + bestserverlatency = 435462 [ns] @ size = 1020 b + bestclientlatency = 35413 [ns] @ size = 1020 b HTS_JCE Varying message size: - bestserverbandwidth = 1.3878 [ns/b] = [s/GB] @ size = 220320 b - bestclientbandwidth = 3.8825 [ns/b] = [s/GB] @ size = 220320 b - bestserverlatency = 61522 [ns] @ size = 8160 b - bestclientlatency = 75159 [ns] @ size = 1020 b + bestserverbandwidth = 1.6133 [ns/b] = [s/GB] @ size = 220320 b + bestclientbandwidth = 4.5496 [ns/b] = [s/GB] @ size = 127500 b + bestserverlatency = 61228 [ns] @ size = 1020 b + bestclientlatency = 56592 [ns] @ size = 1020 b Benchmark Plots of all samples :init_plots [ns/b], message size 130560 b 200 samples - HTS_BCE iexplore serverspeeds - ┌────────────────────────────────────────────────────────────────────────────────┐ - 70 │ . │ - │ | │ - │ | │ - │ | │ - │ | │ - │ [ │ - │ [ │ - │ .\ │ - │ || │ - │ || │ - │ || │ - │ || │ - │ || │ - │ . . ..... ..|l . .. .. ..._ ..,_. .. .. ....._.__. .. .│ - 0 │\-/'\-`"\/"/"\/`""`'""""/`' """"``'"""""`'"\-/`"` """`' `""""`'"""`'"` ` '"""""│ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE iexplore clientspeeds - ┌────────────────────────────────────────────────────────────────────────────────┐ - 7 │ │ - │ , │ - │ l │ - │ ] │ - │ ] | │ - │ ] [ │ - │ ] . | | l | [ │ - │ O,., [ . k..l/a.J . | .,\. , M ,. │ - │ N//\ A[UU,, |N\]||\N O_..\/_N. .ulP/v./,[k. r\. N,,.l^, , \.,.u,.│ - │ ,,| `l \ | ."N/\`|n/-`Y|/` ""Y|| '"" \n\_J '' \O `\| Y\N`Y`" "Y`""``/''\'│ - │'"\` |^_/,.|r/ " || "" "` │ - │ `"` | │ - │ | │ - │ ` │ - 1 │ │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE chrome serverspeeds - ┌────────────────────────────────────────────────────────────────────────────────┐ - 90 │ , │ - │ | │ - │ | │ - │ | │ - │ | │ - │ ] │ - │ ] │ - │ || │ - │ || │ - │ || │ - │ || │ - │ || │ - │, || │ - │\_. ..A.._/_...._ .. __/ .rv.,u_/.... .... . ,./___ || .._... ,..._.\. . r.│ - 0 │ '"`"`'`` "` `"""'/ "` " '`"`"/`""""'"` " ""`'"`" '`""T""` "`""""""`"│ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE chrome clientspeeds - ┌────────────────────────────────────────────────────────────────────────────────┐ - 200 │ │ - │ │ - │ │ - │ │ - │ . │ - │ | │ - │ | │ - │ | ,, │ - │ [ || │ - │ .\ || │ - │ || || │ - │ || || │ - │ || || │ - │ || /\ │ - 0 │\-------f\------.r-v.----------/\----.r--v.._r--------\-----.v-\r---------.-n\r-│ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE phantomjs serverspeeds - ┌────────────────────────────────────────────────────────────────────────────────┐ - 50 │ . │ - │ | │ - │ | │ - │ | │ - │ | │ - │ ] │ - │ /, │ - │ || . │ - │ || | │ - │, || /, │ - │\r, , || . |/\ . │ - │ \f|.. , u. . d , . . . |O. /L. . |\`l., M .\ │ - │ " '/"^V\/'\^``/`..|D| .... .k ..N./,. M ......, . .|"\-` \n/""/ "/``"""G │ - │ '`'"`''""`""`"`"`'`T"""`""`\"/"'"`' `\/"│ - 0 │ │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE phantomjs clientspeeds - ┌────────────────────────────────────────────────────────────────────────────────┐ - 60 │ │ - │ │ - │ , │ - │ | │ - │ | │ - │ | │ - │ | │ - │ [ │ - │ ,\ │ - │ || │ - │ || │ - │ || │ - │ || │ - │ || │ - 0 │/"`--------------------------------------------------------------/"^-^--`-f\----│ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE firefox serverspeeds - ┌────────────────────────────────────────────────────────────────────────────────┐ - 40 │ . │ - │ | │ - │ | │ - │ | . │ - │ [ l │ - │ | [ ] │ - │ [ ,\ ] │ - │ [ || ] │ - │, N , , || ] │ - │| N, [ l . |l /,. , │ - │ \|l, .l ,] A \| |, ||.||] .., . [ │ - │ '\ .. /. ...|| .. . /N.N.Ok |\ . .|"W||^, . /`\ ,,\\ ...^.│ - │ ^.|_.."\/lJ \f,\^/T V`|\ ._/,|"G|/ |r-fl./|r/r_/// '` `\../\./ ""/|l |\||J│ - │ ` '` ' """ "` ' "`` "` \` │ - 0 │ │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE firefox clientspeeds - ┌────────────────────────────────────────────────────────────────────────────────┐ - 70 │ , │ - │ . | | │ - │ | | | │ - │ l | [ | │ - │ ] | [ | │ - │ ] [ [ [ │ - │ ] [ [ [ │ - │ ] || [ [ │ - │ ] || [ [ │ - │ ] || k [ │ - │ N || N N │ - │ N || N N │ - │ . N || . N . , N │ - │,\/-v______-n__Nr_^.vu_\.-\..___.rfl_^J\^.ru_./.___r-v"rf\\^_._-`.^-f\v-V__\rr._│ - 0 │ \ ` │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_JCE serverspeeds - ┌────────────────────────────────────────────────────────────────────────────────┐ - 200 │ │ - │ │ - │ │ - │ │ - │ │ - │ . │ - │ | │ - │ | |│ - │ [ l│ - │ [ ]│ - │ [ ]│ - │ M ]│ - │ N O│ - │ N N│ - 0 │._________N____________________________________________________________________N│ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_JCE clientspeeds - ┌────────────────────────────────────────────────────────────────────────────────┐ - 400 │ │ - │, │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │\ │ - │| . . │ - │| l | │ - │| J M │ - 0 │|______________N________________N_______________________________________________│ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 200 + HTS_BCE iexplore serverspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 20 │ │ + │ │ + │ , │ + │ | | │ + │ | .|| │ + │ | ||| │ + │ ] ||| │ + │ O ||| │ + │ N [|| │ + │ .Nl N|\ , │ + │ ||^.N||, . . 1 , .. , |, . │ + │\ ./"`]|/l[.N [ ..."^k| ||.[ |M| |.,| .._.|\ .v J\ │ + │lKv` \M"A^.|K" `\\ |KG^./"/_kV||/. ,... ,\''\/|.//'P`"\k||│ + │ ` "` """ " ` `" "`"`"/"/""`" """ '''│ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE iexplore clientspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ │ + │ | │ + │ | | │ + │ | | │ + │ [ | │ + │ [ | │ + │ [ ] │ + │ [ ] │ + │ [ /. │ + │ [ || │ + │ , [ || │ + │ || N || │ + │._..^/.^/l__,..._.___/_...._.___.__.N... \... .L_runl._.v.|||...│ + 0 │ "`"` ' ''`" " ' "`"`""""``""" `" ` '"'"'│ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE chrome serverspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 30 │ │ + │ │ + │ │ + │ | │ + │ | │ + │ | . │ + │ | | │ + │ k | │ + │ .N [ │ + │ |N M │ + │G, ||N . N │ + │]L ./| \. l / , N. _ .. |. _ │ + │//F"/ \/| .. O .`"../, rK\.\r_|L`\,r"V, .v/.. ._,.\"^\Y\r.\./`W│ + │ `'""`""""" '`""` '` \ '` '\" ""``'` "" ' │ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE chrome clientspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ | | , │ + │ | | | │ + │ | || | │ + │ | | || | │ + │ | |. || l │ + │ | || || ] │ + │ .| || || ] │ + │ || || || ] │ + │ , |. || || N │ + │ l || || || N │ + │. ] || || |. N │ + │|.O.u,LN| || , . . ||. . , N. ,,│ + │'"""'`''"`^^^/--/``"--^`-/""-""""`//f\/^---\-/--^/`fK"/"""/--`/""│ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE phantomjs serverspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 30 │ │ + │ │ + │ . │ + │ | │ + │ | │ + │ l │ + │|, ] │ + │|| ] , │ + │/| N, | . l , │ + │/|\ NA J /. . ] , \ , [d│ + │ /l N| N./\,, ,. .. \../.\ .vJ| , l. /, l. /, ||]│ + │ r`"u|v` H\^/\\.v\^/`^`"K`'^`v"v^'"'`/\//\-``JJ/v\/^"YK`ln\^fl/│ + │ │ + │ │ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE phantomjs clientspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ │ + │ │ + │ | │ + │ | │ + │ | │ + │ | │ + │ | │ + │ | │ + │ || │ + │ || │ + │ || │ + │ || │ + │. ..\, . . . || . .. . │ + 0 │"""`"""""'""""/"""""""\/"""\`//"'""""""""""""'`^--/`"""""`"""""\-│ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE firefox serverspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 50 │ │ + │ │ + │ , | │ + │ , , | | | │ + │.|, | | |. | │ + │|N| | ] , || |, │ + │|N| l ] .] || ||, │ + │|N| O /, d],||,.|| | │ + │|N| N || ]][|||||] [ || │ + │|N\.N || , ]][/|O||/. @, @| │ + │|/||| |[ ].]][||N|||| . @| M\ │ + │|F\/| |^,N|@/\|MN|.|| | |l|| , .,|| │ + │ |`|] |'``'|l`||`\`^\_kf./n./__^^.,u../|1\ . 1. A .\J lKd,│ + │ ' "'''` "'``^-\-`f ---"-`"'' | ''│ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE firefox clientspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 80 │ │ + │ | │ + │ [ │ + │ [ │ + │ [ │ + │ [ | │ + │ [ , | │ + │ [ | ] │ + │ [ [ ] │ + │ [ [ , ] │ + │ [ . [ | ] │ + │ . . . [ l . [ | /, │ + │,. l .| d. M ] l, .\ d |, || ,[| │ + │vM\V.\/N./J.Nu_N.^\_\_._.__Jl..._. ./..Nk. || . _./\|,.│ + 0 │ '' " `" ` ' ' ` " ` ' ' """`"```""""'""""""""'""" ' '"''│ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_JCE serverspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 600 │ │ + │. │ + │\ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| , , │ + 0 │l__O_______________________________________________________/u____│ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_JCE clientspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 800 │. │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + 0 │l_____________________/____________uL____________u____________a__│ + └─────────────────────────────────────────────────────────────────┘ + 0 200 Benchmark Tables of all samples, :init_tables, message size 130560 b 200 samples HTS_BCE iexplore => Table with 200 rows, 2 columns: serverspeeds clientspeeds ────────────────────────── -4.5221 2.87694 -2.86018 2.97191 -2.91326 3.16742 -2.69259 3.00821 -2.60321 2.98866 -3.5026 2.86576 -3.92436 3.19256 -5.22318 2.77639 -3.5417 6.38512 +5.15613 3.65342 +2.77637 2.99144 +2.85458 2.76242 +3.77073 3.49141 +3.15904 3.011 +3.74279 2.93838 +2.48589 2.86296 +2.80711 3.19255 +4.31818 3.96347 ⋮ -4.51652 3.60595 -4.75952 3.38529 -4.23441 3.55008 -4.24837 3.10039 -3.67857 3.51377 -4.67292 3.45232 -5.12263 3.38529 -4.17016 3.40484 +2.49427 2.92161 +3.73722 2.58085 +2.43282 2.98865 +2.53058 2.6032 +3.10876 2.91324 +3.93274 2.57527 +2.51104 2.8881 +2.50265 2.69817 HTS_BCE chrome => Table with 200 rows, 2 columns: serverspeeds clientspeeds ────────────────────────── -15.3204 10.6307 -7.13368 4.42155 -6.51081 6.62254 -6.39072 5.79856 -6.02202 5.11145 -5.6589 5.63935 -5.30977 5.05558 -5.45501 5.55556 -5.63097 7.03313 +8.9548 8.54699 +5.32372 5.04999 +9.70335 4.56677 +7.00239 4.83212 +5.11423 5.63935 +6.103 5.27623 +4.64777 5.7371 +5.75386 14.2366 +5.86559 4.99133 ⋮ -4.95503 4.85448 -5.02767 4.42155 -4.23719 4.17575 -9.28162 5.23715 -8.24537 5.15335 -8.6336 4.98856 -6.38513 4.79862 -5.52763 4.96621 +5.80693 4.60867 +5.55275 4.56399 +6.57504 5.30975 +6.1058 4.62544 +5.98848 6.70352 +5.06116 4.28746 +4.34892 4.23439 +6.10859 6.24265 HTS_BCE phantomjs => Table with 200 rows, 2 columns: serverspeeds clientspeeds ────────────────────────── -18.0577 2.57806 -13.354 3.30149 -13.7953 3.46908 -14.8763 1.97754 -15.4824 3.41602 -11.7647 2.04179 -9.77321 2.62835 -12.2312 2.12279 -10.3626 2.15072 +10.9435 2.60879 +13.0188 2.89369 +17.6163 1.75409 +9.12797 2.10603 +9.91004 2.30712 +10.5469 1.91608 +11.5273 2.71493 +8.68665 3.25959 +8.15316 1.99709 ⋮ -6.0332 1.50551 -5.77342 1.50271 -5.23994 1.49992 -5.77063 1.85744 -6.24547 1.64795 -5.77622 1.5865 -5.96337 1.58371 -6.08068 1.81834 +6.71749 2.5222 +10.2117 2.02223 +13.2311 2.87413 +6.2287 1.60885 +6.61415 1.55857 +6.14769 1.55857 +11.5245 1.63958 +7.47722 1.89095 HTS_BCE firefox => Table with 200 rows, 2 columns: serverspeeds clientspeeds ────────────────────────── -17.1806 5.455 -14.6305 5.27904 -12.5859 8.54982 -10.7732 6.09464 -10.748 6.25664 -26.2248 2.20938 -15.0467 10.2899 -14.2702 7.28172 -9.98549 6.32367 +10.9658 7.75375 +11.8038 5.96335 +34.255 13.826 +10.8066 6.15048 +13.1501 9.20897 +38.4643 6.82642 +20.3927 9.6028 +12.7758 5.6952 +8.42969 6.44935 ⋮ -6.13375 6.03598 -8.64757 6.37954 -8.9185 6.96889 -14.8176 5.66729 -7.46607 4.86566 -5.42708 5.41311 -5.85164 5.26229 -8.92688 5.63377 +9.13076 5.16171 +8.11127 5.36281 +6.43259 5.26785 +9.39052 7.19511 +8.01071 5.59745 +6.63649 5.2064 +5.79576 4.54164 +5.54157 4.46064 HTS_JCE => Table with 200 rows, 2 columns: serverspeeds clientspeeds ────────────────────────── -2.69538 359.706 -2.89091 5.9047 -2.8518 6.44378 -2.36579 4.90196 -2.1172 4.32099 -1.94962 4.46623 -2.03899 4.67852 -2.6479 4.28748 -2.08648 4.02213 +522.696 789.896 +2.57247 4.56398 +2.92721 4.22881 +2.36858 4.18691 +2.32109 9.30672 +2.12837 4.04726 +2.05854 3.89922 +1.75688 4.2316 +1.72616 5.80413 ⋮ -1.4664 3.69812 -1.38261 4.11988 -1.94682 4.8098 -2.01665 4.14223 -1.86861 3.90481 -1.80996 8.29005 -105.274 5.29859 -2.92442 4.42434 +1.56695 30.5988 +1.94122 5.25947 +1.61163 4.03049 +1.75688 3.88246 +1.4273 4.30143 +1.76526 4.05005 +1.80158 3.96346 +1.76806 3.86291 Benchmark Plots of varying size messages :test_plots [ns/b], VSIZE = [1762560, 1357620, 1020000, 743580, 522240, 349860, 220320, 127500, 65280, 27540, 8160, 1020] - HTS_BCE iexplore serverbandwidth - ┌────────────────────────────────────────────────────────────────────────────────┐ - 60 │, │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │\ │ - │|_. │ - 0 │ '"""""`/""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""` │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE iexplore clientbandwidth - ┌────────────────────────────────────────────────────────────────────────────────┐ - 13 │ │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │\_---.__________r-----------------------------------------------------* │ - 2 │ │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE chrome serverbandwidth - ┌────────────────────────────────────────────────────────────────────────────────┐ - 70 │ │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │l │ - │|_ .____________-----------._________________________________________. │ - 0 │ ""` │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE chrome clientbandwidth - ┌────────────────────────────────────────────────────────────────────────────────┐ - 20 │. │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| .____* │ - │| .f-._ _________r----""""` │ - │l ._ ./` "`-----.___----------/""""""""""""" │ - │"/` "` │ - │ │ - 0 │ │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE phantomjs serverbandwidth - ┌────────────────────────────────────────────────────────────────────────────────┐ - 200 │ │ - │ │ - │ │ - │ │ - │ │ - │ │ - │, │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │. │ - 0 │"-----------------/""""""`--------------------------------------------* │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE phantomjs clientbandwidth - ┌────────────────────────────────────────────────────────────────────────────────┐ - 2.5 │ ._-------/"""""""""` │ - │ __-/"` │ - │ \ _.-/" │ - │ |\ _-/"""""""""" │ - │ ||. _-" │ - │| | \ .__. _-" │ - │| | ". .` """""" │ - │|,` ". /` │ - │|| \ ./ │ - │|| `` │ - │|| │ - │|| │ - │|| │ - │|| │ - 1.5 │"` │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE firefox serverbandwidth - ┌────────────────────────────────────────────────────────────────────────────────┐ - 200 │ │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │\ │ - 0 │ \-------/"""""""""""""""\--------------------------------------------* │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE firefox clientbandwidth - ┌────────────────────────────────────────────────────────────────────────────────┐ - 30 │ │ - │ │ - │ │ - │ │ - │, │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| .____________..__________________________________________r-----* │ - │"-^---"` │ - │ │ - 0 │ │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_JCE serverbandwidth - ┌────────────────────────────────────────────────────────────────────────────────┐ - 90 │ │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - 0 │"----______________________________________________________________---* │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_JCE clientbandwidth - ┌────────────────────────────────────────────────────────────────────────────────┐ - 80 │ │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │'u_ │ - 0 │ """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""` │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE + HTS_BCE iexplore serverbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 200 │ │ + │ │ + │ │ + │ │ + │. │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\ │ + 0 │"---.____________________________________________________. │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE iexplore clientbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 20 │, │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\ │ + │|--"\-.__r-----------------------------------------------/ │ + │ │ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome serverbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 200 │. │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + 0 │|--------------------------------------------------------/ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome clientbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ │ + │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │l . _____________________.------------""""""""` │ + │""""""""""""""" │ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs serverbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 300 │ │ + │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\ │ + 0 │"\-------------------------------------------------------/ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs clientbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 3 │ │ + │, .__-/ │ + │| __--/"` │ + │| _/-._. ______r-----/"" │ + │| _-" '"-._.--/""" │ + │| , .r/ │ + │||"\___./` │ + │|/ │ + │|| │ + │l` │ + │| │ + │ │ + │ │ + │ │ + 1 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox serverbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 500 │ │ + │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\ │ + 0 │|-_______________________________________________________. │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox clientbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │. │ + │| │ + │\ .__----------------------------------/""""""""""""--/ │ + │"""""` │ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE serverbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 70 │ │ + │ │ + │, │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\. │ + 0 │ '\------------------------------------------------------/ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE clientbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 60 │ │ + │, │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │l │ + │""`._____________________________________________________. │ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE Benchmark Tables of varying size messages :test_tables [ns/b] HTS_BCE iexplore => Table with 12 rows, 3 columns: VSIZE serverbandwidth clientbandwidth ───────────────────────────────────────── -1762560 2.84794 3.02725 -1357620 2.89765 3.01044 -1020000 2.72513 3.1075 -743580 3.20976 3.04866 -522240 2.73883 3.18919 -349860 3.09709 2.87533 -220320 2.59964 2.76531 -127500 3.07385 3.01227 -65280 3.77368 3.03525 -27540 5.23605 2.88944 -8160 10.9192 3.6188 -1020 57.8435 12.3667 +1762560 2.85556 3.31603 +1357620 2.84298 3.20976 +1020000 2.97696 3.21081 +743580 3.61289 3.16786 +522240 3.23825 3.12889 +349860 3.7709 3.19613 +220320 3.65095 3.02147 +127500 4.54435 3.65872 +65280 5.62479 3.46019 +27540 9.29125 3.46253 +8160 21.3134 5.14942 +1020 139.719 19.5797 HTS_BCE chrome => Table with 12 rows, 3 columns: VSIZE serverbandwidth clientbandwidth ───────────────────────────────────────── -1762560 5.55213 7.15743 -1357620 5.48115 5.58792 -1020000 5.30875 5.11574 -743580 6.20273 4.77186 -522240 6.3483 4.34657 -349860 6.12922 4.77051 -220320 5.10403 6.23671 -127500 4.80703 3.85924 -65280 4.04126 4.13268 -27540 6.02253 3.4863 -8160 15.4351 5.59254 -1020 65.4499 19.047 +1762560 5.77379 7.83374 +1357620 5.55071 6.69094 +1020000 5.6383 5.99683 +743580 6.51132 5.74968 +522240 6.16186 5.44588 +349860 5.64196 5.07182 +220320 5.24472 5.19075 +127500 8.33349 5.36475 +65280 7.65596 4.94229 +27540 8.5631 4.48888 +8160 31.1207 8.589 +1020 188.954 31.8534 HTS_BCE phantomjs => Table with 12 rows, 3 columns: VSIZE serverbandwidth clientbandwidth ───────────────────────────────────────── -1762560 7.89614 2.49179 -1357620 8.07148 2.46927 -1020000 8.08981 2.29689 -743580 8.54885 2.27637 -522240 9.19892 2.07951 -349860 8.15506 2.11337 -220320 8.02959 1.85921 -127500 7.04513 2.05976 -65280 7.61785 2.34252 -27540 8.88562 1.54509 -8160 18.3139 1.57713 -1020 115.852 2.16479 +1762560 9.21371 2.80276 +1357620 8.29873 2.54377 +1020000 8.4713 2.48141 +743580 8.8489 2.35881 +522240 9.99551 2.55679 +349860 9.1925 2.28953 +220320 7.99395 2.10036 +127500 8.62231 2.07285 +65280 11.2885 2.24716 +27540 16.7748 1.61957 +8160 32.6645 1.9753 +1020 264.541 2.82625 HTS_BCE firefox => Table with 12 rows, 3 columns: VSIZE serverbandwidth clientbandwidth ───────────────────────────────────────── -1762560 7.8439 6.88982 -1357620 7.41197 6.29824 -1020000 8.48551 6.14324 -743580 8.63577 6.31355 -522240 9.17379 6.67839 -349860 9.59405 6.32544 -220320 8.73156 6.46279 -127500 4.58813 4.95919 -65280 6.10207 5.34758 -27540 10.8064 5.09502 -8160 29.9181 5.77555 -1020 189.542 21.3083 +1762560 7.93723 7.08603 +1357620 8.04689 7.23327 +1020000 8.59622 6.29865 +743580 9.22209 6.33995 +522240 9.5801 6.44354 +349860 9.96628 7.03896 +220320 10.5642 6.05941 +127500 6.14458 4.87543 +65280 9.71296 5.16403 +27540 20.6851 4.95048 +8160 67.5271 12.0259 +1020 426.923 34.7188 HTS_JCE => Table with 12 rows, 3 columns: VSIZE serverbandwidth clientbandwidth ───────────────────────────────────────── -1762560 2.0474 4.43299 -1357620 1.81932 4.27325 -1020000 1.96443 4.2789 -743580 1.76729 4.11628 -522240 1.61051 4.09457 -349860 1.61742 4.16369 -220320 1.38784 3.88246 -127500 1.96355 4.36213 -65280 3.23359 5.50702 -27540 3.43824 7.21109 -8160 7.53947 11.2128 -1020 84.4146 73.6853 +1762560 2.6018 5.29989 +1357620 2.03094 4.64923 +1020000 1.96231 5.01191 +743580 1.99671 4.64635 +522240 2.11014 4.69373 +349860 1.85716 4.72792 +220320 1.61326 4.60257 +127500 1.96424 4.54956 +65280 3.57493 7.10317 +27540 4.91413 7.4573 +8160 7.55778 13.2848 +1020 60.0278 55.4819 Benchmark Plots of varying size messages :test_latency_plots [ns], VSIZE = [1762560, 1357620, 1020000, 743580, 522240, 349860, 220320, 127500, 65280, 27540, 8160, 1020] - HTS_BCE iexplore serverlatency - ┌────────────────────────────────────────────────────────────────────────────────┐ - 6000000 │ │ - │ │ - │ ._r* │ - │ ._r-/"` │ - │ ._r-/"` │ - │ ..-/"` │ - │ ._-/"` │ - │ _r-"` │ - │ ____---/""" │ - │ .r/" │ - │ ..-"` │ - │ .__--/` │ - │ _r/""` │ - │ .__-/" │ - 0 │_--""` │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE iexplore clientlatency - ┌────────────────────────────────────────────────────────────────────────────────┐ - 6000000 │ │ - │ ._/ │ - │ ._r-"` │ - │ ._-/"` │ - │ __-/"` │ - │ __--"" │ - │ __--"" │ - │ ._-/" │ - │ ._-/"` │ - │ __-/"` │ - │ __--"" │ - │ _r/" │ - │ __-/" │ - │ ._.-/" │ - 0 │_.-/"` │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE chrome serverlatency - ┌────────────────────────────────────────────────────────────────────────────────┐ - 10000000 │ ._-/ │ - │ ._r-"` │ - │ _.-/` │ - │ ._-/" │ - │ _r-"` │ - │ ._-/" │ - │ ._.-"` │ - │ .__.--/""` │ - │ ._r/"` │ - │ _.-"` │ - │ _r/" │ - │ _r/" │ - │ ._-" │ - │ ._-/` │ - 0 │__--"` │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE chrome clientlatency - ┌────────────────────────────────────────────────────────────────────────────────┐ - 20000000 │ │ - │ │ - │ │ - │ │ - │ │ - │ ._* │ - │ ..-"` │ - │ _r/"` │ - │ ._-/" │ - │ __r--"` │ - │ .__--/"" │ - │ .__.--/""` │ - │ .__.--f""` │ - │ .______r----/""` │ - 0 │____.-/"` │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE phantomjs serverlatency - ┌────────────────────────────────────────────────────────────────────────────────┐ - 20000000 │ │ - │ │ - │ │ - │ │ - │ ._.* │ - │ __r--""` │ - │ .__--/"" │ - │ __r-/"` │ - │ ._.--"" │ - │ .__r-/""` │ - │ __.--""` │ - │ _.-/"" │ - │ __-/" │ - │ _u-/"" │ - 0 │__r--"" │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE phantomjs clientlatency - ┌────────────────────────────────────────────────────────────────────────────────┐ - 5000000 │ │ - │ .. │ - │ ._--"` │ - │ ._-/"` │ - │ _.-/"` │ - │ ._-/" │ - │ _r-"` │ - │ ._-/" │ - │ __-/"` │ - │ ._r-"" │ - │ ._r/"` │ - │ __r-"` │ - │ ._.--"" │ - │ __r-"` │ - 0 │__--/"" │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE firefox serverlatency - ┌────────────────────────────────────────────────────────────────────────────────┐ - 20000000 │ │ - │ │ - │ │ - │ │ - │ ._/ │ - │ ._.-/"` │ - │ _.-/"` │ - │ .___r---"" │ - │ ._.---""""` │ - │ __r-/"` │ - │ __r-/"" │ - │ __r-/"" │ - │ ._--"" │ - │ .r/"` │ - 0 │___r--"` │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE firefox clientlatency - ┌────────────────────────────────────────────────────────────────────────────────┐ - 20000000 │ │ - │ │ - │ │ - │ │ - │ │ - │ . │ - │ __--"" │ - │ .__--"" │ - │ __.-/"` │ - │ __.--/"" │ - │ .__r--/"" │ - │ .___---"""` │ - │ ._.--/""` │ - │ .__---""` │ - 0 │___.--/"` │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_JCE serverlatency - ┌────────────────────────────────────────────────────────────────────────────────┐ - 4000000 │ │ - │ ..* │ - │ ._-/` │ - │ _-/` │ - │ _r/" │ - │ __.-" │ - │ .__--/"" │ - │ ._--""` │ - │ ..-/` │ - │ _.-"` │ - │ ._-/" │ - │ .__-/"` │ - │ ._r-/"` │ - │ .__.-/"` │ - 0 │_-""""` │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_JCE clientlatency - ┌────────────────────────────────────────────────────────────────────────────────┐ - 8000000 │ ._-* │ - │ ._-/` │ - │ _.-"` │ - │ ._r/" │ - │ ._r-"` │ - │ ._r-"` │ - │ _r-"` │ - │ ._-/" │ - │ _r-"` │ - │ _.-/" │ - │ ._.-/" │ - │ ._r-"` │ - │ ._-/"` │ - │ __--"` │ - 0 │_-/"" │ - └────────────────────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE + HTS_BCE iexplore serverlatency + ┌─────────────────────────────────────────────────────────────────┐ + 6000000 │ │ + │ │ + │ ._-/ │ + │ _.-"` │ + │ ._-/" │ + │ ._r-/` │ + │ ._--"` │ + │ ___.--/"` │ + │ _-""" │ + │ .r" │ + │ ._r/` │ + │ .,-/"` │ + │ _r/` │ + │ ..-/" │ + 0 │-"` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE iexplore clientlatency + ┌─────────────────────────────────────────────────────────────────┐ + 6000000 │ _r/ │ + │ ../" │ + │ _-/` │ + │ ../" │ + │ ._-/` │ + │ ._-/` │ + │ ..-"` │ + │ ..-"` │ + │ ._-/` │ + │ ._-/` │ + │ ._-/` │ + │ _.-"` │ + │ ..-" │ + │ ._--/` │ + 0 │_r/` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome serverlatency + ┌─────────────────────────────────────────────────────────────────┐ + 11000000 │ │ + │ _r/` │ + │ ../" │ + │ ._-"` │ + │ _r/` │ + │ ._-/" │ + │ _r-"` │ + │ __r--"" │ + │ .r/"" │ + │ _-"` │ + │ ..-" │ + │ .r/` │ + │ _-/` │ + │ _---/" │ + 0 │.-" │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome clientlatency + ┌─────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ _/ │ + │ _r/" │ + │ ..-" │ + │ ._-/` │ + │ _.-/` │ + │ ._-/" │ + │ ._.-/"` │ + │ .__--/"` │ + │ __r-/"` │ + │ .__,-/"" │ + 0 │__.--/""` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs serverlatency + ┌─────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ .. │ + │ _r/` │ + │ ../" │ + │ _-/` │ + │ ._./" │ + │ __-/"` │ + │ ._r-/" │ + │ ._r-/"` │ + │ .__--/"` │ + │ ..-"` │ + │ _r/` │ + │ ._r/" │ + 0 │.--""` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs clientlatency + ┌─────────────────────────────────────────────────────────────────┐ + 5000000 │ ../` │ + │ ../` │ + │ ./` │ + │ _-" │ + │ ._-" │ + │ ._-/` │ + │ ._-/` │ + │ _-/` │ + │ _r/" │ + │ ._r/" │ + │ .__-/"` │ + │ .r/` │ + │ ._r"` │ + │ ._-/` │ + 0 │_.-/"` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox serverlatency + ┌─────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ ._r/ │ + │ ._.-/"` │ + │ _.-/"` │ + │ __r-/"" │ + │ ._.-/"" │ + │ ._r-/"` │ + │ ._-/"` │ + │ _.-/"` │ + │ ._-/" │ + │ .-"` │ + 0 │----/` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox clientlatency + ┌─────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ │ + │ .__/ │ + │ ._r-/"` │ + │ ._--""` │ + │ ..-"` │ + │ _r-"` │ + │ ._.--"" │ + │ ._.--""` │ + │ .__r--""` │ + │ __-/"` │ + 0 │___--/" │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE serverlatency + ┌─────────────────────────────────────────────────────────────────┐ + 5000000 │ │ + │ .-` │ + │ ./` │ + │ .r" │ + │ _/` │ + │ ../ │ + │ _r` │ + │ _.-/" │ + │ _.-/" │ + │ ._r-/" │ + │ ._r-/"` │ + │ ._--/"` │ + │ ..-/` │ + │ ._.-"` │ + 0 │.-""""` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE clientlatency + ┌─────────────────────────────────────────────────────────────────┐ + 10000000 │ . │ + │ .r" │ + │ .r/` │ + │ .r/` │ + │ ../` │ + │ ._r/` │ + │ ._r-/"` │ + │ ._-""` │ + │ _r/` │ + │ ..-" │ + │ ._-/"` │ + │ ._r/"` │ + │ ._-"` │ + │ ._r-"` │ + 0 │_-""` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE Benchmark Tables of varying size messages :test_latency_tables [ns] +PhantomJS saved render, exits after 30s HTS_BCE iexplore => Table with 12 rows, 3 columns: VSIZE serverlatency clientlatency ───────────────────────────────────── -1762560 5.01966e6 5.33571e6 -1357620 3.93391e6 4.08703e6 -1020000 2.77963e6 3.16965e6 -743580 2.38672e6 2.26692e6 -522240 1.43032e6 1.66552e6 -349860 1.08355e6 1.00596e6 -220320 5.72753e5 6.09253e5 -127500 3.91916e5 3.84064e5 -65280 2.46346e5 1.98141e5 -27540 1.44201e5 79575.2 -8160 89100.4 29529.4 -1020 59000.4 12614.0 +1762560 5.0331e6 5.84469e6 +1357620 3.85969e6 4.35764e6 +1020000 3.0365e6 3.27503e6 +743580 2.68647e6 2.35556e6 +522240 1.69114e6 1.63403e6 +349860 1.31929e6 1.1182e6 +220320 8.04377e5 6.6569e5 +127500 5.79405e5 4.66486e5 +65280 3.67186e5 2.25881e5 +27540 2.55881e5 95358.0 +8160 1.73917e5 42019.3 +1020 1.42514e5 19971.3 HTS_BCE chrome => Table with 12 rows, 3 columns: VSIZE serverlatency clientlatency ───────────────────────────────────── -1762560 9.78596e6 1.26154e7 -1357620 7.44132e6 7.58628e6 -1020000 5.41492e6 5.21805e6 -743580 4.61223e6 3.54826e6 -522240 3.31534e6 2.26995e6 -349860 2.14437e6 1.66901e6 -220320 1.12452e6 1.37407e6 -127500 6.12896e5 4.92053e5 -65280 2.63813e5 2.69781e5 -27540 1.6586e5 96012.8 -8160 1.25951e5 45635.1 -1020 66758.9 19428.0 +1762560 1.01767e7 1.38074e7 +1357620 7.53576e6 9.08375e6 +1020000 5.75107e6 6.11676e6 +743580 4.84169e6 4.27534e6 +522240 3.21797e6 2.84406e6 +349860 1.9739e6 1.77443e6 +220320 1.15552e6 1.14363e6 +127500 1.06252e6 6.84006e5 +65280 4.99781e5 3.22632e5 +27540 2.35828e5 1.23624e5 +8160 2.53945e5 70086.3 +1020 1.92733e5 32490.5 HTS_BCE phantomjs => Table with 12 rows, 3 columns: VSIZE serverlatency clientlatency ───────────────────────────────────── -1762560 1.39174e7 4.39194e6 -1357620 1.0958e7 3.35233e6 -1020000 8.2516e6 2.34283e6 -743580 6.35676e6 1.69266e6 -522240 4.80404e6 1.086e6 -349860 2.85313e6 7.39385e5 -220320 1.76908e6 4.09622e5 -127500 8.98254e5 2.62619e5 -65280 4.97293e5 152920.0 -27540 244710.0 42551.8 -8160 149441.0 12869.3 -1020 1.18169e5 2208.09 +1762560 1.62397e7 4.94004e6 +1357620 1.12665e7 3.45348e6 +1020000 8.64073e6 2.53104e6 +743580 6.57986e6 1.75396e6 +522240 5.22005e6 1.33526e6 +349860 3.21609e6 8.01016e5 +220320 1.76123e6 4.6275e5 +127500 1.09934e6 2.64289e5 +65280 7.3691e5 1.46695e5 +27540 4.61979e5 44603.0 +8160 2.66542e5 16118.5 +1020 2.69832e5 2882.77 HTS_BCE firefox => Table with 12 rows, 3 columns: VSIZE serverlatency clientlatency ───────────────────────────────────── -1762560 1.38254e7 1.21437e7 -1357620 1.00626e7 8.55061e6 -1020000 8.65522e6 6.26611e6 -743580 6.42138e6 4.69463e6 -522240 4.79092e6 3.48772e6 -349860 3.35657e6 2.21302e6 -220320 1.92374e6 1.42388e6 -127500 5.84986e5 6.32297e5 -65280 3.98343e5 3.4909e5 -27540 2.97608e5 1.40317e5 -8160 2.44132e5 47128.5 -1020 1.93333e5 21734.5 +1762560 1.39898e7 1.24896e7 +1357620 1.09246e7 9.82003e6 +1020000 8.76814e6 6.42462e6 +743580 6.85736e6 4.71426e6 +522240 5.00311e6 3.36507e6 +349860 3.4868e6 2.46265e6 +220320 2.3275e6 1.33501e6 +127500 7.83433e5 6.21618e5 +65280 6.34062e5 337108.0 +27540 5.69668e5 1.36336e5 +8160 5.51021e5 98131.3 +1020 4.35462e5 35413.2 HTS_JCE => Table with 12 rows, 3 columns: VSIZE serverlatency clientlatency ───────────────────────────────────── -1762560 3.60866e6 7.81341e6 -1357620 2.46994e6 5.80145e6 -1020000 2.00372e6 4.36448e6 -743580 1.31412e6 3.06078e6 -522240 841072.0 2.13835e6 -349860 5.65872e5 1.45671e6 -220320 305769.0 8.55384e5 -127500 2.50353e5 5.56171e5 -65280 211089.0 359498.0 -27540 94689.1 1.98593e5 -8160 61522.1 91496.4 -1020 86102.9 75159.0 +1762560 4.58583e6 9.34138e6 +1357620 2.75724e6 6.31189e6 +1020000 2.00155e6 5.11215e6 +743580 1.48471e6 3.45493e6 +522240 1.102e6 2.45125e6 +349860 6.49745e5 1.65411e6 +220320 3.55433e5 1.01404e6 +127500 2.5044e5 5.80069e5 +65280 2.33372e5 4.63695e5 +27540 1.35335e5 205374.0 +8160 61671.5 1.08404e5 +1020 61228.4 56591.6 Benchmark Dictionary :test_bestserverlatencies [ns] -HTS_BCE iexplore => 59000 -HTS_BCE chrome => 66759 -HTS_BCE phantomjs => 118169 -HTS_BCE firefox => 193333 -HTS_JCE => 61522 +HTS_BCE iexplore => 142514 +HTS_BCE chrome => 192733 +HTS_BCE phantomjs => 266542 +HTS_BCE firefox => 435462 +HTS_JCE => 61228 Benchmark Dictionary :test_bestclientlatencies [ns] -HTS_BCE iexplore => 12614 -HTS_BCE chrome => 19428 -HTS_BCE phantomjs => 2208 -HTS_BCE firefox => 21735 -HTS_JCE => 75159 +HTS_BCE iexplore => 19971 +HTS_BCE chrome => 32490 +HTS_BCE phantomjs => 2883 +HTS_BCE firefox => 35413 +HTS_JCE => 56592 Benchmark Dictionary :test_bestserverbandwidths [ns/b] -HTS_BCE iexplore => 2.5996 -HTS_BCE chrome => 4.0413 -HTS_BCE phantomjs => 7.0451 -HTS_BCE firefox => 4.5881 -HTS_JCE => 1.3878 +HTS_BCE iexplore => 2.843 +HTS_BCE chrome => 5.2447 +HTS_BCE phantomjs => 7.994 +HTS_BCE firefox => 6.1446 +HTS_JCE => 1.6133 Benchmark Dictionary :test_bestclientbandwidths [ns/b] -HTS_BCE iexplore => 2.7653 -HTS_BCE chrome => 3.4863 -HTS_BCE phantomjs => 1.5451 -HTS_BCE firefox => 4.9592 -HTS_JCE => 3.8825 +HTS_BCE iexplore => 3.0215 +HTS_BCE chrome => 4.4889 +HTS_BCE phantomjs => 1.6196 +HTS_BCE firefox => 4.8754 +HTS_JCE => 4.5496 diff --git a/src/HttpServer.jl b/src/HttpServer.jl index cd23ebb..954aa5a 100644 --- a/src/HttpServer.jl +++ b/src/HttpServer.jl @@ -47,7 +47,7 @@ Typical headers: "Accept-Language" => "en-US,en;q=0.5" """ function websocket_handshake(request, client) - + response = HttpServer.Response(101) if get(request.headers, "Sec-WebSocket-Version", "13") != "13" response = HttpServer.Response(400) response.headers["Sec-WebSocket-Version"] = "13" @@ -74,7 +74,7 @@ function websocket_handshake(request, client) end resp_key = generate_websocket_key(key) - response = HttpServer.Response(101) + response.headers["Upgrade"] = "websocket" response.headers["Connection"] = "Upgrade" response.headers["Sec-WebSocket-Accept"] = resp_key diff --git a/src/WebSockets.jl b/src/WebSockets.jl index 4c22327..f69edba 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -212,9 +212,9 @@ function Base.close(ws::WebSocket; statusnumber = 0) if isopen(ws) ws.state = CLOSING if statusnumber == 0 - locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server) + locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, UInt8[]) else - statuscode = reverse(reinterpret(UInt8, [UInt16(statusnumber)])) + statuscode = reinterpret(UInt8, [hton(UInt16(statusnumber))]) locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, statuscode) end @@ -501,7 +501,10 @@ function generate_websocket_key(key) end """ -Masks or unmasks data, returns the key for reverting mask. + maskswitch!(data) + maskswitch!(data, key:: 4-element Vector{UInt8}) + +Masks or unmasks data in-place, returns the key used. Calling twice with the same key restores data. Ref. RFC 6455 5-3. """ @@ -509,7 +512,7 @@ function maskswitch!(data, mask = rand(UInt8, 4)) for i in 1:length(data) data[i] = data[i] ⊻ mask[((i-1) % 4)+1] end - return mask + return mask end "Used in handshake. See SUBProtocols" diff --git a/test/runtests.jl b/test/runtests.jl index 497484a..59bcb5a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,8 +2,239 @@ using HTTP using HttpServer using WebSockets using Base.Test +import WebSockets: generate_websocket_key, + write_fragment, + read_frame, + websocket_handshake, + maskswitch! +import HttpServer: is_websocket_handshake, + handle +import HttpCommon: Request +@sync yield() # avoid mixing of output with possible deprecation warnings from .juliarc +info("Starting test WebSockets...") +#is_control_frame is one line, checking one bit. +#get_websocket_key grabs a header. +#is_websocket_handshake grabs a header. +#generate_websocket_key makes a call to a library. +info("Test generate_websocket_key") +@test generate_websocket_key("dGhlIHNhbXBsZSBub25jZQ==") == "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" -@testset "HTTP" begin +""" +The dummy websocket don't use TCP. Close won't work, but we can manipulate the contents +using otherwise the same functions as TCP sockets. +""" +dummyws(server::Bool) = WebSocket(BufferStream(), server) + +const io = IOBuffer() + + +info("Test length less than 126") +@testset "Unit test, fragment length less than 126" begin + +for len = [8, 125], fin=[true, false], clientwriting = [false, true] + + op = (rand(UInt8) & 0b1111) + test_str = randstring(len) + # maskswitch two times with same key == unmasked + maskunmask = copy(Vector{UInt8}(test_str)) + mskkey = maskswitch!(maskunmask) + maskswitch!(maskunmask, mskkey) + @test maskunmask == Vector{UInt8}(test_str) + + # websocket fragment as Vector{UInt8} + # for client writing, the data is masked and the mask is contained in the frame. + # for server writing, the data is not masked, and the header is four bytes shorter. + write_fragment(io, fin, op, clientwriting, copy(Vector{UInt8}(test_str))) + # test that the original input string was not masked. + @test maskunmask == Vector{UInt8}(test_str) + frame = take!(io) + # Check the frame header + # Last frame bit + @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] + # payload length bit + @test frame[2] & 0b0111_1111 == len + # ismasked bit + hasmsk = frame[2] & 0b1000_0000 >>> 7 != 0 + @test hasmsk == clientwriting + # payload data + if hasmsk + framedata = copy(frame[7:end]) + maskswitch!(framedata, frame[3:6]) + else + framedata = frame[3:end] + end + + @test framedata == Vector{UInt8}(test_str) + + # Test for WebSocketError when reading + # masked frame-> websocket|server + # unmasked frame -> websocket|client + + # Let's pretend TCP has moved our frame into the peer websocket + receivingws = dummyws(!clientwriting) + write(receivingws.socket, frame) + @test_throws WebSockets.WebSocketError read_frame(receivingws) + close(receivingws.socket) + + # Let's pretend receivingws didn't error like it should, but + # echoed our message back with identical masking. + dws = dummyws(clientwriting) + @test dws.server == clientwriting + write(dws.socket, frame) + # read the frame back, now represented as a WebSocketFragment + + frag_back = read_frame(dws) + close(dws.socket) + @test frag_back.is_last == fin + @test frag_back.rsv1 == false + @test frag_back.rsv2 == false + @test frag_back.rsv3 == false + @test frag_back.opcode == op + @test frag_back.is_masked == clientwriting + @test frag_back.payload_len == len + maskkey = UInt8[] + if clientwriting + maskkey = frame[3:6] + end + @test frag_back.maskkey == maskkey + # the WebSocketFragment stores the data after unmasking + @test Vector{UInt8}(test_str) == frag_back.data +end +end # testset + + +info("Test length 126 or more") +@testset "Unit test, fragment length 126 or more" begin + +for len = 126:129, fin=[true, false], clientwriting = [false, true] + op = 0b1111 + test_str = randstring(len) + write_fragment(io, fin, op, clientwriting, copy(Vector{UInt8}(test_str))) + frame = take!(io) + + @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] + @test frame[2] & 0b0111_1111 == 126 + @test bits(frame[4])*bits(frame[3]) == bits(hton(UInt16(len))) + + dws = dummyws(clientwriting) + write(dws.socket, frame) + frag_back = read_frame(dws) + close(dws.socket) + + @test frag_back.is_last == fin + @test frag_back.rsv1 == false + @test frag_back.rsv2 == false + @test frag_back.rsv3 == false + @test frag_back.opcode == op + @test frag_back.is_masked == clientwriting + @test frag_back.payload_len == len + @test test_str == String(frag_back.data) +end + +end # testset + + +# TODO: test for length > typemax(Uint32) +@testset "Unit test, HttpServer handshake" begin + +info("Tests for is_websocket_handshake") +chromeheaders = Dict{String, String}( + "Connection"=>"Upgrade", + "Upgrade"=>"websocket" + ) +chromerequest = HttpCommon.Request( + "GET", + "", + chromeheaders, + "" + ) + +firefoxheaders = Dict{String, String}( + "Connection"=>"keep-alive, Upgrade", + "Upgrade"=>"websocket" + ) + +firefoxrequest= Request( + "GET", + "", + firefoxheaders, + "" + ) + +wshandler = WebSocketHandler((x,y)->nothing);#Dummy wshandler + +for request in [chromerequest, firefoxrequest] + @test is_websocket_handshake(wshandler,request) == true +end + +info("Test of handshake response") +takefirstline(buf) = split(buf |> take! |> String, "\r\n")[1] + +take!(io) +Base.write(io, "test") +@test takefirstline(io) == "test" + +info("Test reject / switch format") +const SWITCH = "HTTP/1.1 101 Switching Protocols " +const REJECT = "HTTP/1.1 400 Bad Request " +Base.write(io, Response(400)) +@test takefirstline(io) == REJECT +Base.write(io, Response(101)) +@test takefirstline(io) == SWITCH + +function handshakeresponse(request) + cli = HttpServer.Client(2, IOBuffer()) + websocket_handshake(request, cli) + takefirstline(cli.sock) +end + +info("Test simple handshakes that are unacceptable") +for request in [chromerequest, firefoxrequest] + @test handshakeresponse(request) == REJECT + push!(request.headers, "Sec-WebSocket-Version" => "13") + @test handshakeresponse(request) == REJECT + push!(request.headers, "Sec-WebSocket-Key" => "mumbojumbobo") + @test handshakeresponse(request) == REJECT + push!(request.headers, "Sec-WebSocket-Version" => "11") + push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") + @test handshakeresponse(request) == REJECT +end + +info("Test simple handshakes, acceptable") +for request in [chromerequest, firefoxrequest] + push!(request.headers, "Sec-WebSocket-Version" => "13") + push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") + @test handshakeresponse(request) == SWITCH +end + +info("Test unacceptable subprotocol handshake subprotocol") +for request in [chromerequest, firefoxrequest] + push!(request.headers, "Sec-WebSocket-Version" => "13") + push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") + push!(request.headers, "Sec-WebSocket-Protocol" => "my.server/json-zmq") + @test handshakeresponse(request) == REJECT +end + +info("add simple subprotocol to acceptable list") +@test true == WebSockets.addsubproto("xml") + +info("add subprotocol with difficult name") +@test true == WebSockets.addsubproto("my.server/json-zmq") + +info("Test handshake subprotocol now acceptable") +for request in [chromerequest, firefoxrequest] + push!(request.headers, "Sec-WebSocket-Version" => "13") + push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") + push!(request.headers, "Sec-WebSocket-Protocol" => "xml") + @test handshakeresponse(request) == SWITCH + push!(request.headers, "Sec-WebSocket-Protocol" => "my.server/json-zmq") + @test handshakeresponse(request) == SWITCH +end +end # testset + +close(io) + +@testset "Peer-to-peer tests, HTTP client" begin const port_HTTP = 8000 const port_HttpServer = 8081 @@ -34,25 +265,33 @@ server = Server(wsh) sleep(4) servers = [ - ("ws", "ws://echo.websocket.org"), - ("wss", "wss://echo.websocket.org"), ("HTTP", "ws://127.0.0.1:$(port_HTTP)"), - ("HttpServer", "ws://127.0.0.1:$(port_HttpServer)")] + ("HttpServer", "ws://127.0.0.1:$(port_HttpServer)"), + ("ws", "ws://echo.websocket.org"), + ("wss", "wss://echo.websocket.org")] -for (s, url) in servers - info("Testing ws client connecting to $(s) server at $(url)...") +lengths = [0, 3, 125, 126, 127, 2000] + +for (s, url) in servers, len in lengths, closestatus in [false, true] + len == 0 && contains(url, "echo.websocket.org") && continue + info("Testing client -> server at $(url), message length $len") + test_str = randstring(len) + forcecopy_str = test_str |> collect |> copy |> join WebSockets.open(url) do ws print(" -Foo-") write(ws, "Foo") @test String(read(ws)) == "Foo" print(" -Ping-") send_ping(ws) - println(" -Bar-") - write(ws, "Bar") - @test String(read(ws)) == "Bar" + print(" -String length $len-\n") + write(ws, test_str) + @test String(read(ws)) == forcecopy_str + closestatus && close(ws, statusnumber = 1000) sleep(1) end sleep(1) end -end # testset \ No newline at end of file +end # testset + + From dce6a49bea2fc3cd14fdc5e66cbc9d100895065e Mon Sep 17 00:00:00 2001 From: hustf Date: Mon, 28 May 2018 22:23:00 +0200 Subject: [PATCH 28/35] Expand HttpServer tests to HTTP modified: src/HTTP.jl change req.resource -> req.target modified: test/runtests.jl add testset "Test length typemax(UInt16) + 1" include HTTP in handshake testsets --- src/HTTP.jl | 2 +- test/runtests.jl | 254 +++++++++++++++++++++++++++++++++-------------- 2 files changed, 182 insertions(+), 74 deletions(-) diff --git a/src/HTTP.jl b/src/HTTP.jl index 43cc48c..9a00012 100644 --- a/src/HTTP.jl +++ b/src/HTTP.jl @@ -189,7 +189,7 @@ function is_upgrade(r::HTTP.Message) return false end # Inline docs in 'WebSockets.jl' -target(req::HTTP.Messages.Request) = req.resource +target(req::HTTP.Messages.Request) = req.target subprotocol(req::HTTP.Messages.Request) = HTTP.header(req, "Sec-WebSocket-Protocol") origin(req::HTTP.Messages.Request) = HTTP.header(req, "Origin") diff --git a/test/runtests.jl b/test/runtests.jl index 59bcb5a..b643df5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,7 +6,9 @@ import WebSockets: generate_websocket_key, write_fragment, read_frame, websocket_handshake, - maskswitch! + maskswitch!, + is_upgrade, + upgrade import HttpServer: is_websocket_handshake, handle import HttpCommon: Request @@ -25,6 +27,12 @@ using otherwise the same functions as TCP sockets. """ dummyws(server::Bool) = WebSocket(BufferStream(), server) +function dummywshandler(req, dws::WebSockets.WebSocket{BufferStream}) + close(dws.socket) + close(dws) +end + + const io = IOBuffer() @@ -132,87 +140,139 @@ for len = 126:129, fin=[true, false], clientwriting = [false, true] end end # testset +info("Test length typemax(UInt16) + 1") +@testset "Test length typemax(UInt16) + 1" begin +for clientwriting = [false, true] + len = typemax(UInt16) +1 + op = 0b1111 + fin = true + + test_str = randstring(len) + write_fragment(io, fin, op, clientwriting, copy(Vector{UInt8}(test_str))) + frame = take!(io) -# TODO: test for length > typemax(Uint32) -@testset "Unit test, HttpServer handshake" begin + dws = dummyws(clientwriting) + write(dws.socket, frame) + frag_back = read_frame(dws) + close(dws.socket) + + @test frag_back.is_last == fin + @test frag_back.rsv1 == false + @test frag_back.rsv2 == false + @test frag_back.rsv3 == false + @test frag_back.opcode == op + @test frag_back.is_masked == clientwriting + @test frag_back.payload_len == len + @test test_str == String(frag_back.data) +end + +end # testset + + +@testset "Unit test, HttpServer and HTTP handshake" begin info("Tests for is_websocket_handshake") -chromeheaders = Dict{String, String}( - "Connection"=>"Upgrade", - "Upgrade"=>"websocket" - ) -chromerequest = HttpCommon.Request( - "GET", - "", - chromeheaders, - "" - ) - -firefoxheaders = Dict{String, String}( - "Connection"=>"keep-alive, Upgrade", - "Upgrade"=>"websocket" - ) - -firefoxrequest= Request( - "GET", - "", - firefoxheaders, - "" - ) - -wshandler = WebSocketHandler((x,y)->nothing);#Dummy wshandler +function templaterequests() + chromeheaders = Dict{String, String}( "Connection"=>"Upgrade", + "Upgrade"=>"websocket") + firefoxheaders = Dict{String, String}("Connection"=>"keep-alive, Upgrade", + "Upgrade"=>"websocket") + + chromerequest = HttpCommon.Request("GET", "", chromeheaders, "") + firefoxrequest= Request("GET", "", firefoxheaders, "") + + chromerequest_HTTP = HTTP.Messages.Request("GET", "/", collect(chromeheaders)) + firefoxrequest_HTTP = HTTP.Messages.Request("GET", "/", collect(firefoxheaders)) + + return [chromerequest, firefoxrequest, chromerequest_HTTP, firefoxrequest_HTTP] +end + +chromerequest, firefoxrequest, chromerequest_HTTP, firefoxrequest_HTTP = Tuple(templaterequests()) + +wshandler = WebSocketHandler((x,y)->nothing) for request in [chromerequest, firefoxrequest] - @test is_websocket_handshake(wshandler,request) == true + @test is_websocket_handshake(wshandler, request) == true end -info("Test of handshake response") -takefirstline(buf) = split(buf |> take! |> String, "\r\n")[1] +for request in [chromerequest_HTTP, firefoxrequest_HTTP] + @test is_upgrade(request) == true +end +info("Test of handshake response") +takefirstline(buf::IOBuffer) = strip(split(buf |> take! |> String, "\r\n")[1]) +takefirstline(buf::BufferStream) = strip(split(buf |> read |> String, "\r\n")[1]) take!(io) -Base.write(io, "test") -@test takefirstline(io) == "test" info("Test reject / switch format") -const SWITCH = "HTTP/1.1 101 Switching Protocols " -const REJECT = "HTTP/1.1 400 Bad Request " +const REJECT = "HTTP/1.1 400 Bad Request" Base.write(io, Response(400)) @test takefirstline(io) == REJECT +Base.write(io, HTTP.Messages.Response(400)) +@test takefirstline(io) == REJECT + +const SWITCH = "HTTP/1.1 101 Switching Protocols" Base.write(io, Response(101)) @test takefirstline(io) == SWITCH +Base.write(io, HTTP.Messages.Response(101)) +@test takefirstline(io) == SWITCH + -function handshakeresponse(request) +function handshakeresponse(request::Request) cli = HttpServer.Client(2, IOBuffer()) websocket_handshake(request, cli) - takefirstline(cli.sock) + strip(takefirstline(cli.sock)) +end +function handshakeresponse(request::HTTP.Messages.Request) + buf = BufferStream() + c = HTTP.ConnectionPool.Connection(buf) + t = HTTP.Transaction(c) + s = HTTP.Streams.Stream(request, t) + upgrade(dummywshandler, s) + close(buf) + takefirstline(buf) end info("Test simple handshakes that are unacceptable") -for request in [chromerequest, firefoxrequest] - @test handshakeresponse(request) == REJECT - push!(request.headers, "Sec-WebSocket-Version" => "13") - @test handshakeresponse(request) == REJECT - push!(request.headers, "Sec-WebSocket-Key" => "mumbojumbobo") - @test handshakeresponse(request) == REJECT - push!(request.headers, "Sec-WebSocket-Version" => "11") - push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") - @test handshakeresponse(request) == REJECT + +sethd(r::Request, pa::Pair) = push!(r.headers, pa) +sethd(r::HTTP.Messages.Request, pa::Pair) = HTTP.Messages.setheader(r, HTTP.Header(pa)) + +for r in templaterequests() + @test handshakeresponse(r) == REJECT + sethd(r, "Sec-WebSocket-Version" => "11") + @test handshakeresponse(r) == REJECT + sethd(r, "Sec-WebSocket-Version" => "13") + @test handshakeresponse(r) == REJECT + sethd(r, "Sec-WebSocket-Key" => "mumbojumbobo") + @test handshakeresponse(r) == REJECT + sethd(r, "Sec-WebSocket-Key" => "17 bytes key is not accepted") + @test handshakeresponse(r) == REJECT + sethd(r, "Sec-WebSocket-Key" => "16 bytes key this is surely") + sethd(r, "Sec-WebSocket-Protocol" => "unsupported") + @test handshakeresponse(r) == REJECT end + + + + info("Test simple handshakes, acceptable") -for request in [chromerequest, firefoxrequest] - push!(request.headers, "Sec-WebSocket-Version" => "13") - push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") - @test handshakeresponse(request) == SWITCH +for r in templaterequests() + sethd(r, "Sec-WebSocket-Version" => "13") + sethd(r, "Sec-WebSocket-Key" => "16 bytes key this is surely") + @test handshakeresponse(r) == SWITCH end + + info("Test unacceptable subprotocol handshake subprotocol") -for request in [chromerequest, firefoxrequest] - push!(request.headers, "Sec-WebSocket-Version" => "13") - push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") - push!(request.headers, "Sec-WebSocket-Protocol" => "my.server/json-zmq") - @test handshakeresponse(request) == REJECT +for r in templaterequests() + sethd(r, "Sec-WebSocket-Version" => "13") + sethd(r, "Sec-WebSocket-Key" => "16 bytes key this is surely") + sethd(r, "Sec-WebSocket-Protocol" => "my.server/json-zmq") + @test handshakeresponse(r) == REJECT end info("add simple subprotocol to acceptable list") @@ -222,13 +282,13 @@ info("add subprotocol with difficult name") @test true == WebSockets.addsubproto("my.server/json-zmq") info("Test handshake subprotocol now acceptable") -for request in [chromerequest, firefoxrequest] - push!(request.headers, "Sec-WebSocket-Version" => "13") - push!(request.headers, "Sec-WebSocket-Key" => "zkG1WqHM8BJdQMXytFqiUw==") - push!(request.headers, "Sec-WebSocket-Protocol" => "xml") - @test handshakeresponse(request) == SWITCH - push!(request.headers, "Sec-WebSocket-Protocol" => "my.server/json-zmq") - @test handshakeresponse(request) == SWITCH +for r in templaterequests() + sethd(r, "Sec-WebSocket-Version" => "13") + sethd(r, "Sec-WebSocket-Key" => "16 bytes key this is surely") + sethd(r, "Sec-WebSocket-Protocol" => "xml") + @test handshakeresponse(r) == SWITCH + sethd(r, "Sec-WebSocket-Protocol" => "my.server/json-zmq") + @test handshakeresponse(r) == SWITCH end end # testset @@ -237,9 +297,9 @@ close(io) @testset "Peer-to-peer tests, HTTP client" begin const port_HTTP = 8000 +const port_HTTP_ServeWS = 8001 const port_HttpServer = 8081 -info("Start HTTP server on port $(port_HTTP)") function echows(ws) while true @@ -249,24 +309,44 @@ function echows(ws) end end -@async HTTP.listen("127.0.0.1", UInt16(port_HTTP)) do http - if WebSockets.is_upgrade(http.message) - WebSockets.upgrade(echows, http) +function echows(req, ws) + @test origin(req) == "" + @test target(req) == "/" + @test subprotocol(req) == "" + while true + data, success = readguarded(ws) + !success && break + !writeguarded(ws, data) && break end end -info("Start HttpServer on port $(port_HttpServer)") -wsh = WebSocketHandler() do req, ws - echows(ws) +info("Start HTTP listen server on port $port_HTTP") +@async HTTP.listen("127.0.0.1", UInt16(port_HTTP)) do s + if WebSockets.is_upgrade(s.message) + WebSockets.upgrade(echows, s) + end end -server = Server(wsh) -@async run(server,port_HttpServer) + + +info("Start HttpServer on port $port_HttpServer") +server = Server(WebSocketHandler(echows)) +@async run(server, port_HttpServer) + + +info("Start HTTP ServerWS on port $port_HTTP_ServeWS") +server_WS = WebSockets.ServerWS( + HTTP.HandlerFunction(req-> HTTP.Response(200)), + WebSockets.WebsocketHandler(echows)) + +@async WebSockets.serve(server_WS, "127.0.0.1", port_HTTP_ServeWS, false) + sleep(4) servers = [ ("HTTP", "ws://127.0.0.1:$(port_HTTP)"), ("HttpServer", "ws://127.0.0.1:$(port_HttpServer)"), + ("HTTTP ServerWS", "ws://127.0.0.1:$(port_HTTP_ServeWS)"), ("ws", "ws://echo.websocket.org"), ("wss", "wss://echo.websocket.org")] @@ -287,11 +367,39 @@ for (s, url) in servers, len in lengths, closestatus in [false, true] write(ws, test_str) @test String(read(ws)) == forcecopy_str closestatus && close(ws, statusnumber = 1000) - sleep(1) + sleep(0.2) end - sleep(1) + sleep(0.2) end end # testset - +# TODO missing tests before browsertests. + +# WebSockets.jl +# provoke errors WebSocketClosedError +# error("Attempted to send too much data for one websocket fragment\n") +# direct closing of tcp socket, while reading. +# closing with given reason (only from browsertests) +# unknown opcode +# Attempt to read from closed +# Read multiple frames (use dummyws), may require change +# InterruptException +# Protocol error (not masked from client) +# writeguarded, error + + +# HTTP +# open with optionalprotocol (change to subprotocol) +# open with rejected Protocol +# open bad reply to key during handshake (writeframe, dummyws) +# improve error handling in HTTP.open (may require change) +# HTTP messages that are not upgrades +# ugrade with single argument function +# ServeWS with https +# stop ServeWS with InterruptException +# listen with http request + +# HttpServer +# exit without closing websocket +# is_websocket_handshake with normal request From ac868837df36b7dcb6ffbff3e1ee0616f4270deb Mon Sep 17 00:00:00 2001 From: hustf Date: Sat, 2 Jun 2018 11:01:56 +0200 Subject: [PATCH 29/35] Unit tests 93%, excluding browsertests modified: src/HTTP.jl separate _openstream(..) from open for test access modified: src/WebSockets.jl future improvements modified: test/runtests.jl deleted: test/HTTP.jl new file: test/client_server_test.jl new file: test/client_test.jl new file: test/error_test.jl new file: test/frametest.jl new file: test/handshaketest.jl --- src/HTTP.jl | 87 ++++---- src/WebSockets.jl | 7 +- test/HTTP.jl | 80 -------- test/client_server_test.jl | 84 ++++++++ test/client_test.jl | 77 +++++++ test/error_test.jl | 56 ++++++ test/frametest.jl | 139 +++++++++++++ test/handshaketest.jl | 131 ++++++++++++ test/runtests.jl | 399 ++----------------------------------- 9 files changed, 559 insertions(+), 501 deletions(-) delete mode 100644 test/HTTP.jl create mode 100644 test/client_server_test.jl create mode 100644 test/client_test.jl create mode 100644 test/error_test.jl create mode 100644 test/frametest.jl create mode 100644 test/handshaketest.jl diff --git a/src/HTTP.jl b/src/HTTP.jl index 9a00012..aafbc8a 100644 --- a/src/HTTP.jl +++ b/src/HTTP.jl @@ -19,7 +19,7 @@ On exit, a closing handshake is started. If the server is not currently reading (which is a blocking function), this side will reset the underlying connection (ECONNRESET) after a reasonable amount of time and continue execution. """ -function open(f::Function, url; verbose=false, optionalProtocol = "", kw...) +function open(f::Function, url; verbose=false, subprotocol = "", kw...) key = base64encode(rand(UInt8, 16)) headers = [ @@ -28,44 +28,54 @@ function open(f::Function, url; verbose=false, optionalProtocol = "", kw...) "Sec-WebSocket-Key" => key, "Sec-WebSocket-Version" => "13" ] - if optionalProtocol != "" - push!(headers, "Sec-WebSocket-Protocol" => optionalProtocol ) + if subprotocol != "" + push!(headers, "Sec-WebSocket-Protocol" => subprotocol ) end try HTTP.open("GET", url, headers; reuse_limit=0, verbose=verbose ? 2 : 0, kw...) do http + _openstream(f, http, key) + end + catch err + # TODO don't pass on WebSocketClosedError and other known ones. + if typeof(err) <: Base.UVError + throw(WebSocketClosedError(" while open ws|client: $(string(err))")) + elseif typeof(err) <: HTTP.ExceptionRequest.StatusError + return err.response + else + rethrow(err) + end + end +end +"Called by open with a stream connected to a server, after handshake is initiated" +function _openstream(f::Function, http::HTTP.Streams.Stream, key::String) - HTTP.startread(http) - - status = http.message.status - if status != 101 - return - end + HTTP.startread(http) - check_upgrade(http) + status = http.message.status + if status != 101 + return + end + + check_upgrade(http) - if HTTP.header(http, "Sec-WebSocket-Accept") != generate_websocket_key(key) - throw(WebSocketError(0, "Invalid Sec-WebSocket-Accept\n" * - "$(http.message)")) - end + if HTTP.header(http, "Sec-WebSocket-Accept") != generate_websocket_key(key) + throw(WebSocketError(0, "Invalid Sec-WebSocket-Accept\n" * + "$(http.message)")) + end - io = HTTP.ConnectionPool.getrawstream(http) - ws = WebSocket(io,false) - try - f(ws) - finally - close(ws) - end - end - catch err - if typeof(err) == Base.UVError - warn(STDERR, err) - else - rethrow(err) - end + io = HTTP.ConnectionPool.getrawstream(http) + ws = WebSocket(io,false) + try + f(ws) + finally + close(ws) end + end + + """ Used as part of a server definition. Call this if is_upgrade(http.message) returns true. @@ -150,6 +160,7 @@ function upgrade(f::Function, http::HTTP.Stream) f(ws) end catch err + # TODO improve warn("WebSockets.HTTP.upgrade: Caught unhandled error while calling argument function f, the handler / gatekeeper:\n\t") mt = typeof(f).name.mt fnam = splitdir(string(mt.defs.func.file))[2] @@ -213,10 +224,13 @@ end """ -A ServerWS is a variant of HTTP.Server -which aims to hook into some of the higher-level functionality. -It is also possible to hook into the lower-level functionality -using `listen`. +WebSockets.ServerWS is a variant of HTTP.Server which +also includes a WebSockets.WebSocketHandler. +Most or all of the functionality can alternaively be accessed with +'listen' + ``` julia + se = WebSockets.ServerWS(::HTTP.HandlerFunction, WebSockets.WebsocketHandler(gatekeeper)) + ``` """ mutable struct ServerWS{T <: HTTP.Servers.Scheme, H <: HTTP.Handler, W <: WebsocketHandler} handler::H @@ -231,8 +245,6 @@ mutable struct ServerWS{T <: HTTP.Servers.Scheme, H <: HTTP.Handler, W <: Websoc new{T, H, W}(handler, wshandler, logger, ch, ch2, options) end - - ServerWS(h::Function, w::Function, l::IO=HTTP.compat_stdout(); cert::String="", key::String="", args...) = ServerWS(HTTP.HandlerFunction(h), WebsocketHandler(w), l; cert=cert, key=key, args...) @@ -250,7 +262,12 @@ function ServerWS(handler::H, return serverws end - +""" +A variant of HTTP.serve with the WebSockets.ServerWS type. +```julia + @shedule WebSockets.serve(myServerWS, "127.0.0.1", 8080, false) +``` +""" function serve(server::ServerWS{T, H, W}, host, port, verbose) where {T, H, W} tcpserver = Ref{HTTP.Sockets.TCPServer}() diff --git a/src/WebSockets.jl b/src/WebSockets.jl index f69edba..ee84e01 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -16,6 +16,10 @@ includes another Julia session, parallel process or task. Future improvements: 1. Logging of refused requests and closures due to bad behavior of client. 2. Allow users to receive control messages if they want to. +3. Check rsv1 to rsv3 values. This will reduce bandwidth. +4. Optimize maskswitch!, possibly threaded above a certain limit. +5. Split messages over several frames. +6. Allow customizable output (e.g. 'ping'). See HttpServer listen. """ module WebSockets import MbedTLS: digest, MD_SHA1 @@ -362,7 +366,6 @@ function read_frame(ws::WebSocket) rsv2 = a & 0b0010_0000 # If not 0, fail. rsv3 = a & 0b0001_0000 # If not 0, fail. opcode = a & 0b0000_1111 # If not known code, fail. - # TODO: add validation somewhere to ensure rsv, opcode, mask, etc are valid. b = ab[2] mask = b & 0b1000_0000 >>> 7 @@ -376,7 +379,7 @@ function read_frame(ws::WebSocket) end throw(WebSocketError(1002, msg)) end - + * payload_len::UInt64 = b & 0b0111_1111 if payload_len == 126 payload_len = ntoh(read(ws.socket,UInt16)) # 2 bytes diff --git a/test/HTTP.jl b/test/HTTP.jl deleted file mode 100644 index c0afee6..0000000 --- a/test/HTTP.jl +++ /dev/null @@ -1,80 +0,0 @@ -using HTTP -using HttpServer -using WebSockets -using Base.Test - -@testset "HTTP" begin - -info("Testing ws...") -WebSockets.open("ws://echo.websocket.org") do ws - write(ws, "Foo") - @test String(read(ws)) == "Foo" - - close(ws) -end -sleep(1) - -info("Testing wss...") -WebSockets.open("wss://echo.websocket.org") do ws - write(ws, "Foo") - @test String(read(ws)) == "Foo" - - close(ws) -end -sleep(1) - -port_HTTP = 8000 -info("Start HTTP server on port $(port_HTTP)") -@async HTTP.listen("127.0.0.1",UInt16(port_HTTP)) do http - if WebSockets.is_upgrade(http.message) - WebSockets.upgrade(http) do ws - while !eof(ws) - data = String(read(ws)) - write(ws,data) - end - end - end -end - -port_HttpServer = 8081 -info("Start HttpServer on port $(port_HttpServer)") -wsh = WebSocketHandler() do req,ws - while !eof(ws) - msg = String(read(ws)) - write(ws, msg) - end -end -server = Server(wsh) -@async run(server,port_HttpServer) - -sleep(2) - -info("Testing local HTTP server...") -WebSockets.open("ws://127.0.0.1:$(port_HTTP)") do ws - write(ws, "Foo") - @test String(read(ws)) == "Foo" - - write(ws, "Bar") - @test String(read(ws)) == "Bar" - - send_ping(ws) - read(ws) - - close(ws) -end - -info("Testing local HttpServer...") -WebSockets.open("ws://127.0.0.1:$(port_HttpServer)") do ws - write(ws, "Foo") - @test String(read(ws)) == "Foo" - - write(ws, "Bar") - @test String(read(ws)) == "Bar" - - send_ping(ws) - read(ws) - - close(ws) -end - -end # testset \ No newline at end of file diff --git a/test/client_server_test.jl b/test/client_server_test.jl new file mode 100644 index 0000000..a7b973b --- /dev/null +++ b/test/client_server_test.jl @@ -0,0 +1,84 @@ +# included in runtests.jl +using Base.Test +using HTTP +using HttpServer +import WebSockets: is_upgrade, + upgrade +function echows(req, ws) + @test origin(req) == "" + @test target(req) == "/" + @test subprotocol(req) == "" + while true + data, success = readguarded(ws) + !success && break + !writeguarded(ws, data) && break + end +end + +const port_HTTP = 8000 +const port_HTTP_ServeWS = 8001 +const port_HttpServer = 8081 +const TCPREF = Ref{Base.TCPServer}() + +# Start HTTP listen server on port $port_HTTP" +@schedule HTTP.listen("127.0.0.1", port_HTTP, tcpref = TCPREF) do s + if WebSockets.is_upgrade(s.message) + WebSockets.upgrade(echows, s) + end +end + +# Start HttpServer on port $port_HttpServer +const server = Server(HttpHandler((req, res)->Response(200)), + WebSocketHandler(echows)); +@schedule run(server, port_HttpServer) + + +# Start HTTP ServerWS on port $port_HTTP_ServeWS +server_WS = WebSockets.ServerWS( + HTTP.HandlerFunction(req-> HTTP.Response(200)), + WebSockets.WebsocketHandler(echows)) + +@schedule WebSockets.serve(server_WS, "127.0.0.1", port_HTTP_ServeWS, false) + + +sleep(4) +const servers = [ + ("HTTP", "ws://127.0.0.1:$(port_HTTP)"), + ("HttpServer", "ws://127.0.0.1:$(port_HttpServer)"), + ("HTTTP ServerWS", "ws://127.0.0.1:$(port_HTTP_ServeWS)"), + ("ws", "ws://echo.websocket.org"), + ("wss", "wss://echo.websocket.org")] + +const lengths = [0, 125, 126, 127, 2000] + +for (s, url) in servers, len in lengths, closestatus in [false, true] + len == 0 && contains(url, "echo.websocket.org") && continue + info("Testing client -> server at $(url), message length $len") + test_str = randstring(len) + forcecopy_str = test_str |> collect |> copy |> join + WebSockets.open(url) do ws + print(" -Foo-") + write(ws, "Foo") + @test String(read(ws)) == "Foo" + print(" -Ping-") + send_ping(ws) + print(" -String length $len-\n") + write(ws, test_str) + @test String(read(ws)) == forcecopy_str + closestatus && close(ws, statusnumber = 1000) + sleep(0.2) + end + sleep(0.2) +end + + +# make a very simple http request for the servers with defined http handlers +resp = HTTP.request("GET", "http://127.0.0.1:$(port_HTTP_ServeWS)") +@test resp.status == 200 +resp = HTTP.request("GET", "http://127.0.0.1:$(port_HttpServer)") +@test resp.status == 200 + +# Close the servers +close(TCPREF[]) +close(server.http.sock) +put!(server_WS.in, HTTP.Servers.KILL) diff --git a/test/client_test.jl b/test/client_test.jl new file mode 100644 index 0000000..313e679 --- /dev/null +++ b/test/client_test.jl @@ -0,0 +1,77 @@ +# included in runtests.jl +using Base.Test +using HTTP +import WebSockets: is_upgrade, + upgrade, + _openstream, + addsubproto, + generate_websocket_key + +sethd(r::HTTP.Messages.Response, pa::Pair) = HTTP.Messages.setheader(r, HTTP.Header(pa)) + +const NEWPORT = 8091 +const TCPREF2 = Ref{Base.TCPServer}() + + +addsubproto("xml") +@schedule HTTP.listen("127.0.0.1", NEWPORT, tcpref = TCPREF2) do s + if WebSockets.is_upgrade(s.message) + WebSockets.upgrade((_)->nothing, s) + end +end + +# open client with approved subprotocol +const URL = "ws://127.0.0.1:$NEWPORT" +res = WebSockets.open((_)->nothing, URL, subprotocol = "xml"); +@test res.status == 101 +# open with unknown subprotocol +res = WebSockets.open((_)->nothing, URL, subprotocol = "unapproved"); +@test res.status == 400 +# try open with uknown port +caughterr = WebSockets.WebSocketClosedError("") +try +WebSockets.open((_)->nothing, "ws://127.0.0.1:8099"); +catch err + caughterr = err +end +@test typeof(caughterr) <: WebSockets.WebSocketClosedError +@test caughterr.message == " while open ws|client: connect: connection refused (ECONNREFUSED)" + +# start a client websocket that irritates by closing the TCP stream +# connection without a websocket closing handshake. This +# throws an error in the server task +WebSockets.open("ws://127.0.0.1:$(NEWPORT)") do ws + close(ws.socket) +end +# check that the server is still running regardless +res = WebSockets.open((_)->nothing, URL); +@test res.status == 101 +# Open with a ws client handler that throws a domain error +@test_throws DomainError WebSockets.open((_)->sqrt(-2), URL); +# Stop the TCP server +close(TCPREF2[]) + +# emulate a correct first accept response from server +req = HTTP.Messages.Request() +req.method = "GET" +key = base64encode(rand(UInt8, 16)) +resp = HTTP.Response(101) +resp.request = req +sethd(resp, "Sec-WebSocket-Version" => "13") +sethd(resp, "Upgrade" => "websocket") +sethd(resp, "Sec-WebSocket-Accept" => generate_websocket_key(key)) +sethd(resp, "Connection" => "Upgrade") +servsock = BufferStream() +s = HTTP.Streams.Stream(resp, HTTP.Transaction(HTTP.Connection(servsock))) +write(servsock, resp) +function dummywsh(dws::WebSockets.WebSocket{BufferStream}) + close(dws.socket) + close(dws) +end + +@test _openstream(dummywsh, s, key) == WebSockets.CLOSED + +# emulate an incorrect first accept response from server +sethd(resp, "Sec-WebSocket-Accept" => generate_websocket_key(base64encode(rand(UInt8, 16)))) +write(servsock, resp) +@test_throws WebSockets.WebSocketError _openstream(dummywsh, s, key) diff --git a/test/error_test.jl b/test/error_test.jl new file mode 100644 index 0000000..47e1909 --- /dev/null +++ b/test/error_test.jl @@ -0,0 +1,56 @@ +# included in runtests.jl + +# As per June 2018, there is no +# way we can process errors thrown +# by tasks generated by HttpServer +# or by HTTP, for example ECONNRESET. +# +# Try to provide user with relevant and +# concise output. +using Base.Test +import HTTP +import WebSockets: ServerWS, + serve, + open + +const THISPORT = 8092 +info("Start a ws|server which doesn't read much at all. Close from client side. The + close handshake is aborted after $(WebSockets.TIMEOUT_CLOSEHANDSHAKE) seconds...") +server_WS = WebSockets.ServerWS( + HTTP.HandlerFunction(req-> HTTP.Response(200)), + WebSockets.WebsocketHandler(ws-> sleep(16))) +@schedule WebSockets.serve(server_WS, "127.0.0.1", THISPORT, false) +URL = "ws://127.0.0.1:$THISPORT" +res = WebSockets.open((_)->nothing, URL); +@test res.status == 101 +put!(server_WS.in, HTTP.Servers.KILL) + +#= +Activate test when there is a decent way to capture errors +in tasks generated by server package + +server_WS = WebSockets.ServerWS(HTTP.HandlerFunction(req-> HTTP.Response(200)), + WebSockets.WebsocketHandler() do ws_serv + while isopen(ws_serv) + readguarded(ws_serv) + end + end + ) +@schedule WebSockets.serve(server_WS, "127.0.0.1", THISPORT, false) +# attempt to read from closed websocket. +WebSockets.open(URL) do ws_client + close(ws_client) + try + read(ws_client) + catch + end + end; + +function wsh(ws) + close(ws) + read(ws) +end +# improve output of error, the details don't emerge +WebSockets.open(wsh, URL) + +=# diff --git a/test/frametest.jl b/test/frametest.jl new file mode 100644 index 0000000..bfeabab --- /dev/null +++ b/test/frametest.jl @@ -0,0 +1,139 @@ +# included in runtests.jl +using Base.Test +import WebSockets: maskswitch!, + write_fragment, + read_frame, + WebSocket + +""" +The dummy websocket don't use TCP. Close won't work, but we can manipulate the contents +using otherwise the same functions as TCP sockets. +""" +dummyws(server::Bool) = WebSocket(BufferStream(), server) + + +io = IOBuffer() +# Test most basic frame, length <126 + +for len = [8, 125], fin=[true, false], clientwriting = [false, true] + + op = (rand(UInt8) & 0b1111) + test_str = randstring(len) + # maskswitch two times with same key == unmasked + maskunmask = copy(Vector{UInt8}(test_str)) + mskkey = maskswitch!(maskunmask) + maskswitch!(maskunmask, mskkey) + @test maskunmask == Vector{UInt8}(test_str) + + # websocket fragment as Vector{UInt8} + # for client writing, the data is masked and the mask is contained in the frame. + # for server writing, the data is not masked, and the header is four bytes shorter. + write_fragment(io, fin, op, clientwriting, copy(Vector{UInt8}(test_str))) + # test that the original input string was not masked. + @test maskunmask == Vector{UInt8}(test_str) + frame = take!(io) + # Check the frame header + # Last frame bit + @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] + # payload length bit + @test frame[2] & 0b0111_1111 == len + # ismasked bit + hasmsk = frame[2] & 0b1000_0000 >>> 7 != 0 + @test hasmsk == clientwriting + # payload data + if hasmsk + framedata = copy(frame[7:end]) + maskswitch!(framedata, frame[3:6]) + else + framedata = frame[3:end] + end + + @test framedata == Vector{UInt8}(test_str) + + # Test for WebSocketError when reading + # masked frame-> websocket|server + # unmasked frame -> websocket|client + + # Let's pretend TCP has moved our frame into the peer websocket + receivingws = dummyws(!clientwriting) + write(receivingws.socket, frame) + @test_throws WebSockets.WebSocketError read_frame(receivingws) + close(receivingws.socket) + + # Let's pretend receivingws didn't error like it should, but + # echoed our message back with identical masking. + dws = dummyws(clientwriting) + @test dws.server == clientwriting + write(dws.socket, frame) + # read the frame back, now represented as a WebSocketFragment + + frag_back = read_frame(dws) + close(dws.socket) + @test frag_back.is_last == fin + @test frag_back.rsv1 == false + @test frag_back.rsv2 == false + @test frag_back.rsv3 == false + @test frag_back.opcode == op + @test frag_back.is_masked == clientwriting + @test frag_back.payload_len == len + maskkey = UInt8[] + if clientwriting + maskkey = frame[3:6] + end + @test frag_back.maskkey == maskkey + # the WebSocketFragment stores the data after unmasking + @test Vector{UInt8}(test_str) == frag_back.data +end + +# Test length 126 or more + +for len = 126:129, fin=[true, false], clientwriting = [false, true] + op = 0b1111 + test_str = randstring(len) + write_fragment(io, fin, op, clientwriting, copy(Vector{UInt8}(test_str))) + frame = take!(io) + + @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] + @test frame[2] & 0b0111_1111 == 126 + @test bits(frame[4])*bits(frame[3]) == bits(hton(UInt16(len))) + + dws = dummyws(clientwriting) + write(dws.socket, frame) + frag_back = read_frame(dws) + close(dws.socket) + + @test frag_back.is_last == fin + @test frag_back.rsv1 == false + @test frag_back.rsv2 == false + @test frag_back.rsv3 == false + @test frag_back.opcode == op + @test frag_back.is_masked == clientwriting + @test frag_back.payload_len == len + @test test_str == String(frag_back.data) +end + +# Test length typemax(UInt16) + 1 + +for clientwriting = [false, true] + len = typemax(UInt16) +1 + op = 0b1111 + fin = true + + test_str = randstring(len) + write_fragment(io, fin, op, clientwriting, copy(Vector{UInt8}(test_str))) + frame = take!(io) + + dws = dummyws(clientwriting) + write(dws.socket, frame) + frag_back = read_frame(dws) + close(dws.socket) + + @test frag_back.is_last == fin + @test frag_back.rsv1 == false + @test frag_back.rsv2 == false + @test frag_back.rsv3 == false + @test frag_back.opcode == op + @test frag_back.is_masked == clientwriting + @test frag_back.payload_len == len + @test test_str == String(frag_back.data) +end diff --git a/test/handshaketest.jl b/test/handshaketest.jl new file mode 100644 index 0000000..ca2872a --- /dev/null +++ b/test/handshaketest.jl @@ -0,0 +1,131 @@ +# included in runtests.jl +using Base.Test +using HTTP +using HttpServer +using WebSockets +import WebSockets: generate_websocket_key, + websocket_handshake, + upgrade +import HttpCommon:Request +import HttpServer: is_websocket_handshake + + +function templaterequests() + chromeheaders = Dict{String, String}( "Connection"=>"Upgrade", + "Upgrade"=>"websocket") + firefoxheaders = Dict{String, String}("Connection"=>"keep-alive, Upgrade", + "Upgrade"=>"websocket") + chromerequest = HttpCommon.Request("GET", "", chromeheaders, "") + firefoxrequest= Request("GET", "", firefoxheaders, "") + chromerequest_HTTP = HTTP.Messages.Request("GET", "/", collect(chromeheaders)) + firefoxrequest_HTTP = HTTP.Messages.Request("GET", "/", collect(firefoxheaders)) + return [chromerequest, firefoxrequest, chromerequest_HTTP, firefoxrequest_HTTP] +end + +sethd(r::Request, pa::Pair) = push!(r.headers, pa) +sethd(r::HTTP.Messages.Request, pa::Pair) = HTTP.Messages.setheader(r, HTTP.Header(pa)) + +takefirstline(buf::IOBuffer) = strip(split(buf |> take! |> String, "\r\n")[1]) +takefirstline(buf::BufferStream) = strip(split(buf |> read |> String, "\r\n")[1]) +function handshakeresponse(request::Request) + cli = HttpServer.Client(2, IOBuffer()) + websocket_handshake(request, cli) + strip(takefirstline(cli.sock)) +end +function handshakeresponse(request::HTTP.Messages.Request) + buf = BufferStream() + c = HTTP.ConnectionPool.Connection(buf) + t = HTTP.Transaction(c) + s = HTTP.Streams.Stream(request, t) + upgrade(dummywshandler, s) + close(buf) + takefirstline(buf) +end + +""" +The dummy websocket don't use TCP. Close won't work, but we can manipulate the contents +using otherwise the same functions as TCP sockets. +""" +dummyws(server::Bool) = WebSocket(BufferStream(), server) + +function dummywshandler(req, dws::WebSockets.WebSocket{BufferStream}) + close(dws.socket) + close(dws) +end + +# Test generate_websocket_key" +@test generate_websocket_key("dGhlIHNhbXBsZSBub25jZQ==") == "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" + + +# Test reject / switch format" +io = IOBuffer() +const REJECT = "HTTP/1.1 400 Bad Request" +Base.write(io, Response(400)) +@test takefirstline(io) == REJECT +Base.write(io, HTTP.Messages.Response(400)) +@test takefirstline(io) == REJECT + +const SWITCH = "HTTP/1.1 101 Switching Protocols" +Base.write(io, Response(101)) +@test takefirstline(io) == SWITCH +Base.write(io, HTTP.Messages.Response(101)) +@test takefirstline(io) == SWITCH + + +chromerequest, firefoxrequest, chromerequest_HTTP, firefoxrequest_HTTP = Tuple(templaterequests()) +wshandler = WebSocketHandler((x,y)->nothing) +for request in [chromerequest, firefoxrequest] + @test is_websocket_handshake(wshandler, request) == true +end + + +# "Test simple handshakes that are unacceptable" + +for r in templaterequests() + @test handshakeresponse(r) == REJECT + sethd(r, "Sec-WebSocket-Version" => "11") + @test handshakeresponse(r) == REJECT + sethd(r, "Sec-WebSocket-Version" => "13") + @test handshakeresponse(r) == REJECT + sethd(r, "Sec-WebSocket-Key" => "mumbojumbobo") + @test handshakeresponse(r) == REJECT + sethd(r, "Sec-WebSocket-Key" => "17 bytes key is not accepted") + @test handshakeresponse(r) == REJECT + sethd(r, "Sec-WebSocket-Key" => "16 bytes key this is surely") + sethd(r, "Sec-WebSocket-Protocol" => "unsupported") + @test handshakeresponse(r) == REJECT +end + + +# Test simple handshakes, acceptable +for r in templaterequests() + sethd(r, "Sec-WebSocket-Version" => "13") + sethd(r, "Sec-WebSocket-Key" => "16 bytes key this is surely") + @test handshakeresponse(r) == SWITCH +end + + + +# Test unacceptable subprotocol handshake subprotocol +for r in templaterequests() + sethd(r, "Sec-WebSocket-Version" => "13") + sethd(r, "Sec-WebSocket-Key" => "16 bytes key this is surely") + sethd(r, "Sec-WebSocket-Protocol" => "my.server/json-zmq") + @test handshakeresponse(r) == REJECT +end + +info("add simple subprotocol to acceptable list") +@test true == WebSockets.addsubproto("xml") + +info("add subprotocol with difficult name") +@test true == WebSockets.addsubproto("my.server/json-zmq") + +info("Test handshake subprotocol now acceptable") +for r in templaterequests() + sethd(r, "Sec-WebSocket-Version" => "13") + sethd(r, "Sec-WebSocket-Key" => "16 bytes key this is surely") + sethd(r, "Sec-WebSocket-Protocol" => "xml") + @test handshakeresponse(r) == SWITCH + sethd(r, "Sec-WebSocket-Protocol" => "my.server/json-zmq") + @test handshakeresponse(r) == SWITCH +end diff --git a/test/runtests.jl b/test/runtests.jl index b643df5..cf99bad 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,405 +1,36 @@ -using HTTP -using HttpServer -using WebSockets using Base.Test -import WebSockets: generate_websocket_key, - write_fragment, - read_frame, - websocket_handshake, - maskswitch!, - is_upgrade, - upgrade -import HttpServer: is_websocket_handshake, - handle -import HttpCommon: Request -@sync yield() # avoid mixing of output with possible deprecation warnings from .juliarc -info("Starting test WebSockets...") -#is_control_frame is one line, checking one bit. -#get_websocket_key grabs a header. -#is_websocket_handshake grabs a header. -#generate_websocket_key makes a call to a library. -info("Test generate_websocket_key") -@test generate_websocket_key("dGhlIHNhbXBsZSBub25jZQ==") == "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" - -""" -The dummy websocket don't use TCP. Close won't work, but we can manipulate the contents -using otherwise the same functions as TCP sockets. -""" -dummyws(server::Bool) = WebSocket(BufferStream(), server) - -function dummywshandler(req, dws::WebSockets.WebSocket{BufferStream}) - close(dws.socket) - close(dws) -end - - -const io = IOBuffer() - - -info("Test length less than 126") -@testset "Unit test, fragment length less than 126" begin - -for len = [8, 125], fin=[true, false], clientwriting = [false, true] - - op = (rand(UInt8) & 0b1111) - test_str = randstring(len) - # maskswitch two times with same key == unmasked - maskunmask = copy(Vector{UInt8}(test_str)) - mskkey = maskswitch!(maskunmask) - maskswitch!(maskunmask, mskkey) - @test maskunmask == Vector{UInt8}(test_str) - - # websocket fragment as Vector{UInt8} - # for client writing, the data is masked and the mask is contained in the frame. - # for server writing, the data is not masked, and the header is four bytes shorter. - write_fragment(io, fin, op, clientwriting, copy(Vector{UInt8}(test_str))) - # test that the original input string was not masked. - @test maskunmask == Vector{UInt8}(test_str) - frame = take!(io) - # Check the frame header - # Last frame bit - @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] - # payload length bit - @test frame[2] & 0b0111_1111 == len - # ismasked bit - hasmsk = frame[2] & 0b1000_0000 >>> 7 != 0 - @test hasmsk == clientwriting - # payload data - if hasmsk - framedata = copy(frame[7:end]) - maskswitch!(framedata, frame[3:6]) - else - framedata = frame[3:end] - end - - @test framedata == Vector{UInt8}(test_str) - - # Test for WebSocketError when reading - # masked frame-> websocket|server - # unmasked frame -> websocket|client - - # Let's pretend TCP has moved our frame into the peer websocket - receivingws = dummyws(!clientwriting) - write(receivingws.socket, frame) - @test_throws WebSockets.WebSocketError read_frame(receivingws) - close(receivingws.socket) - - # Let's pretend receivingws didn't error like it should, but - # echoed our message back with identical masking. - dws = dummyws(clientwriting) - @test dws.server == clientwriting - write(dws.socket, frame) - # read the frame back, now represented as a WebSocketFragment - - frag_back = read_frame(dws) - close(dws.socket) - @test frag_back.is_last == fin - @test frag_back.rsv1 == false - @test frag_back.rsv2 == false - @test frag_back.rsv3 == false - @test frag_back.opcode == op - @test frag_back.is_masked == clientwriting - @test frag_back.payload_len == len - maskkey = UInt8[] - if clientwriting - maskkey = frame[3:6] - end - @test frag_back.maskkey == maskkey - # the WebSocketFragment stores the data after unmasking - @test Vector{UInt8}(test_str) == frag_back.data -end -end # testset - - -info("Test length 126 or more") -@testset "Unit test, fragment length 126 or more" begin - -for len = 126:129, fin=[true, false], clientwriting = [false, true] - op = 0b1111 - test_str = randstring(len) - write_fragment(io, fin, op, clientwriting, copy(Vector{UInt8}(test_str))) - frame = take!(io) - - @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] - @test frame[2] & 0b0111_1111 == 126 - @test bits(frame[4])*bits(frame[3]) == bits(hton(UInt16(len))) - - dws = dummyws(clientwriting) - write(dws.socket, frame) - frag_back = read_frame(dws) - close(dws.socket) - - @test frag_back.is_last == fin - @test frag_back.rsv1 == false - @test frag_back.rsv2 == false - @test frag_back.rsv3 == false - @test frag_back.opcode == op - @test frag_back.is_masked == clientwriting - @test frag_back.payload_len == len - @test test_str == String(frag_back.data) -end - -end # testset -info("Test length typemax(UInt16) + 1") -@testset "Test length typemax(UInt16) + 1" begin - -for clientwriting = [false, true] - len = typemax(UInt16) +1 - op = 0b1111 - fin = true - - test_str = randstring(len) - write_fragment(io, fin, op, clientwriting, copy(Vector{UInt8}(test_str))) - frame = take!(io) - - dws = dummyws(clientwriting) - write(dws.socket, frame) - frag_back = read_frame(dws) - close(dws.socket) - - @test frag_back.is_last == fin - @test frag_back.rsv1 == false - @test frag_back.rsv2 == false - @test frag_back.rsv3 == false - @test frag_back.opcode == op - @test frag_back.is_masked == clientwriting - @test frag_back.payload_len == len - @test test_str == String(frag_back.data) +cd(Pkg.dir("WebSockets", "test")) +# WebSockets.jl +@testset "Fragment and frame unit tests" begin + include("frametest.jl") end -end # testset - +@sync yield() # avoid mixing of output with possible deprecation warnings from .juliarc +info("Starting test WebSockets...") @testset "Unit test, HttpServer and HTTP handshake" begin - -info("Tests for is_websocket_handshake") -function templaterequests() - chromeheaders = Dict{String, String}( "Connection"=>"Upgrade", - "Upgrade"=>"websocket") - firefoxheaders = Dict{String, String}("Connection"=>"keep-alive, Upgrade", - "Upgrade"=>"websocket") - - chromerequest = HttpCommon.Request("GET", "", chromeheaders, "") - firefoxrequest= Request("GET", "", firefoxheaders, "") - - chromerequest_HTTP = HTTP.Messages.Request("GET", "/", collect(chromeheaders)) - firefoxrequest_HTTP = HTTP.Messages.Request("GET", "/", collect(firefoxheaders)) - - return [chromerequest, firefoxrequest, chromerequest_HTTP, firefoxrequest_HTTP] + include("handshaketest.jl") end -chromerequest, firefoxrequest, chromerequest_HTTP, firefoxrequest_HTTP = Tuple(templaterequests()) - -wshandler = WebSocketHandler((x,y)->nothing) - -for request in [chromerequest, firefoxrequest] - @test is_websocket_handshake(wshandler, request) == true +@testset "Client-server test, HTTP client" begin + include("client_server_test.jl") end - -for request in [chromerequest_HTTP, firefoxrequest_HTTP] - @test is_upgrade(request) == true +@testset "Client test, HTTP client" begin + include("client_test.jl") end -info("Test of handshake response") -takefirstline(buf::IOBuffer) = strip(split(buf |> take! |> String, "\r\n")[1]) -takefirstline(buf::BufferStream) = strip(split(buf |> read |> String, "\r\n")[1]) -take!(io) -info("Test reject / switch format") -const REJECT = "HTTP/1.1 400 Bad Request" -Base.write(io, Response(400)) -@test takefirstline(io) == REJECT -Base.write(io, HTTP.Messages.Response(400)) -@test takefirstline(io) == REJECT - -const SWITCH = "HTTP/1.1 101 Switching Protocols" -Base.write(io, Response(101)) -@test takefirstline(io) == SWITCH -Base.write(io, HTTP.Messages.Response(101)) -@test takefirstline(io) == SWITCH - - -function handshakeresponse(request::Request) - cli = HttpServer.Client(2, IOBuffer()) - websocket_handshake(request, cli) - strip(takefirstline(cli.sock)) +@testset "WebSockets abrupt close & bad timing test" begin + include("error_test.jl") end -function handshakeresponse(request::HTTP.Messages.Request) - buf = BufferStream() - c = HTTP.ConnectionPool.Connection(buf) - t = HTTP.Transaction(c) - s = HTTP.Streams.Stream(request, t) - upgrade(dummywshandler, s) - close(buf) - takefirstline(buf) -end - -info("Test simple handshakes that are unacceptable") - -sethd(r::Request, pa::Pair) = push!(r.headers, pa) -sethd(r::HTTP.Messages.Request, pa::Pair) = HTTP.Messages.setheader(r, HTTP.Header(pa)) - -for r in templaterequests() - @test handshakeresponse(r) == REJECT - sethd(r, "Sec-WebSocket-Version" => "11") - @test handshakeresponse(r) == REJECT - sethd(r, "Sec-WebSocket-Version" => "13") - @test handshakeresponse(r) == REJECT - sethd(r, "Sec-WebSocket-Key" => "mumbojumbobo") - @test handshakeresponse(r) == REJECT - sethd(r, "Sec-WebSocket-Key" => "17 bytes key is not accepted") - @test handshakeresponse(r) == REJECT - sethd(r, "Sec-WebSocket-Key" => "16 bytes key this is surely") - sethd(r, "Sec-WebSocket-Protocol" => "unsupported") - @test handshakeresponse(r) == REJECT -end - - - - - -info("Test simple handshakes, acceptable") -for r in templaterequests() - sethd(r, "Sec-WebSocket-Version" => "13") - sethd(r, "Sec-WebSocket-Key" => "16 bytes key this is surely") - @test handshakeresponse(r) == SWITCH -end - - - -info("Test unacceptable subprotocol handshake subprotocol") -for r in templaterequests() - sethd(r, "Sec-WebSocket-Version" => "13") - sethd(r, "Sec-WebSocket-Key" => "16 bytes key this is surely") - sethd(r, "Sec-WebSocket-Protocol" => "my.server/json-zmq") - @test handshakeresponse(r) == REJECT -end - -info("add simple subprotocol to acceptable list") -@test true == WebSockets.addsubproto("xml") - -info("add subprotocol with difficult name") -@test true == WebSockets.addsubproto("my.server/json-zmq") - -info("Test handshake subprotocol now acceptable") -for r in templaterequests() - sethd(r, "Sec-WebSocket-Version" => "13") - sethd(r, "Sec-WebSocket-Key" => "16 bytes key this is surely") - sethd(r, "Sec-WebSocket-Protocol" => "xml") - @test handshakeresponse(r) == SWITCH - sethd(r, "Sec-WebSocket-Protocol" => "my.server/json-zmq") - @test handshakeresponse(r) == SWITCH -end -end # testset - -close(io) - -@testset "Peer-to-peer tests, HTTP client" begin - -const port_HTTP = 8000 -const port_HTTP_ServeWS = 8001 -const port_HttpServer = 8081 - - -function echows(ws) - while true - data, success = readguarded(ws) - !success && break - !writeguarded(ws, data) && break - end -end - -function echows(req, ws) - @test origin(req) == "" - @test target(req) == "/" - @test subprotocol(req) == "" - while true - data, success = readguarded(ws) - !success && break - !writeguarded(ws, data) && break - end -end - -info("Start HTTP listen server on port $port_HTTP") -@async HTTP.listen("127.0.0.1", UInt16(port_HTTP)) do s - if WebSockets.is_upgrade(s.message) - WebSockets.upgrade(echows, s) - end -end - - -info("Start HttpServer on port $port_HttpServer") -server = Server(WebSocketHandler(echows)) -@async run(server, port_HttpServer) - - -info("Start HTTP ServerWS on port $port_HTTP_ServeWS") -server_WS = WebSockets.ServerWS( - HTTP.HandlerFunction(req-> HTTP.Response(200)), - WebSockets.WebsocketHandler(echows)) - -@async WebSockets.serve(server_WS, "127.0.0.1", port_HTTP_ServeWS, false) - - -sleep(4) - -servers = [ - ("HTTP", "ws://127.0.0.1:$(port_HTTP)"), - ("HttpServer", "ws://127.0.0.1:$(port_HttpServer)"), - ("HTTTP ServerWS", "ws://127.0.0.1:$(port_HTTP_ServeWS)"), - ("ws", "ws://echo.websocket.org"), - ("wss", "wss://echo.websocket.org")] - -lengths = [0, 3, 125, 126, 127, 2000] - -for (s, url) in servers, len in lengths, closestatus in [false, true] - len == 0 && contains(url, "echo.websocket.org") && continue - info("Testing client -> server at $(url), message length $len") - test_str = randstring(len) - forcecopy_str = test_str |> collect |> copy |> join - WebSockets.open(url) do ws - print(" -Foo-") - write(ws, "Foo") - @test String(read(ws)) == "Foo" - print(" -Ping-") - send_ping(ws) - print(" -String length $len-\n") - write(ws, test_str) - @test String(read(ws)) == forcecopy_str - closestatus && close(ws, statusnumber = 1000) - sleep(0.2) - end - sleep(0.2) -end - -end # testset - -# TODO missing tests before browsertests. +# TODO # WebSockets.jl -# provoke errors WebSocketClosedError -# error("Attempted to send too much data for one websocket fragment\n") # direct closing of tcp socket, while reading. # closing with given reason (only from browsertests) # unknown opcode -# Attempt to read from closed # Read multiple frames (use dummyws), may require change # InterruptException # Protocol error (not masked from client) # writeguarded, error - - -# HTTP -# open with optionalprotocol (change to subprotocol) -# open with rejected Protocol -# open bad reply to key during handshake (writeframe, dummyws) -# improve error handling in HTTP.open (may require change) -# HTTP messages that are not upgrades -# ugrade with single argument function -# ServeWS with https -# stop ServeWS with InterruptException -# listen with http request - -# HttpServer -# exit without closing websocket -# is_websocket_handshake with normal request +# restructure browsertests From f8f9303906138a5f1bb541528ac6c045f737d8a1 Mon Sep 17 00:00:00 2001 From: hustf Date: Sat, 2 Jun 2018 11:12:29 +0200 Subject: [PATCH 30/35] modified: src/WebSockets.jl dead code remove --- src/WebSockets.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WebSockets.jl b/src/WebSockets.jl index ee84e01..746f4cd 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -379,7 +379,7 @@ function read_frame(ws::WebSocket) end throw(WebSocketError(1002, msg)) end - * + payload_len::UInt64 = b & 0b0111_1111 if payload_len == 126 payload_len = ntoh(read(ws.socket,UInt16)) # 2 bytes From da008ce8f1ad398de47ce30d6796119d17c1f2e9 Mon Sep 17 00:00:00 2001 From: hustf Date: Sat, 2 Jun 2018 11:41:59 +0200 Subject: [PATCH 31/35] Add sleep(3) after start async server for travis-test modified: test/client_test.jl modified: test/error_test.jl --- test/client_test.jl | 2 +- test/error_test.jl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/client_test.jl b/test/client_test.jl index 313e679..d4232d3 100644 --- a/test/client_test.jl +++ b/test/client_test.jl @@ -19,7 +19,7 @@ addsubproto("xml") WebSockets.upgrade((_)->nothing, s) end end - +sleep(3) # open client with approved subprotocol const URL = "ws://127.0.0.1:$NEWPORT" res = WebSockets.open((_)->nothing, URL, subprotocol = "xml"); diff --git a/test/error_test.jl b/test/error_test.jl index 47e1909..6c2f0c3 100644 --- a/test/error_test.jl +++ b/test/error_test.jl @@ -20,6 +20,7 @@ server_WS = WebSockets.ServerWS( HTTP.HandlerFunction(req-> HTTP.Response(200)), WebSockets.WebsocketHandler(ws-> sleep(16))) @schedule WebSockets.serve(server_WS, "127.0.0.1", THISPORT, false) +sleep(3) URL = "ws://127.0.0.1:$THISPORT" res = WebSockets.open((_)->nothing, URL); @test res.status == 101 From b510f88f5cc3ab5430bdbb0e624c66994c583fdd Mon Sep 17 00:00:00 2001 From: hustf Date: Sun, 3 Jun 2018 13:38:01 +0200 Subject: [PATCH 32/35] Server channels for exceptions and traces More tests mod: src/WebSockets.jl export WebSocketClosedError close keyword argument 'freereason' received close handshake throws WebSocketClosedError improve criterion for bad mask error include reasons in forced close handshakes mod: src/HTTP.jl use channels for error handling on server side removed invisible exception warnings in 'listen' mod: benchmark/logs/benchmark_results_readable.log add: benchmark/logs/benchmark_results_readable_previous.log 12% slower client side, 10% faster server side mod: benchmark/ws_hts.jl mod: benchmark/ws_jce.jl import clog_notime mod: logutils/log_http.jl mod: logutils/log_httpserver.jl mod: logutils/log_ws.jl mod: test/handler_functions_events.jl remove dead code, tabs mod: test/client_server_test.jl wait for istaskstarted mod: test/client_test.jl less console info, more sleep mod: test/error_test.jl test all exceptions and exception interfaces mod: test/frametest.jl add unknown opcodes, multi-frame message, bad mask errors, peerless close, close frames mod: test/handshaketest.jl reduce console info mod: test/runtests.jl add sleep and total group --- benchmark/logs/benchmark_results_readable.log | 1726 ++++++++--------- .../benchmark_results_readable_previous.log | 968 +++++++++ benchmark/ws_hts.jl | 2 +- benchmark/ws_jce.jl | 2 +- logutils/log_http.jl | 30 +- logutils/log_httpserver.jl | 94 +- logutils/log_ws.jl | 24 +- src/HTTP.jl | 57 +- src/WebSockets.jl | 35 +- test/client_server_test.jl | 12 +- test/client_test.jl | 16 +- test/error_test.jl | 229 ++- test/frametest.jl | 212 +- test/handler_functions_events.jl | 1 - test/handshaketest.jl | 6 +- test/runtests.jl | 41 +- 16 files changed, 2402 insertions(+), 1053 deletions(-) create mode 100644 benchmark/logs/benchmark_results_readable_previous.log diff --git a/benchmark/logs/benchmark_results_readable.log b/benchmark/logs/benchmark_results_readable.log index 513417f..5fcf32d 100644 --- a/benchmark/logs/benchmark_results_readable.log +++ b/benchmark/logs/benchmark_results_readable.log @@ -1,968 +1,968 @@ HTS_BCE iexplore Varying message size: - bestserverbandwidth = 2.843 [ns/b] = [s/GB] @ size = 1357620 b - bestclientbandwidth = 3.0215 [ns/b] = [s/GB] @ size = 220320 b - bestserverlatency = 142514 [ns] @ size = 1020 b - bestclientlatency = 19971 [ns] @ size = 1020 b + bestserverbandwidth = 2.9396 [ns/b] = [s/GB] @ size = 1762560 b + bestclientbandwidth = 2.9335 [ns/b] = [s/GB] @ size = 349860 b + bestserverlatency = 111936 [ns] @ size = 8160 b + bestclientlatency = 21268 [ns] @ size = 1020 b HTS_BCE chrome Varying message size: - bestserverbandwidth = 5.2447 [ns/b] = [s/GB] @ size = 220320 b - bestclientbandwidth = 4.4889 [ns/b] = [s/GB] @ size = 27540 b - bestserverlatency = 192733 [ns] @ size = 1020 b - bestclientlatency = 32490 [ns] @ size = 1020 b + bestserverbandwidth = 6.1808 [ns/b] = [s/GB] @ size = 1357620 b + bestclientbandwidth = 4.4656 [ns/b] = [s/GB] @ size = 27540 b + bestserverlatency = 84456 [ns] @ size = 1020 b + bestclientlatency = 21596 [ns] @ size = 1020 b HTS_BCE phantomjs Varying message size: - bestserverbandwidth = 7.994 [ns/b] = [s/GB] @ size = 220320 b - bestclientbandwidth = 1.6196 [ns/b] = [s/GB] @ size = 27540 b - bestserverlatency = 266542 [ns] @ size = 8160 b - bestclientlatency = 2883 [ns] @ size = 1020 b + bestserverbandwidth = 8.5377 [ns/b] = [s/GB] @ size = 1357620 b + bestclientbandwidth = 1.759 [ns/b] = [s/GB] @ size = 8160 b + bestserverlatency = 135242 [ns] @ size = 1020 b + bestclientlatency = 2516 [ns] @ size = 1020 b HTS_BCE firefox Varying message size: - bestserverbandwidth = 6.1446 [ns/b] = [s/GB] @ size = 127500 b - bestclientbandwidth = 4.8754 [ns/b] = [s/GB] @ size = 127500 b - bestserverlatency = 435462 [ns] @ size = 1020 b - bestclientlatency = 35413 [ns] @ size = 1020 b + bestserverbandwidth = 6.5741 [ns/b] = [s/GB] @ size = 127500 b + bestclientbandwidth = 5.0461 [ns/b] = [s/GB] @ size = 65280 b + bestserverlatency = 370618 [ns] @ size = 8160 b + bestclientlatency = 34604 [ns] @ size = 1020 b HTS_JCE Varying message size: - bestserverbandwidth = 1.6133 [ns/b] = [s/GB] @ size = 220320 b - bestclientbandwidth = 4.5496 [ns/b] = [s/GB] @ size = 127500 b - bestserverlatency = 61228 [ns] @ size = 1020 b - bestclientlatency = 56592 [ns] @ size = 1020 b + bestserverbandwidth = 1.7232 [ns/b] = [s/GB] @ size = 220320 b + bestclientbandwidth = 4.0568 [ns/b] = [s/GB] @ size = 220320 b + bestserverlatency = 86726 [ns] @ size = 8160 b + bestclientlatency = 66992 [ns] @ size = 1020 b Benchmark Plots of all samples :init_plots [ns/b], message size 130560 b 200 samples - HTS_BCE iexplore serverspeeds - ┌─────────────────────────────────────────────────────────────────┐ - 20 │ │ - │ │ - │ , │ - │ | | │ - │ | .|| │ - │ | ||| │ - │ ] ||| │ - │ O ||| │ - │ N [|| │ - │ .Nl N|\ , │ - │ ||^.N||, . . 1 , .. , |, . │ - │\ ./"`]|/l[.N [ ..."^k| ||.[ |M| |.,| .._.|\ .v J\ │ - │lKv` \M"A^.|K" `\\ |KG^./"/_kV||/. ,... ,\''\/|.//'P`"\k||│ - │ ` "` """ " ` `" "`"`"/"/""`" """ '''│ - 0 │ │ - └─────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE iexplore clientspeeds - ┌─────────────────────────────────────────────────────────────────┐ - 40 │ │ - │ │ - │ | │ - │ | | │ - │ | | │ - │ [ | │ - │ [ | │ - │ [ ] │ - │ [ ] │ - │ [ /. │ - │ [ || │ - │ , [ || │ - │ || N || │ - │._..^/.^/l__,..._.___/_...._.___.__.N... \... .L_runl._.v.|||...│ - 0 │ "`"` ' ''`" " ' "`"`""""``""" `" ` '"'"'│ - └─────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE chrome serverspeeds - ┌─────────────────────────────────────────────────────────────────┐ - 30 │ │ - │ │ - │ │ - │ | │ - │ | │ - │ | . │ - │ | | │ - │ k | │ - │ .N [ │ - │ |N M │ - │G, ||N . N │ - │]L ./| \. l / , N. _ .. |. _ │ - │//F"/ \/| .. O .`"../, rK\.\r_|L`\,r"V, .v/.. ._,.\"^\Y\r.\./`W│ - │ `'""`""""" '`""` '` \ '` '\" ""``'` "" ' │ - 0 │ │ - └─────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE chrome clientspeeds - ┌─────────────────────────────────────────────────────────────────┐ - 40 │ │ - │ | | , │ - │ | | | │ - │ | || | │ - │ | | || | │ - │ | |. || l │ - │ | || || ] │ - │ .| || || ] │ - │ || || || ] │ - │ , |. || || N │ - │ l || || || N │ - │. ] || || |. N │ - │|.O.u,LN| || , . . ||. . , N. ,,│ - │'"""'`''"`^^^/--/``"--^`-/""-""""`//f\/^---\-/--^/`fK"/"""/--`/""│ - 0 │ │ - └─────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE phantomjs serverspeeds - ┌─────────────────────────────────────────────────────────────────┐ - 30 │ │ - │ │ - │ . │ - │ | │ - │ | │ - │ l │ - │|, ] │ - │|| ] , │ - │/| N, | . l , │ - │/|\ NA J /. . ] , \ , [d│ - │ /l N| N./\,, ,. .. \../.\ .vJ| , l. /, l. /, ||]│ - │ r`"u|v` H\^/\\.v\^/`^`"K`'^`v"v^'"'`/\//\-``JJ/v\/^"YK`ln\^fl/│ - │ │ - │ │ - 0 │ │ - └─────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE phantomjs clientspeeds - ┌─────────────────────────────────────────────────────────────────┐ - 40 │ │ - │ │ - │ │ - │ | │ - │ | │ - │ | │ - │ | │ - │ | │ - │ | │ - │ || │ - │ || │ - │ || │ - │ || │ - │. ..\, . . . || . .. . │ - 0 │"""`"""""'""""/"""""""\/"""\`//"'""""""""""""'`^--/`"""""`"""""\-│ - └─────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE firefox serverspeeds - ┌─────────────────────────────────────────────────────────────────┐ - 50 │ │ - │ │ - │ , | │ - │ , , | | | │ - │.|, | | |. | │ - │|N| | ] , || |, │ - │|N| l ] .] || ||, │ - │|N| O /, d],||,.|| | │ - │|N| N || ]][|||||] [ || │ - │|N\.N || , ]][/|O||/. @, @| │ - │|/||| |[ ].]][||N|||| . @| M\ │ - │|F\/| |^,N|@/\|MN|.|| | |l|| , .,|| │ - │ |`|] |'``'|l`||`\`^\_kf./n./__^^.,u../|1\ . 1. A .\J lKd,│ - │ ' "'''` "'``^-\-`f ---"-`"'' | ''│ - 0 │ │ - └─────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_BCE firefox clientspeeds - ┌─────────────────────────────────────────────────────────────────┐ - 80 │ │ - │ | │ - │ [ │ - │ [ │ - │ [ │ - │ [ | │ - │ [ , | │ - │ [ | ] │ - │ [ [ ] │ - │ [ [ , ] │ - │ [ . [ | ] │ - │ . . . [ l . [ | /, │ - │,. l .| d. M ] l, .\ d |, || ,[| │ - │vM\V.\/N./J.Nu_N.^\_\_._.__Jl..._. ./..Nk. || . _./\|,.│ - 0 │ '' " `" ` ' ' ` " ` ' ' """`"```""""'""""""""'""" ' '"''│ - └─────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_JCE serverspeeds - ┌─────────────────────────────────────────────────────────────────┐ - 600 │ │ - │. │ - │\ │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| , , │ - 0 │l__O_______________________________________________________/u____│ - └─────────────────────────────────────────────────────────────────┘ - 0 200 - - HTS_JCE clientspeeds - ┌─────────────────────────────────────────────────────────────────┐ - 800 │. │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - 0 │l_____________________/____________uL____________u____________a__│ - └─────────────────────────────────────────────────────────────────┘ - 0 200 + HTS_BCE iexplore serverspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ │ + │ . │ + │ | │ + │ | │ + │ [ │ + │ . | [ │ + │ | l [ │ + │ ] ] [ │ + │ ] ] [ │ + │ ||| O N │ + │ ||/. N N │ + │. \_/|||.N.., N . . .. .. .. .│ + │"`` `/^f"Vl|^l./uN/./-`/..._.\____/Lr.^_\_.`-^..r.v-JK.u`"\._,/"-_/.-\r//-.__./│ + 0 │ ' │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE iexplore clientspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ │ + │ │ + │ || │ + │ , || │ + │ | || │ + │ | || │ + │ | || │ + │ [ || │ + │ || || │ + │ || || │ + │ || || │ + │ . || |\ . │ + │.vv,/uA. n _._..__.__\_.,.._..u__._/_../.,..dl/....._/_.../.\._u//..u_.|l_M.,.__│ + 0 │ ' "" " ` `` \ `''` " ` "" "'`` '"`"`'` """`'"' `' '` `"` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE chrome serverspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 50 │ │ + │ │ + │ || │ + │ || │ + │ || │ + │ || │ + │ . || │ + │ ,, | || │ + │ || ] || │ + │ |] ] || │ + │ |||| || || │ + │. |||| ||..\ │ + │|.. |\`U| ... ...,. . . _ . . |__ . . . .\. ||A|| │ + │ ""`| ''/\f"""'"T'\-/'"^`---v/-//`\^/`--/"/`/`/-\--^-`-V` F\`/\v""""/`^//``"/\Y│ + 0 │ │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE chrome clientspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ ,, │ + │ || || │ + │ || || │ + │ || || │ + │ || || │ + │ || || │ + │ || || │ + │ || || │ + │ || || . │ + │ , || |\ | │ + │ .|] /. .| /, │ + │| A.|l|| \ || || .,.. || .│ + │|/`'`' /"`^-/"/\-----/\----`-^---/\------/\v-r---------/|/`-/"\\f\/r--^"`-`""\/`│ + 0 │ "` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE phantomjs serverspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 50 │ │ + │ . │ + │ | │ + │ l │ + │ ] │ + │ ] │ + │ ] │ + │ | | ] │ + │ | | ] │ + │ ,\ ,\ N │ + │ .||, || N . │ + │.\ J/|/,.| . ,,N. . , , .,\ \ │ + │""\``\|-`\-/^--^-/`"^-\vv__.__v`-._../._r/--/L___...//\v--n--r^v-r-v---/"'-O_/_u│ + │ '" `"` │ + 0 │ │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE phantomjs clientspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 30 │ . │ + │ | │ + │ | │ + │ ] │ + │ ] │ + │ ] │ + │ ] │ + │ ] │ + │ ] │ + │ J │ + │ N │ + │ N │ + │ .. N │ + │.. .L.......... .._ . . ,__....... . . .__.... ...N. .._..._.. │ + 0 │ '"` '""`"`""''"`'"T""""""""""` '``'``'"""""""""'""`' '"`""`""``""""""`"'" """│ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE firefox serverspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 50 │ . │ + │ | │ + │ [ │ + │ | [ │ + │ |. . [ │ + │||d l . . , N │ + │||/| ,.N.../| . | , .. . . /. . N .., │ + │/||\/'"|l|l|\.M ,.M /Nk\ .. ..._.Ar., .\ /| /,||, J. . . N|l.l _ │ + │ '` ' "/"`|| /`K|-`'''"`'"-/""` `'"""\r-`""`"`"``/"^^/-//`"`/-/`\-/``"`""`"│ + │------------d+----------------------------A------------------------------------*│ + │ || " │ + │ [ │ + │ F │ + │ | │ + -30 │ | │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE firefox clientspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 40 │ . │ + │ | │ + │ | │ + │ ] │ + │ , ] │ + │ | ] │ + │ ] . ] │ + │ ] | ] │ + │ ], [ ] │ + │ J[ [ N │ + │ NM [ | | l N │ + │ . NN N.. [ /. . . . /, , . N │ + │/.,J/V",N.|\..^ ||. ..,l___. _.ADAk._rl. .. |\.k._.A .. .. ....... .\_N,.-.│ + │ ''"" /"`'"``"\``'"\"` "" `"`'" ""''\/ ' ` "`"/"`'/"`""'"`'"`"""` "` '│ + 0 │ │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_JCE serverspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 500 │ │ + │. │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │l │ + │| │ + │| │ + │| ,| |│ + 0 │|________JL___________________________________________________________________Jl│ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_JCE clientspeeds + ┌────────────────────────────────────────────────────────────────────────────────┐ + 800 │. │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\ │ + │| │ + │| │ + │| │ + 0 │|________________/________________________________n________________.____________│ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 200 Benchmark Tables of all samples, :init_tables, message size 130560 b 200 samples HTS_BCE iexplore => Table with 200 rows, 2 columns: serverspeeds clientspeeds ────────────────────────── -5.15613 3.65342 -2.77637 2.99144 -2.85458 2.76242 -3.77073 3.49141 -3.15904 3.011 -3.74279 2.93838 -2.48589 2.86296 -2.80711 3.19255 -4.31818 3.96347 +5.42427 3.1758 +4.97737 3.8266 +4.83492 3.51935 +4.18971 3.15345 +4.37963 4.13943 +6.50241 2.52779 +5.34885 4.24557 +5.6142 2.78196 +5.55555 3.60594 ⋮ -2.49427 2.92161 -3.73722 2.58085 -2.43282 2.98865 -2.53058 2.6032 -3.10876 2.91324 -3.93274 2.57527 -2.51104 2.8881 -2.50265 2.69817 +3.24562 2.60879 +3.28752 2.5669 +3.19256 2.78755 +3.4998 3.37132 +3.56962 3.32104 +3.47466 3.18696 +5.73431 3.22328 +3.36014 3.3741 HTS_BCE chrome => Table with 200 rows, 2 columns: serverspeeds clientspeeds ────────────────────────── -8.9548 8.54699 -5.32372 5.04999 -9.70335 4.56677 -7.00239 4.83212 -5.11423 5.63935 -6.103 5.27623 -4.64777 5.7371 -5.75386 14.2366 -5.86559 4.99133 +10.2927 7.4046 +8.92965 4.02491 +5.71755 4.22602 +6.19517 4.39919 +5.86558 4.78186 +6.90184 7.54985 +6.31528 5.58627 +5.9382 5.78738 +5.46337 4.96061 ⋮ -5.80693 4.60867 -5.55275 4.56399 -6.57504 5.30975 -6.1058 4.62544 -5.98848 6.70352 -5.06116 4.28746 -4.34892 4.23439 -6.10859 6.24265 +5.08071 4.6394 +4.97178 4.24835 +42.9277 1.59208 +5.74828 4.85168 +5.12261 5.06117 +6.17283 4.8405 +4.41316 5.84882 +6.86831 5.42707 HTS_BCE phantomjs => Table with 200 rows, 2 columns: serverspeeds clientspeeds ────────────────────────── -10.9435 2.60879 -13.0188 2.89369 -17.6163 1.75409 -9.12797 2.10603 -9.91004 2.30712 -10.5469 1.91608 -11.5273 2.71493 -8.68665 3.25959 -8.15316 1.99709 +10.7955 2.6032 +9.97428 2.201 +12.7674 2.0753 +10.762 1.91608 +9.67542 1.9133 +9.40448 1.92727 +8.25651 1.73454 +10.4715 1.99429 +13.4378 2.37138 ⋮ -6.71749 2.5222 -10.2117 2.02223 -13.2311 2.87413 -6.2287 1.60885 -6.61415 1.55857 -6.14769 1.55857 -11.5245 1.63958 -7.47722 1.89095 +7.25935 1.98592 +8.3403 1.97475 +7.49118 1.96637 +7.17835 2.16747 +6.93816 1.96357 +7.88503 1.9943 +7.36271 1.96357 +7.45209 2.00547 HTS_BCE firefox => Table with 200 rows, 2 columns: serverspeeds clientspeeds ────────────────────────── -10.9658 7.75375 -11.8038 5.96335 -34.255 13.826 -10.8066 6.15048 -13.1501 9.20897 -38.4643 6.82642 -20.3927 9.6028 -12.7758 5.6952 -8.42969 6.44935 +10.7843 6.35159 +11.0524 7.2845 +33.297 5.45499 +6.34042 5.06396 +7.3236 6.49683 +19.7167 5.31812 +24.2975 5.25388 +11.1697 6.02479 +11.2871 8.4632 ⋮ -9.13076 5.16171 -8.11127 5.36281 -6.43259 5.26785 -9.39052 7.19511 -8.01071 5.59745 -6.63649 5.2064 -5.79576 4.54164 -5.54157 4.46064 +6.18401 5.99407 +6.11976 5.91586 +6.83201 6.96329 +10.7564 6.69794 +7.45767 6.33204 +6.58063 5.51923 +6.61694 4.86844 +6.37115 6.48845 HTS_JCE => Table with 200 rows, 2 columns: serverspeeds clientspeeds ────────────────────────── -522.696 789.896 -2.57247 4.56398 -2.92721 4.22881 -2.36858 4.18691 -2.32109 9.30672 -2.12837 4.04726 -2.05854 3.89922 -1.75688 4.2316 -1.72616 5.80413 +444.176 751.409 +2.48869 4.01094 +2.41886 4.08915 +1.70661 3.97742 +3.14507 4.03329 +1.65912 3.8657 +1.54181 3.8238 +1.52505 3.83218 +1.63119 3.85173 ⋮ -1.56695 30.5988 -1.94122 5.25947 -1.61163 4.03049 -1.75688 3.88246 -1.4273 4.30143 -1.76526 4.05005 -1.80158 3.96346 -1.76806 3.86291 +1.57253 3.77911 +1.56974 3.7847 +1.56415 3.79029 +1.59209 3.9886 +1.57533 3.80146 +57.2649 4.32936 +1.82392 3.84615 +1.74292 3.84335 Benchmark Plots of varying size messages :test_plots [ns/b], VSIZE = [1762560, 1357620, 1020000, 743580, 522240, 349860, 220320, 127500, 65280, 27540, 8160, 1020] - HTS_BCE iexplore serverbandwidth - ┌─────────────────────────────────────────────────────────────────┐ - 200 │ │ - │ │ - │ │ - │ │ - │. │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │\ │ - 0 │"---.____________________________________________________. │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE iexplore clientbandwidth - ┌─────────────────────────────────────────────────────────────────┐ - 20 │, │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │\ │ - │|--"\-.__r-----------------------------------------------/ │ - │ │ - 0 │ │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE chrome serverbandwidth - ┌─────────────────────────────────────────────────────────────────┐ - 200 │. │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - 0 │|--------------------------------------------------------/ │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE chrome clientbandwidth - ┌─────────────────────────────────────────────────────────────────┐ - 40 │ │ - │ │ - │ │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │l . _____________________.------------""""""""` │ - │""""""""""""""" │ - 0 │ │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE phantomjs serverbandwidth - ┌─────────────────────────────────────────────────────────────────┐ - 300 │ │ - │ │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │\ │ - 0 │"\-------------------------------------------------------/ │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE phantomjs clientbandwidth - ┌─────────────────────────────────────────────────────────────────┐ - 3 │ │ - │, .__-/ │ - │| __--/"` │ - │| _/-._. ______r-----/"" │ - │| _-" '"-._.--/""" │ - │| , .r/ │ - │||"\___./` │ - │|/ │ - │|| │ - │l` │ - │| │ - │ │ - │ │ - │ │ - 1 │ │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE firefox serverbandwidth - ┌─────────────────────────────────────────────────────────────────┐ - 500 │ │ - │ │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │\ │ - 0 │|-_______________________________________________________. │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE firefox clientbandwidth - ┌─────────────────────────────────────────────────────────────────┐ - 40 │ │ - │ │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │. │ - │| │ - │\ .__----------------------------------/""""""""""""--/ │ - │"""""` │ - 0 │ │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_JCE serverbandwidth - ┌─────────────────────────────────────────────────────────────────┐ - 70 │ │ - │ │ - │, │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │\. │ - 0 │ '\------------------------------------------------------/ │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_JCE clientbandwidth - ┌─────────────────────────────────────────────────────────────────┐ - 60 │ │ - │, │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │| │ - │l │ - │""`._____________________________________________________. │ - 0 │ │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE + HTS_BCE iexplore serverbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 200 │ │ + │ │ + │ │ + │ │ + │, │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + 0 │'---__________________________________________________________________. │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE iexplore clientbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 30 │ │ + │ │ + │ │ + │ │ + │. │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │"""\----/-------------------------------------------------------------* │ + 0 │ │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome serverbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 90 │ │ + │| │ + │| │ + │. │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │", │ + │ -.___________________________________________________________________. │ + 0 │ │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome clientbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 30 │ │ + │ │ + │ │ + │ │ + │. │ + │| │ + │| │ + │| │ + │. │ + │| │ + │\ │ + │| ________----------/"""""""""` │ + │"u-----/"""""""""""----------""""""""""""" │ + │ │ + 0 │ │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs serverbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 200 │ │ + │ │ + │ │ + │ │ + │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\. │ + 0 │ '""\-----/""""""""""""""""""""""`------------------------------------* │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs clientbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 2.7 │ │ + │ ._.--/"` │ + │ __r--""` │ + │. __r-/"" │ + │| ,\. ._.--"" │ + │| / '.. ._--------"""""` │ + │| | '-._. ._-/` │ + │| . """\------"` │ + │| | │ + │|,,` │ + │N"\ │ + │N │ + │[ │ + │[ │ + 1.7 │` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox serverbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 500 │ │ + │ │ + │, │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │. │ + 0 │".____________________________________________________________________. │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox clientbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ │ + │, │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │r..._.--------------------.____.-----------------------------------/""` │ + │ '` │ + 0 │ │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE serverbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 90 │. │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\ │ + 0 │ """-__________________________.------------__________________________. │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE clientbandwidth + ┌────────────────────────────────────────────────────────────────────────────────┐ + 70 │. │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │L │ + │ `.__ │ + 0 │ """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE Benchmark Tables of varying size messages :test_tables [ns/b] HTS_BCE iexplore => Table with 12 rows, 3 columns: VSIZE serverbandwidth clientbandwidth ───────────────────────────────────────── -1762560 2.85556 3.31603 -1357620 2.84298 3.20976 -1020000 2.97696 3.21081 -743580 3.61289 3.16786 -522240 3.23825 3.12889 -349860 3.7709 3.19613 -220320 3.65095 3.02147 -127500 4.54435 3.65872 -65280 5.62479 3.46019 -27540 9.29125 3.46253 -8160 21.3134 5.14942 -1020 139.719 19.5797 +1762560 2.93964 3.00496 +1357620 3.03359 3.05538 +1020000 3.18315 3.09781 +743580 3.269 3.21041 +522240 3.16076 3.25224 +349860 3.95211 2.93349 +220320 3.41565 3.35517 +127500 3.63724 3.05993 +65280 5.3764 3.56742 +27540 7.03191 3.41558 +8160 13.7176 3.85587 +1020 141.73 20.8507 HTS_BCE chrome => Table with 12 rows, 3 columns: VSIZE serverbandwidth clientbandwidth ───────────────────────────────────────── -1762560 5.77379 7.83374 -1357620 5.55071 6.69094 -1020000 5.6383 5.99683 -743580 6.51132 5.74968 -522240 6.16186 5.44588 -349860 5.64196 5.07182 -220320 5.24472 5.19075 -127500 8.33349 5.36475 -65280 7.65596 4.94229 -27540 8.5631 4.48888 -8160 31.1207 8.589 -1020 188.954 31.8534 +1762560 6.58524 7.82265 +1357620 6.18083 7.01997 +1020000 7.55952 5.901 +743580 7.82293 5.34699 +522240 7.23085 5.21598 +349860 7.32768 5.6382 +220320 6.54803 5.56201 +127500 7.16353 4.80599 +65280 7.9837 4.82014 +27540 9.56091 4.46564 +8160 68.4336 13.2017 +1020 82.8002 21.1724 HTS_BCE phantomjs => Table with 12 rows, 3 columns: VSIZE serverbandwidth clientbandwidth ───────────────────────────────────────── -1762560 9.21371 2.80276 -1357620 8.29873 2.54377 -1020000 8.4713 2.48141 -743580 8.8489 2.35881 -522240 9.99551 2.55679 -349860 9.1925 2.28953 -220320 7.99395 2.10036 -127500 8.62231 2.07285 -65280 11.2885 2.24716 -27540 16.7748 1.61957 -8160 32.6645 1.9753 -1020 264.541 2.82625 +1762560 8.76095 2.63199 +1357620 8.53769 2.48405 +1020000 8.65545 2.34874 +743580 8.99798 2.33596 +522240 9.89303 2.19347 +349860 9.0338 2.21736 +220320 8.81227 2.28016 +127500 8.71783 2.41444 +65280 9.74343 2.00539 +27540 13.8662 2.06535 +8160 26.9265 1.75901 +1020 132.59 2.46688 HTS_BCE firefox => Table with 12 rows, 3 columns: VSIZE serverbandwidth clientbandwidth ───────────────────────────────────────── -1762560 7.93723 7.08603 -1357620 8.04689 7.23327 -1020000 8.59622 6.29865 -743580 9.22209 6.33995 -522240 9.5801 6.44354 -349860 9.96628 7.03896 -220320 10.5642 6.05941 -127500 6.14458 4.87543 -65280 9.71296 5.16403 -27540 20.6851 4.95048 -8160 67.5271 12.0259 -1020 426.923 34.7188 +1762560 7.50156 7.21121 +1357620 7.36962 6.64654 +1020000 7.76317 6.77497 +743580 8.18686 6.10902 +522240 9.11455 6.4394 +349860 10.6628 7.03435 +220320 8.41041 6.64883 +127500 6.5741 6.13918 +65280 7.69537 5.04611 +27540 14.0782 7.07422 +8160 45.4188 6.10177 +1020 422.115 33.9253 HTS_JCE => Table with 12 rows, 3 columns: VSIZE serverbandwidth clientbandwidth ───────────────────────────────────────── -1762560 2.6018 5.29989 -1357620 2.03094 4.64923 -1020000 1.96231 5.01191 -743580 1.99671 4.64635 -522240 2.11014 4.69373 -349860 1.85716 4.72792 -220320 1.61326 4.60257 -127500 1.96424 4.54956 -65280 3.57493 7.10317 -27540 4.91413 7.4573 -8160 7.55778 13.2848 -1020 60.0278 55.4819 +1762560 1.92309 4.34566 +1357620 1.90076 4.3161 +1020000 2.0336 4.28842 +743580 1.99315 4.26777 +522240 1.89237 4.32442 +349860 1.75345 4.2868 +220320 1.72322 4.05675 +127500 1.98503 4.5993 +65280 5.84204 5.90265 +27540 5.11944 8.51887 +8160 10.6282 10.8941 +1020 85.0579 65.6785 Benchmark Plots of varying size messages :test_latency_plots [ns], VSIZE = [1762560, 1357620, 1020000, 743580, 522240, 349860, 220320, 127500, 65280, 27540, 8160, 1020] - HTS_BCE iexplore serverlatency - ┌─────────────────────────────────────────────────────────────────┐ - 6000000 │ │ - │ │ - │ ._-/ │ - │ _.-"` │ - │ ._-/" │ - │ ._r-/` │ - │ ._--"` │ - │ ___.--/"` │ - │ _-""" │ - │ .r" │ - │ ._r/` │ - │ .,-/"` │ - │ _r/` │ - │ ..-/" │ - 0 │-"` │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE iexplore clientlatency - ┌─────────────────────────────────────────────────────────────────┐ - 6000000 │ _r/ │ - │ ../" │ - │ _-/` │ - │ ../" │ - │ ._-/` │ - │ ._-/` │ - │ ..-"` │ - │ ..-"` │ - │ ._-/` │ - │ ._-/` │ - │ ._-/` │ - │ _.-"` │ - │ ..-" │ - │ ._--/` │ - 0 │_r/` │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE chrome serverlatency - ┌─────────────────────────────────────────────────────────────────┐ - 11000000 │ │ - │ _r/` │ - │ ../" │ - │ ._-"` │ - │ _r/` │ - │ ._-/" │ - │ _r-"` │ - │ __r--"" │ - │ .r/"" │ - │ _-"` │ - │ ..-" │ - │ .r/` │ - │ _-/` │ - │ _---/" │ - 0 │.-" │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE chrome clientlatency - ┌─────────────────────────────────────────────────────────────────┐ - 20000000 │ │ - │ │ - │ │ - │ │ - │ _/ │ - │ _r/" │ - │ ..-" │ - │ ._-/` │ - │ _.-/` │ - │ ._-/" │ - │ ._.-/"` │ - │ .__--/"` │ - │ __r-/"` │ - │ .__,-/"" │ - 0 │__.--/""` │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE phantomjs serverlatency - ┌─────────────────────────────────────────────────────────────────┐ - 20000000 │ │ - │ │ - │ .. │ - │ _r/` │ - │ ../" │ - │ _-/` │ - │ ._./" │ - │ __-/"` │ - │ ._r-/" │ - │ ._r-/"` │ - │ .__--/"` │ - │ ..-"` │ - │ _r/` │ - │ ._r/" │ - 0 │.--""` │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE phantomjs clientlatency - ┌─────────────────────────────────────────────────────────────────┐ - 5000000 │ ../` │ - │ ../` │ - │ ./` │ - │ _-" │ - │ ._-" │ - │ ._-/` │ - │ ._-/` │ - │ _-/` │ - │ _r/" │ - │ ._r/" │ - │ .__-/"` │ - │ .r/` │ - │ ._r"` │ - │ ._-/` │ - 0 │_.-/"` │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE firefox serverlatency - ┌─────────────────────────────────────────────────────────────────┐ - 20000000 │ │ - │ │ - │ │ - │ │ - │ ._r/ │ - │ ._.-/"` │ - │ _.-/"` │ - │ __r-/"" │ - │ ._.-/"" │ - │ ._r-/"` │ - │ ._-/"` │ - │ _.-/"` │ - │ ._-/" │ - │ .-"` │ - 0 │----/` │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_BCE firefox clientlatency - ┌─────────────────────────────────────────────────────────────────┐ - 20000000 │ │ - │ │ - │ │ - │ │ - │ │ - │ .__/ │ - │ ._r-/"` │ - │ ._--""` │ - │ ..-"` │ - │ _r-"` │ - │ ._.--"" │ - │ ._.--""` │ - │ .__r--""` │ - │ __-/"` │ - 0 │___--/" │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_JCE serverlatency - ┌─────────────────────────────────────────────────────────────────┐ - 5000000 │ │ - │ .-` │ - │ ./` │ - │ .r" │ - │ _/` │ - │ ../ │ - │ _r` │ - │ _.-/" │ - │ _.-/" │ - │ ._r-/" │ - │ ._r-/"` │ - │ ._--/"` │ - │ ..-/` │ - │ ._.-"` │ - 0 │.-""""` │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE - - HTS_JCE clientlatency - ┌─────────────────────────────────────────────────────────────────┐ - 10000000 │ . │ - │ .r" │ - │ .r/` │ - │ .r/` │ - │ ../` │ - │ ._r/` │ - │ ._r-/"` │ - │ ._-""` │ - │ _r/` │ - │ ..-" │ - │ ._-/"` │ - │ ._r/"` │ - │ ._-"` │ - │ ._r-"` │ - 0 │_-""` │ - └─────────────────────────────────────────────────────────────────┘ - 0 2000000 - VSIZE + HTS_BCE iexplore serverlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 6000000 │ │ + │ │ + │ ._.-/"` │ + │ ._.-/"` │ + │ ._.-/"` │ + │ __r-/"` │ + │ ._--"" │ + │ ._.-/"` │ + │ ._--"` │ + │ _r-"` │ + │ .__-/" │ + │ .r--"""` │ + │ _-/` │ + │ .__-/" │ + 0 │./"` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE -Benchmark Tables of varying size messages :test_latency_tables [ns] PhantomJS saved render, exits after 30s + HTS_BCE iexplore clientlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 6000000 │ │ + │ _. │ + │ ._r-/" │ + │ _.-/"` │ + │ __--"" │ + │ ._r-/" │ + │ ._--"` │ + │ ._.-/"` │ + │ __-/"` │ + │ _.-/" │ + │ ._r-"" │ + │ ._-/` │ + │ .__--/` │ + │ ..-"` │ + 0 │_r-/"` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome serverlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ │ + │ │ + │ .__--/ │ + │ __r-/"` │ + │ ._______.-/"" │ + │ .__r--""""""` │ + │ ._.--""` │ + │ __-/"` │ + │ .__--/"" │ + │ __r-/"` │ + 0 │n_---""" │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome clientlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ ._/ │ + │ ._r-"` │ + │ ._r-"` │ + │ ._r-"` │ + │ ._-/"` │ + │ __-/"` │ + │ __.-/" │ + │ __r-/"" │ + │ .__.---""" │ + │ .__r---/"""` │ + 0 │___.--/""` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs serverlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ ._-* │ + │ ._.-/"` │ + │ __-/"` │ + │ ._.--"" │ + │ __r-/"` │ + │ .__--"" │ + │ .__--/"` │ + │ __.--""` │ + │ ._r-"" │ + │ ._.-/` │ + │ __--"` │ + 0 │_r--"" │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs clientlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 5000000 │ │ + │ ._-/` │ + │ ..-/` │ + │ _r-"` │ + │ ._r/" │ + │ _.-"` │ + │ ._-/" │ + │ ._r-"` │ + │ ._--"` │ + │ ._r-/"` │ + │ ._r-"` │ + │ __r-"` │ + │ .__--"" │ + │ ._.-/"` │ + 0 │__--"` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox serverlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ │ + │ ._.-/"` │ + │ .__--""` │ + │ .__.--/"` │ + │ .__---""` │ + │ .__--/""` │ + │ .__.--/""` │ + │ .__.---""` │ + │ _-/"` │ + │ _./" │ + 0 │_.---/" │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox clientlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ │ + │ ._r* │ + │ ._r-/"` │ + │ ._.-/"` │ + │ .__r--/"` │ + │ .__--/""` │ + │ ._.-/"` │ + │ .___--/"` │ + │ .___---/""` │ + │ __r-/""` │ + 0 │___r-/"" │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE serverlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 4000000 │ │ + │ │ + │ ._r-/ │ + │ ._--"` │ + │ _.-/"` │ + │ ._.--"" │ + │ ._.--""` │ + │ _.-/"` │ + │ _.-/" │ + │ _.-/" │ + │ ._-/" │ + │ ._-/"` │ + │ __-/"` │ + │ .r._.__--"" │ + 0 │r/ ` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE clientlatency + ┌────────────────────────────────────────────────────────────────────────────────┐ + 8000000 │ ._/ │ + │ _.-"` │ + │ _.-/" │ + │ ._-/" │ + │ ._-/"` │ + │ ._-/"` │ + │ _r-"` │ + │ _.-/" │ + │ _.-/" │ + │ _.-/" │ + │ ._r-/" │ + │ _r-"` │ + │ _.-/" │ + │ ._.-/" │ + 0 │_-/"` │ + └────────────────────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + +Benchmark Tables of varying size messages :test_latency_tables [ns] HTS_BCE iexplore => Table with 12 rows, 3 columns: VSIZE serverlatency clientlatency ───────────────────────────────────── -1762560 5.0331e6 5.84469e6 -1357620 3.85969e6 4.35764e6 -1020000 3.0365e6 3.27503e6 -743580 2.68647e6 2.35556e6 -522240 1.69114e6 1.63403e6 -349860 1.31929e6 1.1182e6 -220320 8.04377e5 6.6569e5 -127500 5.79405e5 4.66486e5 -65280 3.67186e5 2.25881e5 -27540 2.55881e5 95358.0 -8160 1.73917e5 42019.3 -1020 1.42514e5 19971.3 +1762560 5.1813e6 5.29642e6 +1357620 4.11847e6 4.14804e6 +1020000 3.24682e6 3.15977e6 +743580 2.43077e6 2.3872e6 +522240 1.65068e6 1.69845e6 +349860 1.38268e6 1.02631e6 +220320 7.52537e5 7.39212e5 +127500 4.63748e5 3.9014e5 +65280 3.50971e5 2.32881e5 +27540 1.93659e5 94065.2 +8160 111936.0 31463.9 +1020 144565.0 21267.7 HTS_BCE chrome => Table with 12 rows, 3 columns: VSIZE serverlatency clientlatency ───────────────────────────────────── -1762560 1.01767e7 1.38074e7 -1357620 7.53576e6 9.08375e6 -1020000 5.75107e6 6.11676e6 -743580 4.84169e6 4.27534e6 -522240 3.21797e6 2.84406e6 -349860 1.9739e6 1.77443e6 -220320 1.15552e6 1.14363e6 -127500 1.06252e6 6.84006e5 -65280 4.99781e5 3.22632e5 -27540 2.35828e5 1.23624e5 -8160 2.53945e5 70086.3 -1020 1.92733e5 32490.5 +1762560 1.16069e7 1.37879e7 +1357620 8.39122e6 9.53045e6 +1020000 7.71071e6 6.01902e6 +743580 5.81698e6 3.97591e6 +522240 3.77624e6 2.72399e6 +349860 2.56366e6 1.97258e6 +220320 1.44266e6 1.22542e6 +127500 9.1335e5 6.12763e5 +65280 5.21176e5 3.14659e5 +27540 2.63308e5 1.22984e5 +8160 5.58418e5 1.07726e5 +1020 84456.2 21595.9 HTS_BCE phantomjs => Table with 12 rows, 3 columns: VSIZE serverlatency clientlatency ───────────────────────────────────── -1762560 1.62397e7 4.94004e6 -1357620 1.12665e7 3.45348e6 -1020000 8.64073e6 2.53104e6 -743580 6.57986e6 1.75396e6 -522240 5.22005e6 1.33526e6 -349860 3.21609e6 8.01016e5 -220320 1.76123e6 4.6275e5 -127500 1.09934e6 2.64289e5 -65280 7.3691e5 1.46695e5 -27540 4.61979e5 44603.0 -8160 2.66542e5 16118.5 -1020 2.69832e5 2882.77 +1762560 1.54417e7 4.63904e6 +1357620 1.15909e7 3.37239e6 +1020000 8.82856e6 2.39572e6 +743580 6.69072e6 1.73697e6 +522240 5.16653e6 1.14552e6 +349860 3.16056e6 7.75764e5 +220320 1.94152e6 5.02365e5 +127500 1.11152e6 3.07841e5 +65280 6.36051e5 1.30912e5 +27540 3.81875e5 56879.7 +8160 2.1972e5 14353.5 +1020 1.35242e5 2516.22 HTS_BCE firefox => Table with 12 rows, 3 columns: VSIZE serverlatency clientlatency ───────────────────────────────────── -1762560 1.39898e7 1.24896e7 -1357620 1.09246e7 9.82003e6 -1020000 8.76814e6 6.42462e6 -743580 6.85736e6 4.71426e6 -522240 5.00311e6 3.36507e6 -349860 3.4868e6 2.46265e6 -220320 2.3275e6 1.33501e6 -127500 7.83433e5 6.21618e5 -65280 6.34062e5 337108.0 -27540 5.69668e5 1.36336e5 -8160 5.51021e5 98131.3 -1020 4.35462e5 35413.2 +1762560 1.32219e7 1.27102e7 +1357620 1.00051e7 9.02348e6 +1020000 7.91843e6 6.91047e6 +743580 6.08759e6 4.54255e6 +522240 4.75998e6 3.36291e6 +349860 3.73048e6 2.46104e6 +220320 1.85298e6 1.46487e6 +127500 8.38198e5 782746.0 +65280 5.02354e5 3.2941e5 +27540 3.87714e5 194824.0 +8160 3.70618e5 49790.4 +1020 430557.0 34603.8 HTS_JCE => Table with 12 rows, 3 columns: VSIZE serverlatency clientlatency ───────────────────────────────────── -1762560 4.58583e6 9.34138e6 -1357620 2.75724e6 6.31189e6 -1020000 2.00155e6 5.11215e6 -743580 1.48471e6 3.45493e6 -522240 1.102e6 2.45125e6 -349860 6.49745e5 1.65411e6 -220320 3.55433e5 1.01404e6 -127500 2.5044e5 5.80069e5 -65280 2.33372e5 4.63695e5 -27540 1.35335e5 205374.0 -8160 61671.5 1.08404e5 -1020 61228.4 56591.6 +1762560 3.38956e6 7.65948e6 +1357620 2.58051e6 5.85962e6 +1020000 2.07427e6 4.37419e6 +743580 1.48207e6 3.17343e6 +522240 9.88273e5 2.25838e6 +349860 6.13464e5 1.49978e6 +220320 3.7966e5 8.93783e5 +127500 2.53091e5 5.8641e5 +65280 3.81368e5 385325.0 +27540 1.40989e5 2.3461e5 +8160 86726.2 88896.0 +1020 86759.1 66992.1 Benchmark Dictionary :test_bestserverlatencies [ns] -HTS_BCE iexplore => 142514 -HTS_BCE chrome => 192733 -HTS_BCE phantomjs => 266542 -HTS_BCE firefox => 435462 -HTS_JCE => 61228 +HTS_BCE iexplore => 111936 +HTS_BCE chrome => 84456 +HTS_BCE phantomjs => 135242 +HTS_BCE firefox => 370618 +HTS_JCE => 86726 Benchmark Dictionary :test_bestclientlatencies [ns] -HTS_BCE iexplore => 19971 -HTS_BCE chrome => 32490 -HTS_BCE phantomjs => 2883 -HTS_BCE firefox => 35413 -HTS_JCE => 56592 +HTS_BCE iexplore => 21268 +HTS_BCE chrome => 21596 +HTS_BCE phantomjs => 2516 +HTS_BCE firefox => 34604 +HTS_JCE => 66992 Benchmark Dictionary :test_bestserverbandwidths [ns/b] -HTS_BCE iexplore => 2.843 -HTS_BCE chrome => 5.2447 -HTS_BCE phantomjs => 7.994 -HTS_BCE firefox => 6.1446 -HTS_JCE => 1.6133 +HTS_BCE iexplore => 2.9396 +HTS_BCE chrome => 6.1808 +HTS_BCE phantomjs => 8.5377 +HTS_BCE firefox => 6.5741 +HTS_JCE => 1.7232 Benchmark Dictionary :test_bestclientbandwidths [ns/b] -HTS_BCE iexplore => 3.0215 -HTS_BCE chrome => 4.4889 -HTS_BCE phantomjs => 1.6196 -HTS_BCE firefox => 4.8754 -HTS_JCE => 4.5496 +HTS_BCE iexplore => 2.9335 +HTS_BCE chrome => 4.4656 +HTS_BCE phantomjs => 1.759 +HTS_BCE firefox => 5.0461 +HTS_JCE => 4.0568 diff --git a/benchmark/logs/benchmark_results_readable_previous.log b/benchmark/logs/benchmark_results_readable_previous.log new file mode 100644 index 0000000..513417f --- /dev/null +++ b/benchmark/logs/benchmark_results_readable_previous.log @@ -0,0 +1,968 @@ +HTS_BCE iexplore Varying message size: + bestserverbandwidth = 2.843 [ns/b] = [s/GB] @ size = 1357620 b + bestclientbandwidth = 3.0215 [ns/b] = [s/GB] @ size = 220320 b + bestserverlatency = 142514 [ns] @ size = 1020 b + bestclientlatency = 19971 [ns] @ size = 1020 b + +HTS_BCE chrome Varying message size: + bestserverbandwidth = 5.2447 [ns/b] = [s/GB] @ size = 220320 b + bestclientbandwidth = 4.4889 [ns/b] = [s/GB] @ size = 27540 b + bestserverlatency = 192733 [ns] @ size = 1020 b + bestclientlatency = 32490 [ns] @ size = 1020 b + +HTS_BCE phantomjs Varying message size: + bestserverbandwidth = 7.994 [ns/b] = [s/GB] @ size = 220320 b + bestclientbandwidth = 1.6196 [ns/b] = [s/GB] @ size = 27540 b + bestserverlatency = 266542 [ns] @ size = 8160 b + bestclientlatency = 2883 [ns] @ size = 1020 b + +HTS_BCE firefox Varying message size: + bestserverbandwidth = 6.1446 [ns/b] = [s/GB] @ size = 127500 b + bestclientbandwidth = 4.8754 [ns/b] = [s/GB] @ size = 127500 b + bestserverlatency = 435462 [ns] @ size = 1020 b + bestclientlatency = 35413 [ns] @ size = 1020 b + +HTS_JCE Varying message size: + bestserverbandwidth = 1.6133 [ns/b] = [s/GB] @ size = 220320 b + bestclientbandwidth = 4.5496 [ns/b] = [s/GB] @ size = 127500 b + bestserverlatency = 61228 [ns] @ size = 1020 b + bestclientlatency = 56592 [ns] @ size = 1020 b + +Benchmark Plots of all samples :init_plots [ns/b], message size 130560 b 200 samples + HTS_BCE iexplore serverspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 20 │ │ + │ │ + │ , │ + │ | | │ + │ | .|| │ + │ | ||| │ + │ ] ||| │ + │ O ||| │ + │ N [|| │ + │ .Nl N|\ , │ + │ ||^.N||, . . 1 , .. , |, . │ + │\ ./"`]|/l[.N [ ..."^k| ||.[ |M| |.,| .._.|\ .v J\ │ + │lKv` \M"A^.|K" `\\ |KG^./"/_kV||/. ,... ,\''\/|.//'P`"\k||│ + │ ` "` """ " ` `" "`"`"/"/""`" """ '''│ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE iexplore clientspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ │ + │ | │ + │ | | │ + │ | | │ + │ [ | │ + │ [ | │ + │ [ ] │ + │ [ ] │ + │ [ /. │ + │ [ || │ + │ , [ || │ + │ || N || │ + │._..^/.^/l__,..._.___/_...._.___.__.N... \... .L_runl._.v.|||...│ + 0 │ "`"` ' ''`" " ' "`"`""""``""" `" ` '"'"'│ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE chrome serverspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 30 │ │ + │ │ + │ │ + │ | │ + │ | │ + │ | . │ + │ | | │ + │ k | │ + │ .N [ │ + │ |N M │ + │G, ||N . N │ + │]L ./| \. l / , N. _ .. |. _ │ + │//F"/ \/| .. O .`"../, rK\.\r_|L`\,r"V, .v/.. ._,.\"^\Y\r.\./`W│ + │ `'""`""""" '`""` '` \ '` '\" ""``'` "" ' │ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE chrome clientspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ | | , │ + │ | | | │ + │ | || | │ + │ | | || | │ + │ | |. || l │ + │ | || || ] │ + │ .| || || ] │ + │ || || || ] │ + │ , |. || || N │ + │ l || || || N │ + │. ] || || |. N │ + │|.O.u,LN| || , . . ||. . , N. ,,│ + │'"""'`''"`^^^/--/``"--^`-/""-""""`//f\/^---\-/--^/`fK"/"""/--`/""│ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE phantomjs serverspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 30 │ │ + │ │ + │ . │ + │ | │ + │ | │ + │ l │ + │|, ] │ + │|| ] , │ + │/| N, | . l , │ + │/|\ NA J /. . ] , \ , [d│ + │ /l N| N./\,, ,. .. \../.\ .vJ| , l. /, l. /, ||]│ + │ r`"u|v` H\^/\\.v\^/`^`"K`'^`v"v^'"'`/\//\-``JJ/v\/^"YK`ln\^fl/│ + │ │ + │ │ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE phantomjs clientspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ │ + │ │ + │ | │ + │ | │ + │ | │ + │ | │ + │ | │ + │ | │ + │ || │ + │ || │ + │ || │ + │ || │ + │. ..\, . . . || . .. . │ + 0 │"""`"""""'""""/"""""""\/"""\`//"'""""""""""""'`^--/`"""""`"""""\-│ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE firefox serverspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 50 │ │ + │ │ + │ , | │ + │ , , | | | │ + │.|, | | |. | │ + │|N| | ] , || |, │ + │|N| l ] .] || ||, │ + │|N| O /, d],||,.|| | │ + │|N| N || ]][|||||] [ || │ + │|N\.N || , ]][/|O||/. @, @| │ + │|/||| |[ ].]][||N|||| . @| M\ │ + │|F\/| |^,N|@/\|MN|.|| | |l|| , .,|| │ + │ |`|] |'``'|l`||`\`^\_kf./n./__^^.,u../|1\ . 1. A .\J lKd,│ + │ ' "'''` "'``^-\-`f ---"-`"'' | ''│ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_BCE firefox clientspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 80 │ │ + │ | │ + │ [ │ + │ [ │ + │ [ │ + │ [ | │ + │ [ , | │ + │ [ | ] │ + │ [ [ ] │ + │ [ [ , ] │ + │ [ . [ | ] │ + │ . . . [ l . [ | /, │ + │,. l .| d. M ] l, .\ d |, || ,[| │ + │vM\V.\/N./J.Nu_N.^\_\_._.__Jl..._. ./..Nk. || . _./\|,.│ + 0 │ '' " `" ` ' ' ` " ` ' ' """`"```""""'""""""""'""" ' '"''│ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_JCE serverspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 600 │ │ + │. │ + │\ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| , , │ + 0 │l__O_______________________________________________________/u____│ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + + HTS_JCE clientspeeds + ┌─────────────────────────────────────────────────────────────────┐ + 800 │. │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + 0 │l_____________________/____________uL____________u____________a__│ + └─────────────────────────────────────────────────────────────────┘ + 0 200 + +Benchmark Tables of all samples, :init_tables, message size 130560 b 200 samples +HTS_BCE iexplore +=> Table with 200 rows, 2 columns: +serverspeeds clientspeeds +────────────────────────── +5.15613 3.65342 +2.77637 2.99144 +2.85458 2.76242 +3.77073 3.49141 +3.15904 3.011 +3.74279 2.93838 +2.48589 2.86296 +2.80711 3.19255 +4.31818 3.96347 +⋮ +2.49427 2.92161 +3.73722 2.58085 +2.43282 2.98865 +2.53058 2.6032 +3.10876 2.91324 +3.93274 2.57527 +2.51104 2.8881 +2.50265 2.69817 + +HTS_BCE chrome +=> Table with 200 rows, 2 columns: +serverspeeds clientspeeds +────────────────────────── +8.9548 8.54699 +5.32372 5.04999 +9.70335 4.56677 +7.00239 4.83212 +5.11423 5.63935 +6.103 5.27623 +4.64777 5.7371 +5.75386 14.2366 +5.86559 4.99133 +⋮ +5.80693 4.60867 +5.55275 4.56399 +6.57504 5.30975 +6.1058 4.62544 +5.98848 6.70352 +5.06116 4.28746 +4.34892 4.23439 +6.10859 6.24265 + +HTS_BCE phantomjs +=> Table with 200 rows, 2 columns: +serverspeeds clientspeeds +────────────────────────── +10.9435 2.60879 +13.0188 2.89369 +17.6163 1.75409 +9.12797 2.10603 +9.91004 2.30712 +10.5469 1.91608 +11.5273 2.71493 +8.68665 3.25959 +8.15316 1.99709 +⋮ +6.71749 2.5222 +10.2117 2.02223 +13.2311 2.87413 +6.2287 1.60885 +6.61415 1.55857 +6.14769 1.55857 +11.5245 1.63958 +7.47722 1.89095 + +HTS_BCE firefox +=> Table with 200 rows, 2 columns: +serverspeeds clientspeeds +────────────────────────── +10.9658 7.75375 +11.8038 5.96335 +34.255 13.826 +10.8066 6.15048 +13.1501 9.20897 +38.4643 6.82642 +20.3927 9.6028 +12.7758 5.6952 +8.42969 6.44935 +⋮ +9.13076 5.16171 +8.11127 5.36281 +6.43259 5.26785 +9.39052 7.19511 +8.01071 5.59745 +6.63649 5.2064 +5.79576 4.54164 +5.54157 4.46064 + +HTS_JCE +=> Table with 200 rows, 2 columns: +serverspeeds clientspeeds +────────────────────────── +522.696 789.896 +2.57247 4.56398 +2.92721 4.22881 +2.36858 4.18691 +2.32109 9.30672 +2.12837 4.04726 +2.05854 3.89922 +1.75688 4.2316 +1.72616 5.80413 +⋮ +1.56695 30.5988 +1.94122 5.25947 +1.61163 4.03049 +1.75688 3.88246 +1.4273 4.30143 +1.76526 4.05005 +1.80158 3.96346 +1.76806 3.86291 + +Benchmark Plots of varying size messages :test_plots [ns/b], + VSIZE = [1762560, 1357620, 1020000, 743580, 522240, 349860, 220320, 127500, 65280, 27540, 8160, 1020] + HTS_BCE iexplore serverbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 200 │ │ + │ │ + │ │ + │ │ + │. │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\ │ + 0 │"---.____________________________________________________. │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE iexplore clientbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 20 │, │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\ │ + │|--"\-.__r-----------------------------------------------/ │ + │ │ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome serverbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 200 │. │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + 0 │|--------------------------------------------------------/ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome clientbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ │ + │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │l . _____________________.------------""""""""` │ + │""""""""""""""" │ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs serverbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 300 │ │ + │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\ │ + 0 │"\-------------------------------------------------------/ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs clientbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 3 │ │ + │, .__-/ │ + │| __--/"` │ + │| _/-._. ______r-----/"" │ + │| _-" '"-._.--/""" │ + │| , .r/ │ + │||"\___./` │ + │|/ │ + │|| │ + │l` │ + │| │ + │ │ + │ │ + │ │ + 1 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox serverbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 500 │ │ + │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\ │ + 0 │|-_______________________________________________________. │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox clientbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 40 │ │ + │ │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │. │ + │| │ + │\ .__----------------------------------/""""""""""""--/ │ + │"""""` │ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE serverbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 70 │ │ + │ │ + │, │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │\. │ + 0 │ '\------------------------------------------------------/ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE clientbandwidth + ┌─────────────────────────────────────────────────────────────────┐ + 60 │ │ + │, │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │| │ + │l │ + │""`._____________________________________________________. │ + 0 │ │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + +Benchmark Tables of varying size messages :test_tables [ns/b] +HTS_BCE iexplore +=> Table with 12 rows, 3 columns: +VSIZE serverbandwidth clientbandwidth +───────────────────────────────────────── +1762560 2.85556 3.31603 +1357620 2.84298 3.20976 +1020000 2.97696 3.21081 +743580 3.61289 3.16786 +522240 3.23825 3.12889 +349860 3.7709 3.19613 +220320 3.65095 3.02147 +127500 4.54435 3.65872 +65280 5.62479 3.46019 +27540 9.29125 3.46253 +8160 21.3134 5.14942 +1020 139.719 19.5797 + +HTS_BCE chrome +=> Table with 12 rows, 3 columns: +VSIZE serverbandwidth clientbandwidth +───────────────────────────────────────── +1762560 5.77379 7.83374 +1357620 5.55071 6.69094 +1020000 5.6383 5.99683 +743580 6.51132 5.74968 +522240 6.16186 5.44588 +349860 5.64196 5.07182 +220320 5.24472 5.19075 +127500 8.33349 5.36475 +65280 7.65596 4.94229 +27540 8.5631 4.48888 +8160 31.1207 8.589 +1020 188.954 31.8534 + +HTS_BCE phantomjs +=> Table with 12 rows, 3 columns: +VSIZE serverbandwidth clientbandwidth +───────────────────────────────────────── +1762560 9.21371 2.80276 +1357620 8.29873 2.54377 +1020000 8.4713 2.48141 +743580 8.8489 2.35881 +522240 9.99551 2.55679 +349860 9.1925 2.28953 +220320 7.99395 2.10036 +127500 8.62231 2.07285 +65280 11.2885 2.24716 +27540 16.7748 1.61957 +8160 32.6645 1.9753 +1020 264.541 2.82625 + +HTS_BCE firefox +=> Table with 12 rows, 3 columns: +VSIZE serverbandwidth clientbandwidth +───────────────────────────────────────── +1762560 7.93723 7.08603 +1357620 8.04689 7.23327 +1020000 8.59622 6.29865 +743580 9.22209 6.33995 +522240 9.5801 6.44354 +349860 9.96628 7.03896 +220320 10.5642 6.05941 +127500 6.14458 4.87543 +65280 9.71296 5.16403 +27540 20.6851 4.95048 +8160 67.5271 12.0259 +1020 426.923 34.7188 + +HTS_JCE +=> Table with 12 rows, 3 columns: +VSIZE serverbandwidth clientbandwidth +───────────────────────────────────────── +1762560 2.6018 5.29989 +1357620 2.03094 4.64923 +1020000 1.96231 5.01191 +743580 1.99671 4.64635 +522240 2.11014 4.69373 +349860 1.85716 4.72792 +220320 1.61326 4.60257 +127500 1.96424 4.54956 +65280 3.57493 7.10317 +27540 4.91413 7.4573 +8160 7.55778 13.2848 +1020 60.0278 55.4819 + +Benchmark Plots of varying size messages :test_latency_plots [ns], + VSIZE = [1762560, 1357620, 1020000, 743580, 522240, 349860, 220320, 127500, 65280, 27540, 8160, 1020] + HTS_BCE iexplore serverlatency + ┌─────────────────────────────────────────────────────────────────┐ + 6000000 │ │ + │ │ + │ ._-/ │ + │ _.-"` │ + │ ._-/" │ + │ ._r-/` │ + │ ._--"` │ + │ ___.--/"` │ + │ _-""" │ + │ .r" │ + │ ._r/` │ + │ .,-/"` │ + │ _r/` │ + │ ..-/" │ + 0 │-"` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE iexplore clientlatency + ┌─────────────────────────────────────────────────────────────────┐ + 6000000 │ _r/ │ + │ ../" │ + │ _-/` │ + │ ../" │ + │ ._-/` │ + │ ._-/` │ + │ ..-"` │ + │ ..-"` │ + │ ._-/` │ + │ ._-/` │ + │ ._-/` │ + │ _.-"` │ + │ ..-" │ + │ ._--/` │ + 0 │_r/` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome serverlatency + ┌─────────────────────────────────────────────────────────────────┐ + 11000000 │ │ + │ _r/` │ + │ ../" │ + │ ._-"` │ + │ _r/` │ + │ ._-/" │ + │ _r-"` │ + │ __r--"" │ + │ .r/"" │ + │ _-"` │ + │ ..-" │ + │ .r/` │ + │ _-/` │ + │ _---/" │ + 0 │.-" │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE chrome clientlatency + ┌─────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ _/ │ + │ _r/" │ + │ ..-" │ + │ ._-/` │ + │ _.-/` │ + │ ._-/" │ + │ ._.-/"` │ + │ .__--/"` │ + │ __r-/"` │ + │ .__,-/"" │ + 0 │__.--/""` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs serverlatency + ┌─────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ .. │ + │ _r/` │ + │ ../" │ + │ _-/` │ + │ ._./" │ + │ __-/"` │ + │ ._r-/" │ + │ ._r-/"` │ + │ .__--/"` │ + │ ..-"` │ + │ _r/` │ + │ ._r/" │ + 0 │.--""` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE phantomjs clientlatency + ┌─────────────────────────────────────────────────────────────────┐ + 5000000 │ ../` │ + │ ../` │ + │ ./` │ + │ _-" │ + │ ._-" │ + │ ._-/` │ + │ ._-/` │ + │ _-/` │ + │ _r/" │ + │ ._r/" │ + │ .__-/"` │ + │ .r/` │ + │ ._r"` │ + │ ._-/` │ + 0 │_.-/"` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox serverlatency + ┌─────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ ._r/ │ + │ ._.-/"` │ + │ _.-/"` │ + │ __r-/"" │ + │ ._.-/"" │ + │ ._r-/"` │ + │ ._-/"` │ + │ _.-/"` │ + │ ._-/" │ + │ .-"` │ + 0 │----/` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_BCE firefox clientlatency + ┌─────────────────────────────────────────────────────────────────┐ + 20000000 │ │ + │ │ + │ │ + │ │ + │ │ + │ .__/ │ + │ ._r-/"` │ + │ ._--""` │ + │ ..-"` │ + │ _r-"` │ + │ ._.--"" │ + │ ._.--""` │ + │ .__r--""` │ + │ __-/"` │ + 0 │___--/" │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE serverlatency + ┌─────────────────────────────────────────────────────────────────┐ + 5000000 │ │ + │ .-` │ + │ ./` │ + │ .r" │ + │ _/` │ + │ ../ │ + │ _r` │ + │ _.-/" │ + │ _.-/" │ + │ ._r-/" │ + │ ._r-/"` │ + │ ._--/"` │ + │ ..-/` │ + │ ._.-"` │ + 0 │.-""""` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + + HTS_JCE clientlatency + ┌─────────────────────────────────────────────────────────────────┐ + 10000000 │ . │ + │ .r" │ + │ .r/` │ + │ .r/` │ + │ ../` │ + │ ._r/` │ + │ ._r-/"` │ + │ ._-""` │ + │ _r/` │ + │ ..-" │ + │ ._-/"` │ + │ ._r/"` │ + │ ._-"` │ + │ ._r-"` │ + 0 │_-""` │ + └─────────────────────────────────────────────────────────────────┘ + 0 2000000 + VSIZE + +Benchmark Tables of varying size messages :test_latency_tables [ns] +PhantomJS saved render, exits after 30s +HTS_BCE iexplore +=> Table with 12 rows, 3 columns: +VSIZE serverlatency clientlatency +───────────────────────────────────── +1762560 5.0331e6 5.84469e6 +1357620 3.85969e6 4.35764e6 +1020000 3.0365e6 3.27503e6 +743580 2.68647e6 2.35556e6 +522240 1.69114e6 1.63403e6 +349860 1.31929e6 1.1182e6 +220320 8.04377e5 6.6569e5 +127500 5.79405e5 4.66486e5 +65280 3.67186e5 2.25881e5 +27540 2.55881e5 95358.0 +8160 1.73917e5 42019.3 +1020 1.42514e5 19971.3 + +HTS_BCE chrome +=> Table with 12 rows, 3 columns: +VSIZE serverlatency clientlatency +───────────────────────────────────── +1762560 1.01767e7 1.38074e7 +1357620 7.53576e6 9.08375e6 +1020000 5.75107e6 6.11676e6 +743580 4.84169e6 4.27534e6 +522240 3.21797e6 2.84406e6 +349860 1.9739e6 1.77443e6 +220320 1.15552e6 1.14363e6 +127500 1.06252e6 6.84006e5 +65280 4.99781e5 3.22632e5 +27540 2.35828e5 1.23624e5 +8160 2.53945e5 70086.3 +1020 1.92733e5 32490.5 + +HTS_BCE phantomjs +=> Table with 12 rows, 3 columns: +VSIZE serverlatency clientlatency +───────────────────────────────────── +1762560 1.62397e7 4.94004e6 +1357620 1.12665e7 3.45348e6 +1020000 8.64073e6 2.53104e6 +743580 6.57986e6 1.75396e6 +522240 5.22005e6 1.33526e6 +349860 3.21609e6 8.01016e5 +220320 1.76123e6 4.6275e5 +127500 1.09934e6 2.64289e5 +65280 7.3691e5 1.46695e5 +27540 4.61979e5 44603.0 +8160 2.66542e5 16118.5 +1020 2.69832e5 2882.77 + +HTS_BCE firefox +=> Table with 12 rows, 3 columns: +VSIZE serverlatency clientlatency +───────────────────────────────────── +1762560 1.39898e7 1.24896e7 +1357620 1.09246e7 9.82003e6 +1020000 8.76814e6 6.42462e6 +743580 6.85736e6 4.71426e6 +522240 5.00311e6 3.36507e6 +349860 3.4868e6 2.46265e6 +220320 2.3275e6 1.33501e6 +127500 7.83433e5 6.21618e5 +65280 6.34062e5 337108.0 +27540 5.69668e5 1.36336e5 +8160 5.51021e5 98131.3 +1020 4.35462e5 35413.2 + +HTS_JCE +=> Table with 12 rows, 3 columns: +VSIZE serverlatency clientlatency +───────────────────────────────────── +1762560 4.58583e6 9.34138e6 +1357620 2.75724e6 6.31189e6 +1020000 2.00155e6 5.11215e6 +743580 1.48471e6 3.45493e6 +522240 1.102e6 2.45125e6 +349860 6.49745e5 1.65411e6 +220320 3.55433e5 1.01404e6 +127500 2.5044e5 5.80069e5 +65280 2.33372e5 4.63695e5 +27540 1.35335e5 205374.0 +8160 61671.5 1.08404e5 +1020 61228.4 56591.6 + +Benchmark Dictionary :test_bestserverlatencies [ns] +HTS_BCE iexplore => 142514 +HTS_BCE chrome => 192733 +HTS_BCE phantomjs => 266542 +HTS_BCE firefox => 435462 +HTS_JCE => 61228 +Benchmark Dictionary :test_bestclientlatencies [ns] +HTS_BCE iexplore => 19971 +HTS_BCE chrome => 32490 +HTS_BCE phantomjs => 2883 +HTS_BCE firefox => 35413 +HTS_JCE => 56592 +Benchmark Dictionary :test_bestserverbandwidths [ns/b] +HTS_BCE iexplore => 2.843 +HTS_BCE chrome => 5.2447 +HTS_BCE phantomjs => 7.994 +HTS_BCE firefox => 6.1446 +HTS_JCE => 1.6133 +Benchmark Dictionary :test_bestclientbandwidths [ns/b] +HTS_BCE iexplore => 3.0215 +HTS_BCE chrome => 4.4889 +HTS_BCE phantomjs => 1.6196 +HTS_BCE firefox => 4.8754 +HTS_JCE => 4.5496 diff --git a/benchmark/ws_hts.jl b/benchmark/ws_hts.jl index 14c4ec3..087a223 100644 --- a/benchmark/ws_hts.jl +++ b/benchmark/ws_hts.jl @@ -10,7 +10,7 @@ import HTTP.Header using ..WebSockets # We want to log to a separate file, so # we use our own instance of logutils_ws here. -import logutils_ws: logto, clog, zlog, zflush +import logutils_ws: logto, clog, zlog, zflush, clog_notime const SRCPATH = Base.source_dir() == nothing ? Pkg.dir("WebSockets", "benchmark") :Base.source_dir() const SERVEFILE = "bce.html" const PORT = 8000 diff --git a/benchmark/ws_jce.jl b/benchmark/ws_jce.jl index 52632cd..ad26d09 100644 --- a/benchmark/ws_jce.jl +++ b/benchmark/ws_jce.jl @@ -11,7 +11,7 @@ using ..HTTP using ..WebSockets # We want to log to a separate file, and so use our own # instance of logutils_ws in this process -import logutils_ws: logto, clog, zlog, zflush +import logutils_ws: logto, clog, zlog, zflush, clog_notime const SRCPATH = Base.source_dir() == nothing ? Pkg.dir("WebSockets", "benchmark") :Base.source_dir() const LOGFILE = "ws_jce.log" diff --git a/logutils/log_http.jl b/logutils/log_http.jl index 1ecf05c..c8b750f 100644 --- a/logutils/log_http.jl +++ b/logutils/log_http.jl @@ -8,9 +8,9 @@ show method does not print binary data well as per now." function _show(d::AbstractDevice, response::HTTP.Messages.Response) _log(d, :green, "Response status: ", :bold, response.status," ") response.status > 0 && _log(d, HTTP.Messages.STATUS_MESSAGES[response.status], " ") - if !isempty(response.headers) - _log(d, :green, " Headers: ", :bold, length(response.headers)) - _log(d, :green, "\n", response.headers) + if !isempty(response.headers) + _log(d, :green, " Headers: ", :bold, length(response.headers)) + _log(d, :green, "\n", response.headers) end if isdefined(response, :cookies) if !isempty(response.cookies) @@ -18,25 +18,25 @@ function _show(d::AbstractDevice, response::HTTP.Messages.Response) _log(d, "\n", response.cookies) end end - if !isempty(response.body) + if !isempty(response.body) _log(d, "\t", DataDispatch(response.body, HTTP.header(response, "content-type", ""))) - end - nothing + end + nothing end "HTTP.Request already has a show method, we're not overwriting that. This metod is called only when logging to an Abstractdevice" function _show(d::AbstractDevice, request::HTTP.Messages.Request) - _log(d, :normal, :light_yellow, "Request ", :normal) - _log(d, :bold, request.method, " ", :cyan, request.target, "\n", :normal) - if !isempty(request.body) - _log(d, "\t", DataDispatch(request.body, HTTP.header(request, "content-type", ""))) - end + _log(d, :normal, :light_yellow, "Request ", :normal) + _log(d, :bold, request.method, " ", :cyan, request.target, "\n", :normal) + if !isempty(request.body) + _log(d, "\t", DataDispatch(request.body, HTTP.header(request, "content-type", ""))) + end if !isempty(request.headers) - _log(d, "\t", :cyan, " Headers: ", length(request.headers)) - _log(d, :cyan, "\n", request.headers) - end - nothing + _log(d, "\t", :cyan, " Headers: ", length(request.headers)) + _log(d, :cyan, "\n", request.headers) + end + nothing end diff --git a/logutils/log_httpserver.jl b/logutils/log_httpserver.jl index f0af9a9..9166ba6 100644 --- a/logutils/log_httpserver.jl +++ b/logutils/log_httpserver.jl @@ -11,77 +11,77 @@ import URIParser.URI export printstartinfo show(io::IO, client::Client) = directto_abstractdevice(io, client) function _show(d::AbstractDevice, client::Client) - _log(d, typeof(client), " id ", :bold, client.id, :normal) - _log(d, " ", client.sock, "\n") - _log(d, "\t\t\t", client.parser) - nothing + _log(d, typeof(client), " id ", :bold, client.id, :normal) + _log(d, " ", client.sock, "\n") + _log(d, "\t\t\t", client.parser) + nothing end # TODO is this good enough? function show(io::IO, p::ClientParser) - print(io, "HttpServer.ClientParser.HttpParser.libhttp-parser: v", HttpParser.version()) - if p.parser.http_major > 0 - print(io, " HTTP/", p.parser.http_major, ".", p.parser.http_minor) - end - nothing + print(io, "HttpServer.ClientParser.HttpParser.libhttp-parser: v", HttpParser.version()) + if p.parser.http_major > 0 + print(io, " HTTP/", p.parser.http_major, ".", p.parser.http_minor) + end + nothing end "Response already has a decent show method, we're not overwriting that. This metod is called only when logging to an Abstractdevice" function _show(d::AbstractDevice, response::HttpServer.Response) - _log(d, :green, "Response status: ", :bold, response.status," ") - _log(d, get(STATUS_CODES, response.status, "--"), " ") - if !isempty(response.headers) - _log(d, :green, " Headers: ", :bold, response.headers.count) - _log(d, :green, "\n", response.headers) - end - if !isempty(response.cookies) - _log(d, " Cookies: ", :bold, length(response.cookies)) - _log(d, "\n", response.cookies) - end - if !isempty(response.data) - _log(d, "\t", DataDispatch(response.data, get(response.headers, "Content-Type","---"))) - end - nothing + _log(d, :green, "Response status: ", :bold, response.status," ") + _log(d, get(STATUS_CODES, response.status, "--"), " ") + if !isempty(response.headers) + _log(d, :green, " Headers: ", :bold, response.headers.count) + _log(d, :green, "\n", response.headers) + end + if !isempty(response.cookies) + _log(d, " Cookies: ", :bold, length(response.cookies)) + _log(d, "\n", response.cookies) + end + if !isempty(response.data) + _log(d, "\t", DataDispatch(response.data, get(response.headers, "Content-Type","---"))) + end + nothing end "Request already has a decent show method, we're not overwriting that. This metod is called only when logging to an Abstractdevice" function _show(d::AbstractDevice, request::Request) - _log(d, :normal, :light_yellow, "Request ", :normal) - _log(d, :bold, request.method, " ", :cyan, request.resource, "\n", :normal) - if !isempty(request.data) - _log(d, "\t", DataDispatch(request.data, get(request.headers, "Content-Type", "text/html; charset=utf-8"))) - end - if request.uri != URI("") - _log(d, :cyan, "\tUri:", :bold, _string(request.uri), :normal, "\n\t") - end - if !isempty(request.headers) - _log(d, "\t", :cyan, " .headers: ", request.headers.count) - _log(d, :cyan, "\n", request.headers) - end - nothing + _log(d, :normal, :light_yellow, "Request ", :normal) + _log(d, :bold, request.method, " ", :cyan, request.resource, "\n", :normal) + if !isempty(request.data) + _log(d, "\t", DataDispatch(request.data, get(request.headers, "Content-Type", "text/html; charset=utf-8"))) + end + if request.uri != URI("") + _log(d, :cyan, "\tUri:", :bold, _string(request.uri), :normal, "\n\t") + end + if !isempty(request.headers) + _log(d, "\t", :cyan, " .headers: ", request.headers.count) + _log(d, :cyan, "\n", request.headers) + end + nothing end "HttpServer does not define a show method for its server type. Defining this is not piracy." show(io::IO, server::Server) = directto_abstractdevice(io, server) function _show(d::AbstractDevice, server::Server) - _log(d, :bold , :green, typeof(server), "(\n", :normal) - server.http != nothing && _log(d, "\t", server.http) - server.websock != nothing && _log(d, "\t", server.websock) - _log(d, :bold, :green, ")") - nothing + _log(d, :bold , :green, typeof(server), "(\n", :normal) + server.http != nothing && _log(d, "\t", server.http) + server.websock != nothing && _log(d, "\t", server.websock) + _log(d, :bold, :green, ")") + nothing end "HttpServer does not define a show method for its HttpHandler type. Defining this is not piracy." show(io::IO, httphandler::HttpHandler) = directto_abstractdevice(io, httphandler) function _show(d::AbstractDevice, httphandler::HttpHandler) - _log(d, :bold, Base.info_color(), typeof(httphandler), "( " , :normal) - _log(d, ".handle: ", :blue, :bold, httphandler.handle, :normal, "\n") - _log(d, "\t\t\t.events:\n") - _log(d, httphandler.events) - _log(d, "\t\t\t", ".socket:\t", httphandler.sock, :bold, Base.info_color(), ")\n") - nothing + _log(d, :bold, Base.info_color(), typeof(httphandler), "( " , :normal) + _log(d, ".handle: ", :blue, :bold, httphandler.handle, :normal, "\n") + _log(d, "\t\t\t.events:\n") + _log(d, httphandler.events) + _log(d, "\t\t\t", ".socket:\t", httphandler.sock, :bold, Base.info_color(), ")\n") + nothing end nothing \ No newline at end of file diff --git a/logutils/log_ws.jl b/logutils/log_ws.jl index 956536b..a4649dd 100644 --- a/logutils/log_ws.jl +++ b/logutils/log_ws.jl @@ -5,26 +5,18 @@ Included in logutils.jl import WebSockets.WebSocket # todo remove when including in WebSockets itself. show(io::IO, ws::WebSocket) = directto_abstractdevice(io, ws) function _show(d::AbstractDevice, ws::WebSocket{T}) where T - _log(d, "WebSocket{", T, "}(") - _log(d, ws.server ? "server, " : "client, ") - _log(d, ws.socket, " ") - showcompact(d.s, ws.state) - _log(d, ")") -# size = ws.socket.buffer.size -# if size == 0 -# _log(d, size, " b)") -# elseif size < 1000 -# _log(d, :yellow, size, " b)") -# else -# _log(d, :bold, :yellow, div(size, 1000), " kB)") -# end + _log(d, "WebSocket{", T, "}(") + _log(d, ws.server ? "server, " : "client, ") + _log(d, ws.socket, " ") + showcompact(d.s, ws.state) + _log(d, ")") nothing end @require HttpServer import WebSockets.WebSocketHandler @require HttpServer show(io::IO, wsh::WebSocketHandler) = directto_abstractdevice(io, wsh) @require HttpServer function _show(d::AbstractDevice, wsh::WebSocketHandler) - _log(d, typeof(wsh), "( " , :blue, :bold, wsh.handle, :normal, ")") - nothing - end + _log(d, typeof(wsh), "( " , :blue, :bold, wsh.handle, :normal, ")") + nothing + end nothing \ No newline at end of file diff --git a/src/HTTP.jl b/src/HTTP.jl index aafbc8a..04441af 100644 --- a/src/HTTP.jl +++ b/src/HTTP.jl @@ -1,8 +1,8 @@ info("Loading HTTP methods...") """ -Initiate a websocket connection to server defined by url. If the server accepts -the connection and the upgrade to websocket, f is called with an open client type websocket. +Initiate a websocket|client connection to server defined by url. If the server accepts +the connection and the upgrade to websocket, f is called with an open websocket|client e.g. say hello, close and leave ```julia @@ -113,7 +113,7 @@ try end end catch err - showerror(err) + showerror(STDERR, err) println.(catch_stacktrace()[1:4]) end ``` @@ -153,19 +153,20 @@ function upgrade(f::Function, http::HTTP.Stream) # Pass the connection on as a WebSocket. io = HTTP.ConnectionPool.getrawstream(http) ws = WebSocket(io, true) + # If the callback function f has two methods, + # prefer the more secure one which takes (request, websocket) try if applicable(f, http.message, ws) f(http.message, ws) else f(ws) end - catch err - # TODO improve - warn("WebSockets.HTTP.upgrade: Caught unhandled error while calling argument function f, the handler / gatekeeper:\n\t") - mt = typeof(f).name.mt - fnam = splitdir(string(mt.defs.func.file))[2] - print_with_color(:yellow, STDERR, "f = ", string(f) * " at " * fnam * ":" * string(mt.defs.func.line) * "\nERROR:\t") - showerror(STDERR, err, catch_stacktrace()) +# catch err +# warn("WebSockets.HTTP.upgrade: Caught unhandled error while calling argument function f, the handler / gatekeeper:\n\t") +# mt = typeof(f).name.mt +# fnam = splitdir(string(mt.defs.func.file))[2] +# print_with_color(:yellow, STDERR, "f = ", string(f) * " at " * fnam * ":" * string(mt.defs.func.line) * "\nERROR:\t") +# showerror(STDERR, err, catch_stacktrace()) finally close(ws) end @@ -240,7 +241,7 @@ mutable struct ServerWS{T <: HTTP.Servers.Scheme, H <: HTTP.Handler, W <: Websoc out::Channel{Any} options::HTTP.ServerOptions - ServerWS{T, H, W}(handler::H, wshandler::W, logger::IO = HTTP.compat_stdout(), ch=Channel(1), ch2=Channel(1), + ServerWS{T, H, W}(handler::H, wshandler::W, logger::IO = HTTP.compat_stdout(), ch=Channel(1), ch2=Channel(2), options=HTTP.ServerOptions()) where {T, H, W} = new{T, H, W}(handler, wshandler, logger, ch, ch2, options) end @@ -255,18 +256,30 @@ function ServerWS(handler::H, key::String = "", args...) where {H <: HTTP.Handler, W <: WebsocketHandler} if cert != "" && key != "" - serverws = ServerWS{HTTP.Servers.https, H, W}(handler, wshandler, logger, Channel(1), Channel(1), HTTP.ServerOptions(; sslconfig=HTTP.MbedTLS.SSLConfig(cert, key), args...)) + serverws = ServerWS{HTTP.Servers.https, H, W}(handler, wshandler, logger, Channel(1), Channel(2), HTTP.ServerOptions(; sslconfig=HTTP.MbedTLS.SSLConfig(cert, key), args...)) else - serverws = ServerWS{HTTP.Servers.http, H, W}(handler, wshandler, logger, Channel(1), Channel(1), HTTP.ServerOptions(; args...)) + serverws = ServerWS{HTTP.Servers.http, H, W}(handler, wshandler, logger, Channel(1), Channel(2), HTTP.ServerOptions(; args...)) end return serverws end """ A variant of HTTP.serve with the WebSockets.ServerWS type. +Puts any caught error and stacktrace on the serve.out channel. ```julia @shedule WebSockets.serve(myServerWS, "127.0.0.1", 8080, false) ``` +After a suspected 'connection task' failure: +```julia + if isready(myserver_WS.out) + err = take!(myserver_WS.out) + @test typeof(err) <: WebSocketClosedError + end + if isready(myserver_WS.out) + stack_trace = take!(server_WS.out) + end +``` + """ function serve(server::ServerWS{T, H, W}, host, port, verbose) where {T, H, W} @@ -291,12 +304,16 @@ function serve(server::ServerWS{T, H, W}, host, port, verbose) where {T, H, W} (tcp; kw...) -> true, ratelimits = Dict{IPAddr, HTTP.Servers.RateLimit}(), ratelimit = server.options.ratelimit) do stream::HTTP.Stream - - if is_upgrade(stream.message) - upgrade(server.wshandler.func, stream) - else - HTTP.Servers.handle_request(server.handler.func, stream) - end - end + try + if is_upgrade(stream.message) + upgrade(server.wshandler.func, stream) + else + HTTP.Servers.handle_request(server.handler.func, stream) + end + catch err + put!(server.out, err) + put!(server.out, catch_stacktrace()) + end + end return end \ No newline at end of file diff --git a/src/WebSockets.jl b/src/WebSockets.jl index 746f4cd..093edaf 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -34,7 +34,8 @@ export WebSocket, target, origin, send_ping, - send_pong + send_pong, + WebSocketClosedError const TCPSock = Base.TCPSocket "A reasonable amount of time" @@ -206,20 +207,25 @@ send_pong(ws, data...) = write_pong(ws.socket, !ws.server, data...) """ close(ws::WebSocket) - close(ws::WebSocket, statusnumber) + close(ws::WebSocket, statusnumber = n) + close(ws::WebSocket, statusnumber = n, freereason = "my reason") Send an OPCODE_CLOSE frame, and wait for the same response or until a reasonable amount of time, $(round(TIMEOUT_CLOSEHANDSHAKE, 1)) s, has passed. Data received while closing is dropped. -Status codes according to RFC 6455 7.4.1 can be included, see WebSockets.codeDesc +Status number n according to RFC 6455 7.4.1 can be included, see WebSockets.codeDesc """ -function Base.close(ws::WebSocket; statusnumber = 0) +function Base.close(ws::WebSocket; statusnumber = 0, freereason = "") if isopen(ws) ws.state = CLOSING if statusnumber == 0 locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, UInt8[]) - else + elseif freereason == "" statuscode = reinterpret(UInt8, [hton(UInt16(statusnumber))]) - locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, statuscode) + locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, copy(statuscode)) + else + statuscode = vcat(reinterpret(UInt8, [hton(UInt16(statusnumber))]), + Vector{UInt8}(freereason)) + locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, copy(statuscode)) end # Wait till the peer responds with an OPCODE_CLOSE while discarding any @@ -320,7 +326,6 @@ is_control_frame(msg::WebSocketFragment) = (msg.opcode & 0b0000_1000) > 0 """ Respond to pings, ignore pongs, respond to close.""" function handle_control_frame(ws::WebSocket, wsf::WebSocketFragment) if wsf.opcode == OPCODE_CLOSE - #info("ws|$(ws.server ? "server" : "client") received OPCODE_CLOSE") ws.state = CLOSED try locked_write(ws.socket, true, OPCODE_CLOSE, !ws.server, UInt8[]) @@ -336,9 +341,7 @@ function handle_control_frame(ws::WebSocket, wsf::WebSocketFragment) scode = Int(reinterpret(UInt16, reverse(wsf.data[1:2]))[1]) reason = string(scode) * ":" * String(wsf.data[3:end]) end - #info(reason) - throw("ws|$(ws.server ? "server" : "client") received OPCODE_CLOSE, replied with same. " * - "Received reason: " * reason) + throw(WebSocketClosedError("ws|$(ws.server ? "server" : "client") respond to OPCODE_CLOSE " * reason)) elseif wsf.opcode == OPCODE_PING info("ws|$(ws.server ? "server" : "client") received OPCODE_PING") send_pong(ws, wsf.data) @@ -357,8 +360,7 @@ function read_frame(ws::WebSocket) #= Browsers will seldom close in the middle of writing to a socket, but other clients often do, and the stacktraces can be very long. - ab can be assigned, but of length 1. An enclosing try..catch in the calling function - seems to + ab can be assigned, but of length 1. Use an enclosing try..catch in the calling function =# a = ab[1] fin = a & 0b1000_0000 >>> 7 # If fin, then is final fragment @@ -371,7 +373,7 @@ function read_frame(ws::WebSocket) mask = b & 0b1000_0000 >>> 7 hasmask = mask != 0 - if mask != ws.server + if hasmask != ws.server if ws.server msg = "WebSocket|server cannot handle incoming messages without mask. Ref. rcf6455 5.3" else @@ -429,13 +431,14 @@ function Base.read(ws::WebSocket) try errtyp = typeof(err) if errtyp <: InterruptException + msg = " while read(ws|$(ws.server ? "server" : "client") received InterruptException." # This exception originates from this side. Follow close protocol so as not to irritate the other side. - close(ws) - throw(WebSocketClosedError(" while read(ws|$(ws.server ? "server" : "client") received local interrupt exception. Performed closing handshake.")) + close(ws, statusnumber = 1006, freereason = msg) + throw(WebSocketClosedError(msg * " Performed closing handshake.")) elseif errtyp <: WebSocketError # This exception originates on the other side. Follow close protocol with reason. close(ws, statusnumber = err.status) - throw(WebSocketClosedError(" while read(ws|$(ws.server ? "server" : "client") $(err.message) - Performed closing handshake.")) + throw(WebSocketClosedError(" while read(ws|$(ws.server ? "server" : "client")) $(err.message) - Performed closing handshake.")) elseif errtyp <: Base.UVError || errtyp <: Base.BoundsError || errtyp <: Base.EOFError || diff --git a/test/client_server_test.jl b/test/client_server_test.jl index a7b973b..1b17a76 100644 --- a/test/client_server_test.jl +++ b/test/client_server_test.jl @@ -21,16 +21,18 @@ const port_HttpServer = 8081 const TCPREF = Ref{Base.TCPServer}() # Start HTTP listen server on port $port_HTTP" -@schedule HTTP.listen("127.0.0.1", port_HTTP, tcpref = TCPREF) do s +tas = @schedule HTTP.listen("127.0.0.1", port_HTTP, tcpref = TCPREF) do s if WebSockets.is_upgrade(s.message) WebSockets.upgrade(echows, s) end end +while !istaskstarted(tas);yield();end # Start HttpServer on port $port_HttpServer const server = Server(HttpHandler((req, res)->Response(200)), WebSocketHandler(echows)); -@schedule run(server, port_HttpServer) +tas = @schedule run(server, port_HttpServer) +while !istaskstarted(tas);yield();end # Start HTTP ServerWS on port $port_HTTP_ServeWS @@ -38,10 +40,9 @@ server_WS = WebSockets.ServerWS( HTTP.HandlerFunction(req-> HTTP.Response(200)), WebSockets.WebsocketHandler(echows)) -@schedule WebSockets.serve(server_WS, "127.0.0.1", port_HTTP_ServeWS, false) +tas = @schedule WebSockets.serve(server_WS, "127.0.0.1", port_HTTP_ServeWS, false) +while !istaskstarted(tas);yield();end - -sleep(4) const servers = [ ("HTTP", "ws://127.0.0.1:$(port_HTTP)"), ("HttpServer", "ws://127.0.0.1:$(port_HttpServer)"), @@ -66,7 +67,6 @@ for (s, url) in servers, len in lengths, closestatus in [false, true] write(ws, test_str) @test String(read(ws)) == forcecopy_str closestatus && close(ws, statusnumber = 1000) - sleep(0.2) end sleep(0.2) end diff --git a/test/client_test.jl b/test/client_test.jl index d4232d3..be89b91 100644 --- a/test/client_test.jl +++ b/test/client_test.jl @@ -1,4 +1,5 @@ # included in runtests.jl +# focus on HTTP.jl using Base.Test using HTTP import WebSockets: is_upgrade, @@ -12,21 +13,24 @@ sethd(r::HTTP.Messages.Response, pa::Pair) = HTTP.Messages.setheader(r, HTTP.Hea const NEWPORT = 8091 const TCPREF2 = Ref{Base.TCPServer}() - +# start HTTP server addsubproto("xml") -@schedule HTTP.listen("127.0.0.1", NEWPORT, tcpref = TCPREF2) do s +tas = @schedule HTTP.listen("127.0.0.1", NEWPORT, tcpref = TCPREF2) do s if WebSockets.is_upgrade(s.message) WebSockets.upgrade((_)->nothing, s) end end -sleep(3) +while !istaskstarted(tas);yield();end + # open client with approved subprotocol -const URL = "ws://127.0.0.1:$NEWPORT" +URL = "ws://127.0.0.1:$NEWPORT" res = WebSockets.open((_)->nothing, URL, subprotocol = "xml"); @test res.status == 101 + # open with unknown subprotocol res = WebSockets.open((_)->nothing, URL, subprotocol = "unapproved"); @test res.status == 400 + # try open with uknown port caughterr = WebSockets.WebSocketClosedError("") try @@ -43,11 +47,14 @@ end WebSockets.open("ws://127.0.0.1:$(NEWPORT)") do ws close(ws.socket) end + # check that the server is still running regardless res = WebSockets.open((_)->nothing, URL); @test res.status == 101 + # Open with a ws client handler that throws a domain error @test_throws DomainError WebSockets.open((_)->sqrt(-2), URL); + # Stop the TCP server close(TCPREF2[]) @@ -68,7 +75,6 @@ function dummywsh(dws::WebSockets.WebSocket{BufferStream}) close(dws.socket) close(dws) end - @test _openstream(dummywsh, s, key) == WebSockets.CLOSED # emulate an incorrect first accept response from server diff --git a/test/error_test.jl b/test/error_test.jl index 6c2f0c3..0c41174 100644 --- a/test/error_test.jl +++ b/test/error_test.jl @@ -1,57 +1,214 @@ # included in runtests.jl -# As per June 2018, there is no -# way we can process errors thrown -# by tasks generated by HttpServer -# or by HTTP, for example ECONNRESET. -# -# Try to provide user with relevant and -# concise output. using Base.Test import HTTP +import HttpServer: HttpHandler, + Server import WebSockets: ServerWS, serve, - open + open, + readguarded, + writeguarded, + WebSocketHandler, + WebSocketClosedError, + close + const THISPORT = 8092 -info("Start a ws|server which doesn't read much at all. Close from client side. The - close handshake is aborted after $(WebSockets.TIMEOUT_CLOSEHANDSHAKE) seconds...") -server_WS = WebSockets.ServerWS( - HTTP.HandlerFunction(req-> HTTP.Response(200)), - WebSockets.WebsocketHandler(ws-> sleep(16))) -@schedule WebSockets.serve(server_WS, "127.0.0.1", THISPORT, false) -sleep(3) URL = "ws://127.0.0.1:$THISPORT" + +info("Start a HTTP server with a ws handler that is unresponsive. Close from client side. The + close handshake aborts after $(WebSockets.TIMEOUT_CLOSEHANDSHAKE) seconds...") +server_WS = ServerWS( HTTP.HandlerFunction(req-> HTTP.Response(200)), + WebSockets.WebsocketHandler(ws-> sleep(16))) +tas = @schedule WebSockets.serve(server_WS, "127.0.0.1", THISPORT, false) +while !istaskstarted(tas); yield(); end res = WebSockets.open((_)->nothing, URL); @test res.status == 101 put!(server_WS.in, HTTP.Servers.KILL) - -#= -Activate test when there is a decent way to capture errors -in tasks generated by server package - -server_WS = WebSockets.ServerWS(HTTP.HandlerFunction(req-> HTTP.Response(200)), - WebSockets.WebsocketHandler() do ws_serv +server_WS = ServerWS( HTTP.HandlerFunction(req-> HTTP.Response(200)), + WebSockets.WebsocketHandler() do req, ws_serv while isopen(ws_serv) readguarded(ws_serv) end - end - ) -@schedule WebSockets.serve(server_WS, "127.0.0.1", THISPORT, false) -# attempt to read from closed websocket. + end); +tas = @schedule WebSockets.serve(server_WS, "127.0.0.1", THISPORT, false) +while !istaskstarted(tas); yield(); end +sleep(1) +# attempt to read guarded from closed websocket WebSockets.open(URL) do ws_client close(ws_client) - try - read(ws_client) - catch - end + @test (UInt8[], false) == readguarded(ws_client) end; +sleep(1) -function wsh(ws) - close(ws) - read(ws) +# attempt to write guarded to closed websocket +WebSockets.open(URL) do ws_client + close(ws_client) + @test false == writeguarded(ws_client, "writethis") +end; +sleep(1) + +# attempt to read from closed websocket +try + WebSockets.open(URL) do ws_client + close(ws_client) + read(ws_client) + end +catch err + @test typeof(err) <: ErrorException + @test err.msg == "Attempt to read from closed WebSocket|client. First isopen(ws), or use readguarded(ws)!" end -# improve output of error, the details don't emerge -WebSockets.open(wsh, URL) +sleep(1) + -=# +info("Attempt to write to a closed websocket, served by HttpServer (this takes some time, there is no check + in WebSockets against it") +try + WebSockets.open(URL) do ws_client + close(ws_client) + write(ws_client, "writethis") + end +catch err + @test typeof(err) <: HTTP.IOExtras.IOError +end + +put!(server_WS.in, HTTP.Servers.KILL) + + +# Start a HttpServer +server = Server(HttpHandler() do req, res + Response(200) + end, + WebSocketHandler() do req, ws_serv + while isopen(ws_serv) + readguarded(ws_serv) + end + end) +tas = @schedule run(server, THISPORT) +while !istaskstarted(tas) + yield() +end +sleep(3) +try + WebSockets.open(URL) do ws_client + close(ws_client) + write(ws_client, "writethis") + end +catch err + @test typeof(err) <: HTTP.IOExtras.IOError +end +close(server) +sleep(1) + +# Capture ws|server handler errors in user's handler while async. +# This debug info might otherwise get lost when using listen(..) directly. +chfromserv=Channel(2) +server_WS = ServerWS( HTTP.HandlerFunction(req-> HTTP.Response(200)), + WebSockets.WebsocketHandler() do ws_serv + while isopen(ws_serv) + try + read(ws_serv) + catch err + put!(chfromserv, err) + put!(chfromserv, catch_stacktrace()[1:2]) + end + end + end); +sleep(3) + +tas = @schedule WebSockets.serve(server_WS, "127.0.0.1", THISPORT, false) +while !istaskstarted(tas); yield(); end +sleep(1) +res = WebSockets.open((ws)-> close(ws.socket), URL) +@test res.status == 101 +sleep(1) +wait(chfromserv) +err = take!(chfromserv) +@test typeof(err) <: WebSocketClosedError +@test err.message == " while read(ws|server) BoundsError(UInt8[], (1,))" +wait(chfromserv) +stack_trace = take!(chfromserv) +@test length(stack_trace) == 2 +put!(server_WS.in, HTTP.Servers.KILL) +sleep(1) + +# Capture ws|server side error "automatically" +server_WS = ServerWS( HTTP.HandlerFunction(req-> HTTP.Response(200)), + WebSockets.WebsocketHandler() do ws_serv + while isopen(ws_serv) + read(ws_serv) + end + end); +tas = @schedule WebSockets.serve(server_WS, "127.0.0.1", THISPORT, false) +while !istaskstarted(tas); yield(); end +sleep(3) + +# Close out of protocol +WebSockets.open((ws)-> close(ws.socket), URL); +sleep(1) +wait(server_WS.out) +err = take!(server_WS.out) +@test typeof(err) <: WebSocketClosedError +@test err.message == " while read(ws|server) BoundsError(UInt8[], (1,))" +wait(server_WS.out) +stack_trace = take!(server_WS.out); +@test length(stack_trace) == 6 + + +while isready(server_WS.out) + take!(server_WS.out) +end +sleep(1) + +# Close using Status codes according to RFC 6455 7.4.1 +for (ke, va) in WebSockets.codeDesc + info("Closing ws|client with reason ", ke, " ", va) + sleep(0.3) + WebSockets.open((ws)-> close(ws, statusnumber = ke), URL) + wait(server_WS.out) + err = take!(server_WS.out) + @test typeof(err) <: WebSocketClosedError + @test err.message == "ws|server respond to OPCODE_CLOSE $ke:$va" + wait(server_WS.out) + stacktra = take!(server_WS.out) + @test length(stacktra) == 0 + while isready(server_WS.out) + take!(server_WS.out) + end + sleep(1) +end + +# Close with a given reason +va = 1000 +info("Closing ws|client with reason", va, " ", WebSockets.codeDesc[va], " and goodbye!") +WebSockets.open((ws)-> close(ws, statusnumber = 1000, freereason = "goodbye!"), URL) +wait(server_WS.out) +err = take!(server_WS.out) +@test typeof(err) <: WebSocketClosedError +@test err.message == "ws|server respond to OPCODE_CLOSE 1000:goodbye!" +stack_trace = take!(server_WS.out) +sleep(1) + +println() +info(" React to an InterruptException with a closing handshake. + A lot of error text will spill over into REPL, but the test is unaffected") +println() + +function selfinterruptinghandler(ws) + task = @schedule WebSockets.open((ws)-> read(ws), URL) + sleep(3) + @schedule Base.throwto(task, InterruptException()) + sleep(1) + nothing +end +WebSockets.open(selfinterruptinghandler, URL) +sleep(6) +wait(server_WS.out) +err = take!(server_WS.out) +@test typeof(err) <: WebSocketClosedError +@test err.message == "ws|server respond to OPCODE_CLOSE 1006: while read(ws|client received InterruptException." +wait(server_WS.out) +stack_trace = take!(server_WS.out) +put!(server_WS.in, HTTP.Servers.KILL) +sleep(1) diff --git a/test/frametest.jl b/test/frametest.jl index bfeabab..a5a10de 100644 --- a/test/frametest.jl +++ b/test/frametest.jl @@ -3,16 +3,25 @@ using Base.Test import WebSockets: maskswitch!, write_fragment, read_frame, - WebSocket + is_control_frame, + handle_control_frame, + WebSocket, + WebSocketClosedError, + locked_write, + WebSockets.codeDesc """ The dummy websocket don't use TCP. Close won't work, but we can manipulate the contents using otherwise the same functions as TCP sockets. """ dummyws(server::Bool) = WebSocket(BufferStream(), server) - - io = IOBuffer() + +# maskswitch +empty1 = UInt8[] +empty2 = UInt8[] +@test length(maskswitch!(empty1)) == 4 +@test empty1 == empty2 # Test most basic frame, length <126 for len = [8, 125], fin=[true, false], clientwriting = [false, true] @@ -137,3 +146,200 @@ for clientwriting = [false, true] @test frag_back.payload_len == len @test test_str == String(frag_back.data) end + +# Test unknown opcodes + +for op in 0xB:0xF + clientwriting = false + len = 10 + fin = true + + test_str = randstring(len) + write_fragment(io, fin, op, clientwriting, copy(Vector{UInt8}(test_str))) + frame = take!(io) + + dws = dummyws(clientwriting) + write(dws.socket, frame) + frag_back = read_frame(dws) + + @test is_control_frame(frag_back) + thiserror = ArgumentError("") + try + handle_control_frame(dws, frag_back) + catch err + thiserror = err + end + @test typeof(thiserror) <: ErrorException + @test thiserror.msg == " while handle_control_frame(ws|client, wsf): Unknown opcode $op" + + close(dws.socket) +end + + +# Test multi-frame message + +for clientwriting = [false, true] + + op = WebSockets.OPCODE_TEXT + full_str = "123456" + first_str = "123" + second_str = "456" + fin = false + dws = dummyws(clientwriting) + write_fragment(dws.socket, fin, op, clientwriting, copy(Vector{UInt8}(first_str))) + fin = true + write_fragment(dws.socket, fin, op, clientwriting, copy(Vector{UInt8}(second_str))) + + @test read(dws) == Vector{UInt8}(full_str) +end + + + + +# Test read(ws) bad mask error handling + +info("Provoking close handshake from protocol error without a peer. Waits a reasonable time") +for clientwriting in [false, true] + op = WebSockets.OPCODE_TEXT + test_str = "123456" + fin = true + write_fragment(io, fin, op, clientwriting, copy(Vector{UInt8}(test_str))) + frame = take!(io) + # let's put this frame on the same kind of socket, and then read it as if it came from the peer + # This will provoke a close handshake, but since there is no peer it times out. + dws = dummyws(!clientwriting) + write(dws.socket, frame) + thiserror = ArgumentError("") + try + read(dws) + catch err + thiserror = err + end + @test typeof(thiserror) <: WebSocketClosedError + expmsg = " while read(ws|$(dws.server ? "server" : "client")) WebSocket|$(dws.server ? "server" : "client") cannot handle incoming messages with$(dws.server ? "out" : "") mask. Ref. rcf6455 5.3 - Performed closing handshake." + @test thiserror.message == expmsg + @test !isopen(dws) + close(dws.socket) + + # simple close frame + clientwriting = false +end + + + +# Close frame with no reason. +for clientwriting = [false, true] + fin = true + op = WebSockets.OPCODE_CLOSE + write_fragment(io, fin, op, clientwriting, UInt8[]) + frame = take!(io) + len = 0 + # Check the frame header + # Last frame bit + @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] + # payload length bit + @test frame[2] & 0b0111_1111 == len + # ismasked bit + hasmsk = frame[2] & 0b1000_0000 >>> 7 != 0 + @test hasmsk == clientwriting + # the peer of the writer is + dws = dummyws(clientwriting) + write(dws.socket, frame) + frag_back = read_frame(dws) + @test frag_back.is_last == fin + @test frag_back.opcode == op + @test frag_back.is_masked == clientwriting + @test frag_back.payload_len == len + @test is_control_frame(frag_back) +end + + +# Close with no reason +for clientwriting in [false, true] + op = WebSockets.OPCODE_CLOSE + fin = true + thisws = dummyws(!clientwriting) + locked_write(thisws.socket, true, op, !thisws.server, UInt8[]) + close(thisws.socket) + frame = read(thisws.socket) + peerws = dummyws(clientwriting) + write(peerws.socket, frame) + close(peerws.socket) + wsf = read_frame(peerws) + @test is_control_frame(wsf) + @test wsf.opcode == WebSockets.OPCODE_CLOSE + @test wsf.payload_len == 0 +end + +# Close with status number +for clientwriting in [false, true], statusnumber in keys(codeDesc) + op = WebSockets.OPCODE_CLOSE + freereason = "" + fin = true + thisws = dummyws(!clientwriting) + statuscode = reinterpret(UInt8, [hton(UInt16(statusnumber))]) + locked_write(thisws.socket, true, op, !thisws.server, statuscode) + close(thisws.socket) + frame = read(thisws.socket) + + # Check the frame header + # Last frame bit + @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] + # payload length bit + @test frame[2] & 0b0111_1111 == 2 + # ismasked bit + hasmsk = frame[2] & 0b1000_0000 >>> 7 != 0 + @test hasmsk == clientwriting + # the peer of the writer is + peerws = dummyws(clientwriting) + write(peerws.socket, frame) + close(peerws.socket) + wsf = read_frame(peerws) + @test is_control_frame(wsf) + @test wsf.opcode == WebSockets.OPCODE_CLOSE + @test wsf.payload_len == 2 + scode = Int(reinterpret(UInt16, reverse(wsf.data))[1]) + @test scode == statusnumber + reason = string(scode) * ":" * get(codeDesc, scode, "") +end + +# Close with status number and freereason +for clientwriting in [false, true], statusnumber in keys(codeDesc) + freereason = "q.e.d" + op = WebSockets.OPCODE_CLOSE + fin = true + thisws = dummyws(!clientwriting) + statuscode = vcat(reinterpret(UInt8, [hton(UInt16(statusnumber))]), + Vector{UInt8}(freereason)) + locked_write(thisws.socket, true, op, !thisws.server, copy(statuscode)) + close(thisws.socket) + frame = read(thisws.socket) + + # Check the frame header + # Last frame bit + @test bits(frame[1]) == (fin ? "1" : "0") * "000" * bits(op)[end-3:end] + # payload length bit + @test frame[2] & 0b0111_1111 == length(statuscode) + # ismasked bit + hasmsk = frame[2] & 0b1000_0000 >>> 7 != 0 + @test hasmsk == clientwriting + # the peer of the writer is + peerws = dummyws(clientwriting) + write(peerws.socket, frame) + close(peerws.socket) + wsf = read_frame(peerws) + @test is_control_frame(wsf) + @test wsf.opcode == WebSockets.OPCODE_CLOSE + @test wsf.payload_len == length(statuscode) + scode = Int(reinterpret(UInt16, reverse(wsf.data[1:2]))[1]) + reason = string(scode) * ":" * + get(codeDesc, scode, "") * + " " * String(wsf.data[3:end]) + @test reason == string(statusnumber) * ":" * + get(codeDesc, statusnumber, "") * + " " * freereason +end + + +close(io) + diff --git a/test/handler_functions_events.jl b/test/handler_functions_events.jl index 2f1bd37..1446203 100644 --- a/test/handler_functions_events.jl +++ b/test/handler_functions_events.jl @@ -5,7 +5,6 @@ function ev_error(client::Client, args...) end function ev_listen(port, args...) id = "events.ev_listen\t" - #clog(id, :yellow, " from port ", :bold, port , :normal, " ", args..., " that was all I think. \n") nothing end function ev_connect(client::Client, args...) diff --git a/test/handshaketest.jl b/test/handshaketest.jl index ca2872a..26c26f9 100644 --- a/test/handshaketest.jl +++ b/test/handshaketest.jl @@ -114,13 +114,13 @@ for r in templaterequests() @test handshakeresponse(r) == REJECT end -info("add simple subprotocol to acceptable list") +# add simple subprotocol to acceptable list @test true == WebSockets.addsubproto("xml") -info("add subprotocol with difficult name") +# add subprotocol with difficult name @test true == WebSockets.addsubproto("my.server/json-zmq") -info("Test handshake subprotocol now acceptable") +# "Test handshake subprotocol now acceptable" for r in templaterequests() sethd(r, "Sec-WebSocket-Version" => "13") sethd(r, "Sec-WebSocket-Key" => "16 bytes key this is surely") diff --git a/test/runtests.jl b/test/runtests.jl index cf99bad..2e77ad4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,36 +1,37 @@ using Base.Test cd(Pkg.dir("WebSockets", "test")) -# WebSockets.jl -@testset "Fragment and frame unit tests" begin - include("frametest.jl") -end @sync yield() # avoid mixing of output with possible deprecation warnings from .juliarc -info("Starting test WebSockets...") -@testset "Unit test, HttpServer and HTTP handshake" begin - include("handshaketest.jl") +@testset "WebSockets" begin +info("\nFragment and unit tests\n") +@testset "Fragment and frame unit tests" begin + include("frametest.jl");sleep(1) +end + +info("\nHttpServer and HTTP handshake\n") +@testset "HttpServer and HTTP handshake" begin + include("handshaketest.jl");sleep(1) end +info("\nClient-server test, HTTP client\n") @testset "Client-server test, HTTP client" begin - include("client_server_test.jl") + include("client_server_test.jl");sleep(1) end + +info("\nClient test, HTTP client\n") @testset "Client test, HTTP client" begin - include("client_test.jl") + include("client_test.jl");sleep(1) end - -@testset "WebSockets abrupt close & bad timing test" begin - include("error_test.jl") +info("\nAbrupt close & error handling test\n") +@testset "Abrupt close & error handling test" begin + include("error_test.jl");sleep(1) end +@testset "Bad frames, multiframes, loaded ping" begin + include("error_test.jl");sleep(1) +end +end # TODO -# WebSockets.jl -# direct closing of tcp socket, while reading. -# closing with given reason (only from browsertests) -# unknown opcode -# Read multiple frames (use dummyws), may require change -# InterruptException -# Protocol error (not masked from client) -# writeguarded, error # restructure browsertests From a4279f6ba482a8cc1eb7345f29f653700e85d4a2 Mon Sep 17 00:00:00 2001 From: hustf Date: Sun, 3 Jun 2018 19:51:30 +0200 Subject: [PATCH 33/35] Info during tests, more sleeping mod: test/client_test.jl mod: test/error_test.jl mod: test/runtests.jl --- test/client_test.jl | 36 ++++++++++++------- test/error_test.jl | 84 +++++++++++++++++++++++++++------------------ test/runtests.jl | 3 -- 3 files changed, 73 insertions(+), 50 deletions(-) diff --git a/test/client_test.jl b/test/client_test.jl index be89b91..0b66a4b 100644 --- a/test/client_test.jl +++ b/test/client_test.jl @@ -13,7 +13,8 @@ sethd(r::HTTP.Messages.Response, pa::Pair) = HTTP.Messages.setheader(r, HTTP.Hea const NEWPORT = 8091 const TCPREF2 = Ref{Base.TCPServer}() -# start HTTP server +info("start HTTP server\n") +sleep(1) addsubproto("xml") tas = @schedule HTTP.listen("127.0.0.1", NEWPORT, tcpref = TCPREF2) do s if WebSockets.is_upgrade(s.message) @@ -22,16 +23,19 @@ tas = @schedule HTTP.listen("127.0.0.1", NEWPORT, tcpref = TCPREF2) do s end while !istaskstarted(tas);yield();end -# open client with approved subprotocol +info("open client with approved subprotocol\n") +sleep(1) URL = "ws://127.0.0.1:$NEWPORT" res = WebSockets.open((_)->nothing, URL, subprotocol = "xml"); @test res.status == 101 -# open with unknown subprotocol +info("open with unknown subprotocol\n") +sleep(1) res = WebSockets.open((_)->nothing, URL, subprotocol = "unapproved"); @test res.status == 400 -# try open with uknown port +info("try open with uknown port\n") +sleep(1) caughterr = WebSockets.WebSocketClosedError("") try WebSockets.open((_)->nothing, "ws://127.0.0.1:8099"); @@ -41,24 +45,29 @@ end @test typeof(caughterr) <: WebSockets.WebSocketClosedError @test caughterr.message == " while open ws|client: connect: connection refused (ECONNREFUSED)" -# start a client websocket that irritates by closing the TCP stream -# connection without a websocket closing handshake. This -# throws an error in the server task +info("start a client websocket that irritates by closing the TCP stream + connection without a websocket closing handshake. This + throws an error in the server task\n") +sleep(1) WebSockets.open("ws://127.0.0.1:$(NEWPORT)") do ws close(ws.socket) end -# check that the server is still running regardless +info("check that the server is still running regardless\n") +sleep(1) res = WebSockets.open((_)->nothing, URL); @test res.status == 101 -# Open with a ws client handler that throws a domain error +info("Open with a ws client handler that throws a domain error\n") +sleep(1) @test_throws DomainError WebSockets.open((_)->sqrt(-2), URL); -# Stop the TCP server +info("Stop the TCP server\n") +sleep(1) close(TCPREF2[]) - -# emulate a correct first accept response from server +sleep(1) +info("Emulate a correct first accept response from server, with BufferStream socket\n") +sleep(1) req = HTTP.Messages.Request() req.method = "GET" key = base64encode(rand(UInt8, 16)) @@ -77,7 +86,8 @@ function dummywsh(dws::WebSockets.WebSocket{BufferStream}) end @test _openstream(dummywsh, s, key) == WebSockets.CLOSED -# emulate an incorrect first accept response from server +info("emulate an incorrect first accept response from server\n") +sleep(1) sethd(resp, "Sec-WebSocket-Accept" => generate_websocket_key(base64encode(rand(UInt8, 16)))) write(servsock, resp) @test_throws WebSockets.WebSocketError _openstream(dummywsh, s, key) diff --git a/test/error_test.jl b/test/error_test.jl index 0c41174..f165b79 100644 --- a/test/error_test.jl +++ b/test/error_test.jl @@ -17,15 +17,22 @@ import WebSockets: ServerWS, const THISPORT = 8092 URL = "ws://127.0.0.1:$THISPORT" -info("Start a HTTP server with a ws handler that is unresponsive. Close from client side. The - close handshake aborts after $(WebSockets.TIMEOUT_CLOSEHANDSHAKE) seconds...") + +info("Start a HTTP server with a ws handler that is unresponsive. Close from client side. The " * + " close handshake aborts after $(WebSockets.TIMEOUT_CLOSEHANDSHAKE) seconds...\n") +sleep(1) server_WS = ServerWS( HTTP.HandlerFunction(req-> HTTP.Response(200)), WebSockets.WebsocketHandler(ws-> sleep(16))) tas = @schedule WebSockets.serve(server_WS, "127.0.0.1", THISPORT, false) while !istaskstarted(tas); yield(); end +sleep(1) res = WebSockets.open((_)->nothing, URL); @test res.status == 101 put!(server_WS.in, HTTP.Servers.KILL) + + +info("Start a HTTP server with a ws handler that always reads guarded.\n") +sleep(1) server_WS = ServerWS( HTTP.HandlerFunction(req-> HTTP.Response(200)), WebSockets.WebsocketHandler() do req, ws_serv while isopen(ws_serv) @@ -35,21 +42,27 @@ server_WS = ServerWS( HTTP.HandlerFunction(req-> HTTP.Response(200)), tas = @schedule WebSockets.serve(server_WS, "127.0.0.1", THISPORT, false) while !istaskstarted(tas); yield(); end sleep(1) -# attempt to read guarded from closed websocket + +info("Attempt to read guarded from a closing ws|client. Check for return false.\n") +sleep(1) WebSockets.open(URL) do ws_client close(ws_client) @test (UInt8[], false) == readguarded(ws_client) end; sleep(1) -# attempt to write guarded to closed websocket + +info("Attempt to write guarded from a closing ws|client. Check for return false.\n") +sleep(1) WebSockets.open(URL) do ws_client close(ws_client) @test false == writeguarded(ws_client, "writethis") end; sleep(1) -# attempt to read from closed websocket + +info("Attempt to read from closing ws|client. Check caught error.\n") +sleep(1) try WebSockets.open(URL) do ws_client close(ws_client) @@ -62,8 +75,9 @@ end sleep(1) -info("Attempt to write to a closed websocket, served by HttpServer (this takes some time, there is no check - in WebSockets against it") +info("Attempt to write to a closing ws|client (this takes some time, there is no check + in WebSockets against it). Check caught error.\n") +sleep(1) try WebSockets.open(URL) do ws_client close(ws_client) @@ -76,7 +90,8 @@ end put!(server_WS.in, HTTP.Servers.KILL) -# Start a HttpServer +info("\n\nStart a HttpServer\n") +sleep(1) server = Server(HttpHandler() do req, res Response(200) end, @@ -86,10 +101,10 @@ server = Server(HttpHandler() do req, res end end) tas = @schedule run(server, THISPORT) -while !istaskstarted(tas) - yield() -end +while !istaskstarted(tas);yield();end sleep(3) +info("Attempt to write to a closing ws|client, served by HttpServer (this takes some time, there is no check + in WebSockets against it). Check caught error.") try WebSockets.open(URL) do ws_client close(ws_client) @@ -101,8 +116,9 @@ end close(server) sleep(1) -# Capture ws|server handler errors in user's handler while async. -# This debug info might otherwise get lost when using listen(..) directly. + +info("\nStart an async HTTP server. The wshandler use global channels for inspecting caught errors.\n") +sleep(1) chfromserv=Channel(2) server_WS = ServerWS( HTTP.HandlerFunction(req-> HTTP.Response(200)), WebSockets.WebsocketHandler() do ws_serv @@ -115,25 +131,24 @@ server_WS = ServerWS( HTTP.HandlerFunction(req-> HTTP.Response(200)), end end end); -sleep(3) - tas = @schedule WebSockets.serve(server_WS, "127.0.0.1", THISPORT, false) while !istaskstarted(tas); yield(); end -sleep(1) +sleep(3) + +info("Open a ws|client, close it out of protocol. Check server error on channel.\n") res = WebSockets.open((ws)-> close(ws.socket), URL) @test res.status == 101 sleep(1) -wait(chfromserv) err = take!(chfromserv) @test typeof(err) <: WebSocketClosedError @test err.message == " while read(ws|server) BoundsError(UInt8[], (1,))" -wait(chfromserv) stack_trace = take!(chfromserv) @test length(stack_trace) == 2 put!(server_WS.in, HTTP.Servers.KILL) sleep(1) -# Capture ws|server side error "automatically" +info("\nStart an async HTTP server. Errors are output on built-in channel\n") +sleep(1) server_WS = ServerWS( HTTP.HandlerFunction(req-> HTTP.Response(200)), WebSockets.WebsocketHandler() do ws_serv while isopen(ws_serv) @@ -144,24 +159,24 @@ tas = @schedule WebSockets.serve(server_WS, "127.0.0.1", THISPORT, false) while !istaskstarted(tas); yield(); end sleep(3) -# Close out of protocol -WebSockets.open((ws)-> close(ws.socket), URL); +info("Open a ws|client, close it out of protocol. Check server error on server.out channel.\n") sleep(1) -wait(server_WS.out) +WebSockets.open((ws)-> close(ws.socket), URL); err = take!(server_WS.out) @test typeof(err) <: WebSocketClosedError @test err.message == " while read(ws|server) BoundsError(UInt8[], (1,))" -wait(server_WS.out) stack_trace = take!(server_WS.out); @test length(stack_trace) == 6 - while isready(server_WS.out) take!(server_WS.out) end sleep(1) -# Close using Status codes according to RFC 6455 7.4.1 + +info("Open ws|clients, close using every status code from RFC 6455 7.4.1\n" * + " Verify error messages on server.out reflect the codes.") +sleep(1) for (ke, va) in WebSockets.codeDesc info("Closing ws|client with reason ", ke, " ", va) sleep(0.3) @@ -179,10 +194,13 @@ for (ke, va) in WebSockets.codeDesc sleep(1) end -# Close with a given reason +info("Open a ws|client, close it using a status code from RFC 6455 7.4.1\n" * + " and also a custom reason string. Verify error messages on server.out reflect the codes.") + +sleep(1) va = 1000 info("Closing ws|client with reason", va, " ", WebSockets.codeDesc[va], " and goodbye!") -WebSockets.open((ws)-> close(ws, statusnumber = 1000, freereason = "goodbye!"), URL) +WebSockets.open((ws)-> close(ws, statusnumber = va, freereason = "goodbye!"), URL) wait(server_WS.out) err = take!(server_WS.out) @test typeof(err) <: WebSocketClosedError @@ -190,11 +208,11 @@ err = take!(server_WS.out) stack_trace = take!(server_WS.out) sleep(1) -println() -info(" React to an InterruptException with a closing handshake. - A lot of error text will spill over into REPL, but the test is unaffected") -println() +info("\nOpen a ws|client. Throw an InterruptException to it. Check that the ws|server\n " * + "error shows the reason for the close.\n " * + "A lot of error text will spill over into REPL, but the test is unaffected\n\n") +sleep(1) function selfinterruptinghandler(ws) task = @schedule WebSockets.open((ws)-> read(ws), URL) sleep(3) @@ -204,11 +222,9 @@ function selfinterruptinghandler(ws) end WebSockets.open(selfinterruptinghandler, URL) sleep(6) -wait(server_WS.out) err = take!(server_WS.out) @test typeof(err) <: WebSocketClosedError @test err.message == "ws|server respond to OPCODE_CLOSE 1006: while read(ws|client received InterruptException." -wait(server_WS.out) stack_trace = take!(server_WS.out) put!(server_WS.in, HTTP.Servers.KILL) -sleep(1) +sleep(2) diff --git a/test/runtests.jl b/test/runtests.jl index 2e77ad4..54f8e23 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -29,9 +29,6 @@ info("\nAbrupt close & error handling test\n") include("error_test.jl");sleep(1) end -@testset "Bad frames, multiframes, loaded ping" begin - include("error_test.jl");sleep(1) -end end # TODO # restructure browsertests From 7f244b00f9121395553ec0bf41e09f1e40c80f16 Mon Sep 17 00:00:00 2001 From: hustf Date: Mon, 4 Jun 2018 23:52:44 +0200 Subject: [PATCH 34/35] mod: README.md Large rewrite mod: src/WebSockets.jl Add 'end' in example --- README.md | 208 +++++++++++++++++++++++++--------------------- src/WebSockets.jl | 3 +- 2 files changed, 113 insertions(+), 98 deletions(-) diff --git a/README.md b/README.md index 7210e85..7f719ae 100644 --- a/README.md +++ b/README.md @@ -3,21 +3,46 @@ WebSockets.jl [![Build Status](https://travis-ci.org/JuliaWeb/WebSockets.jl.png)](https://travis-ci.org/JuliaWeb/WebSockets.jl) [![Coverage Status](https://img.shields.io/coveralls/JuliaWeb/WebSockets.jl.svg)](https://coveralls.io/r/JuliaWeb/WebSockets.jl) - [![WebSockets](http://pkg.julialang.org/badges/WebSockets_0.6.svg)](http://pkg.julialang.org/?pkg=WebSockets&ver=0.6) -Temporary badges: -[![Build status](https://ci.appveyor.com/api/projects/status/sx6i51rjc9ajjdh8?svg=true)](https://ci.appveyor.com/project/hustf/websockets-jl-nfuiv) -[![Build status](https://ci.appveyor.com/api/projects/status/sx6i51rjc9ajjdh8/branch/master?svg=true)](https://ci.appveyor.com/project/hustf/websockets-jl-nfuiv/branch/master) +# (still temporary version) +[Websockets](https://tools.ietf.org/html/rfc6455) is a message protocol on top of TCP with less overhead and restrictions than HTTP(S). Originally, scripts hosted on a web page initiated the connection via HTTP/S. This implementation can now act as either client (`open`) or server (`listen` or `serve`). + + +Client websockets are created by calling `open`. Just import [HTTP.jl](https://github.com/JuliaWeb/HttpServer.jl) before +WebSockets. + +Server websockets are always asyncronous [tasks](https://docs.julialang.org/en/stable/stdlib/parallel/#Tasks-1), spawned by +[HttpServer.jl](https://github.com/JuliaWeb/HttpServer.jl) or HTTP.jl. + + +## What does `WebSockets.jl` enable? +- read +- write +- build a network +- heartbeating +- define your own protocols, or implement existing ones +- low latency and overhead + +Some other [protocols](https://github.com/JuliaInterop) struggle including browsers and Javascript in the network, although for example ZMQ / IJulia / Jupyter says otherwise. WebSockets are well suited for user interactions via a browser. When respecting Javascript as a compiled language with powerful parallel capabilities, user interaction and graphics workload, even development can be moved off Julia resources. -[![Build status](https://ci.appveyor.com/api/projects/status/sx6i51rjc9ajjdh8/branch/master?svg=true)](https://ci.appveyor.com/project/hustf/websockets-jl-nfuiv/branch/change_dependencies) +You may also prefer Julia packages [DandelionWebSockets](https://github.com/dandeliondeathray/DandelionWebSockets.jl) or the implementation directly in HTTP.jl itself. -This is an implementation of the WebSockets protocol in Julia for both server-side and client-side applications. +## What are the main downsides to WebSockets (in Julia)? -This package works with either [HttpServer.jl](https://github.com/JuliaWeb/HttpServer.jl) or [HTTP.jl](https://github.com/JuliaWeb/HttpServer.jl), which is what you use to set up a server that accepts HTTP(S) connections. +- Logging. We need customizable and very fast logging for building networked applications. +- Security. Http(s) servers are currently not working. Take care. +- Non-compliant proxies on the internet, company firewalls. Commercial applications often use competing technologies for this reason. HTTP.jl lets you access the network without the restriction of structured messages. +- 'Warm-up', i.e. compilation when a method is first used. These are excluded from current benchmarks. +- Garbage collection, which increases message latency at semi-random intervals. See benchmark plots. +- TCP. If a connection is broken, the underlying protocol will throw ECONNRESET messages. +- TCP quirks, including 'warm-up' time with low transmission speed after a pause. Heartbeats can alleviate. +- Neither HTTP.jl or HttpServer.jl are made just for connecting WebSockets. You may need strong points from both. +- The optional dependencies increases load time compared to fixed dependencies. -## Using with HttpServer.jl + +## Server side As a first example, we can create a WebSockets echo server: @@ -25,47 +50,92 @@ As a first example, we can create a WebSockets echo server: using HttpServer using WebSockets -wsh = WebSocketHandler() do req,client - while true - msg = read(client) - println(msg) # Write the received message to the REPL - write(client, msg) +function coroutine(ws) + while isopen(ws) + data, = readguarded(ws) + s = String(data) + if s == "" + break + end + println(s) + if s[1] == "P" + writeguarded(ws, "No, I'm not!") + else + writeguarded(ws, "Why?") + end end - end +end -server = Server(wsh) -run(server,8080) +function gatekeeper(req, ws) + if origin(req) == "http://127.0.0.1:8080" || origin(req) == "http://localhost:8080" + coroutine(ws) + else + println(origin(req)) + end +end + +handle(req, res) = Response(200) + +server = Server(HttpHandler(handle), + WebSocketHandler(gatekeeper)) + +run(server, 8080) ``` -This sets up a server running on localhost, port 8080. -It will accept WebSockets connections. -The function in `wsh` will be called once per connection; it takes over that connection. -In this case, it reads each `msg` from the `client` and then writes the same message back: a basic echo server. +Now open a browser on http://127.0.0.1:8080/ and press F12. In the console, type the lines following ≫: +```javascript +≫ws = new WebSocket("ws://127.0.0.1:8080") + ← WebSocket { url: "ws://127.0.0.1:8080/", readyState: 0, bufferedAmount: 0, onopen: null, onerror: null, onclose: null, extensions: "", protocol: "", onmessage: null, binaryType: "blob" } +≫ws.send("Peer, you're lying!") + ← undefined +≫ws.onmessage = function(e){console.log(e.data)} + ← function onmessage() +≫ws.send("Well, then.") + ← undefined +Why? debugger eval code:1:28 +``` + +If you now navigate or close the browser, this happens: +1. the client side of the websocket connection will quickly send a close request and go away. +2. Server side `readguarded(ws)` has been waiting for messages, but instead closes 'ws' and returns ("", false) +3. `coroutine(ws)` is finished and the task's control flow returns to HttpServer +4. HttpServer does nothing other than exit this task. In fact, it often crashes because + somebody else (the browser) has closed the underlying TCP stream. If you had replaced the last Julia line with '@async run(server, 8080', you would see some long error messages. +5. The server, which spawned the task, continues to listen for incoming connections, and you're stuck. Ctrl + C! + +You could replace 'using HttpServer' with 'using HTTP'. Also: + Serve -> ServeWS + HttpHandler -> HTTP.Handler + WebSocketHandler -> WebSockets.WebsocketHandler -The function that you pass to the `WebSocketHandler` constructor takes two arguments: -a `Request` from [HttpCommon.jl](https://github.com/JuliaWeb/HttpCommon.jl/blob/master/src/HttpCommon.jl#L142), -and a `WebSocket` from [here](https://github.com/JuliaWeb/WebSockets.jl/blob/master/src/WebSockets.jl#L17). -## Using with HTTP.jl +## Client side -The following example demonstrates how to use WebSockets.jl as bother a server and client. +You need to use [HTTP.jl](https://github.com/JuliaWeb/HttpServer.jl). + +What you can't do is use a browser as the server side. The server side can be the example above, running in an asyncronous task. The server can also be running in a separate REPL, or in a a parallel task. The benchmarks puts the `client` side on a parallel task. + +The following example +- runs server in an asyncronous task, client in the REPL control flow +- uses [Do-Block-Syntax](https://docs.julialang.org/en/v0.6.3/manual/functions/#Do-Block-Syntax-for-Function-Arguments-1), which is a style choice +- the server `ugrade` skips checking origin(req)` +- the server is invoked with `listen(..)` instead of `serve()` +- read(ws) and write(ws, msg) instead of readguarded(ws), writeguarded(ws) ```julia using HTTP using WebSockets -using Base.Test -port = 8000 +const PORT = 8080 -# Start the echo server -@async HTTP.listen("127.0.0.1",UInt16(port)) do http +# Server side +@async HTTP.listen("127.0.0.1", PORT) do http if WebSockets.is_upgrade(http.message) - WebSockets.upgrade(http) do ws - while ws.state == WebSockets.CONNECTED + WebSockets.upgrade(http) do req, ws + while isopen(ws) msg = String(read(ws)) - println(msg) # Write the received message to the REPL - write(ws,msg) + write(ws, msg) end end end @@ -73,86 +143,30 @@ end sleep(2) -# Connect a client to the server above -WebSockets.open("ws://127.0.0.1:$(port)") do ws - write(ws, "Foo") - @test String(read(ws)) == "Foo" - - write(ws, "Bar") - @test String(read(ws)) == "Bar" - close(ws) +WebSockets.open("ws://127.0.0.1:$PORT") do ws + write(ws, "Peer, about your hunting") + println("echo received:" * String(read(ws))) end ``` -## What can you do with a `WebSocket`? -You can: +The output in a console session is barely readable, which is irritating. To build real-time applications, we need more code. -* `write` data to it -* `read` data from it -* send `ping` or `pong` messages -* `close` the connection +Some logging utilties for a running relay server are available in /logutils. -## Installation/Setup -WebSockets.jl must be used with either HttpServer.jl or HTTP.jl, but neither is a dependency of this package, so you will need to first add one of the two, i.e. -~~~julia -julia> Pkg.add("HttpServer") -~~~ +## Installation/Setup -or +WebSockets.jl must be used with either HttpServer.jl or HTTP.jl, but neither is a dependency of this package. You will need to first add one or both, i.e.: ~~~julia +julia> Pkg.add("HttpServer") julia> Pkg.add("HTTP") -~~~ - -Once you have one of the two, you can add WebSockets.jl via - -~~~julia julia> Pkg.add("WebSockets") ~~~ -At this point, you can use the examples below to test that it all works. - -## [Chat client/server example](https://github.com/JuliaWeb/WebSockets.jl/blob/master/examples/chat.jl): - -1. From the REPL, run - -```julia -include(joinpath(Pkg.dir("WebSockets"),"examples","chat.jl")); -``` - -2. In a web browser, open `localhost:8000` -3. You should see a basic IRC-like chat application - - -## Echo server example: - -~~~julia -using HttpServer -using WebSockets - -wsh = WebSocketHandler() do req,client - while true - msg = read(client) - write(client, msg) - end - end - -server = Server(wsh) -run(server,8080) -~~~ - -To play with a WebSockets echo server, you can: -1. Paste the above code in to the Julia REPL -2. Open `localhost:8080` in Chrome -3. Open the Chrome developers tools console -4. Type `ws = new WebSocket("ws://localhost:8080");` into the console -5. Type `ws.send("hi")` into the console and you should see "hi" printed to the REPL -6. Switch to the 'Network' tab; click on the request; click on the 'frames' tab. -7. You will see the two frames containing "hi": one sent and one received. ~~~~ :::::::::::::::: diff --git a/src/WebSockets.jl b/src/WebSockets.jl index 093edaf..3893824 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -611,7 +611,8 @@ E.g. function gatekeeper(req, ws) orig = WebSockets.origin(req) if startswith(orig, "http://localhost") || startswith(orig, "http://127.0.0.1") - handlewebsocket(ws) + handlewebsocket(ws) + end end end ``` From ada7f2dcfddc4405a894ee04c73770f4be81ee71 Mon Sep 17 00:00:00 2001 From: hustf Date: Wed, 6 Jun 2018 23:31:17 +0200 Subject: [PATCH 35/35] modified: README.md Another spin modified: src/HTTP.jl Improve inline docs modified: src/WebSockets.jl RSV1 compression bit remarks --- README.md | 72 +++++++++++++++++++++++++++-------------------- src/HTTP.jl | 26 ++++++++--------- src/WebSockets.jl | 7 +++-- 3 files changed, 58 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 7f719ae..b3b792b 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,70 @@ -WebSockets.jl -============= +# WebSockets.jl + [![Build Status](https://travis-ci.org/JuliaWeb/WebSockets.jl.png)](https://travis-ci.org/JuliaWeb/WebSockets.jl) [![Coverage Status](https://img.shields.io/coveralls/JuliaWeb/WebSockets.jl.svg)](https://coveralls.io/r/JuliaWeb/WebSockets.jl) [![WebSockets](http://pkg.julialang.org/badges/WebSockets_0.6.svg)](http://pkg.julialang.org/?pkg=WebSockets&ver=0.6) -# (still temporary version) -[Websockets](https://tools.ietf.org/html/rfc6455) is a message protocol on top of TCP with less overhead and restrictions than HTTP(S). Originally, scripts hosted on a web page initiated the connection via HTTP/S. This implementation can now act as either client (`open`) or server (`listen` or `serve`). +Server and client side [Websockets](https://tools.ietf.org/html/rfc6455) protocol in Julia. WebSockets is a small overhead message protocol layered over [TCP](https://tools.ietf.org/html/rfc793). It uses HTTP(S) for establishing the connections. + +## Getting started +WebSockets.jl must be used with either HttpServer.jl or HTTP.jl, but neither is a dependency of this package. You will need to first add one or both, i.e.: + +```julia +julia> Pkg.add("HttpServer") +julia> Pkg.add("HTTP") +julia> Pkg.add("WebSockets") +``` +### Open a client side connection +Client side websockets are created by calling `WebSockets.open` (with a server running). Client side websockets require [HTTP.jl](https://github.com/JuliaWeb/HttpServer.jl). +### Accept server side connections -Client websockets are created by calling `open`. Just import [HTTP.jl](https://github.com/JuliaWeb/HttpServer.jl) before -WebSockets. +Server side websockets are asyncronous [tasks](https://docs.julialang.org/en/stable/stdlib/parallel/#Tasks-1), spawned by either +[HttpServer.jl](https://github.com/JuliaWeb/HttpServer.jl) or HTTP.jl. -Server websockets are always asyncronous [tasks](https://docs.julialang.org/en/stable/stdlib/parallel/#Tasks-1), spawned by -[HttpServer.jl](https://github.com/JuliaWeb/HttpServer.jl) or HTTP.jl. +##### Using HttpServer +Call `run`, which is a wrapper for calling `listen`. See inline docs. +##### Using HTTP +Call `WebSockets.serve`, which is a wrapper for `HTTP.listen`. See inline docs. ## What does `WebSockets.jl` enable? -- read -- write -- build a network -- heartbeating -- define your own protocols, or implement existing ones -- low latency and overhead -Some other [protocols](https://github.com/JuliaInterop) struggle including browsers and Javascript in the network, although for example ZMQ / IJulia / Jupyter says otherwise. WebSockets are well suited for user interactions via a browser. When respecting Javascript as a compiled language with powerful parallel capabilities, user interaction and graphics workload, even development can be moved off Julia resources. +- reading and writing between entities you can program or know about +- low latency messaging +- implement your own 'if X send this, Y do that' subprotocols +- implement registered [websocket subprotocols](https://www.iana.org/assignments/websocket/websocket.xml#version-number) +- heartbeating, relaying +- build a network including browser clients +- convenience functions for gatekeeping with a common interface for HttpServer and HTTP +- writing http handlers and websockets 'handlers' in the same process can be an advantage. Exchanging unique tokens via http(s) + before accepting websockets is recommended for improved security. -You may also prefer Julia packages [DandelionWebSockets](https://github.com/dandeliondeathray/DandelionWebSockets.jl) or the implementation directly in HTTP.jl itself. +WebSockets are well suited for user interactions via a browser. By calling compiled Javascript functions in browsers and using parallel workers, +user interaction and graphics workload, even development time can be moved off Julia resources. + +The /logutils folder contains some logging functionality that is quite fast and can make working with multiple asyncronous tasks easier. This functionality may be moved out of WebSockets in the future, depending on how other logging capabilities develop. + +You should also have a look at Julia packages [DandelionWebSockets](https://github.com/dandeliondeathray/DandelionWebSockets.jl) or the implementation currently part of HTTP.jl. ## What are the main downsides to WebSockets (in Julia)? - Logging. We need customizable and very fast logging for building networked applications. -- Security. Http(s) servers are currently not working. Take care. -- Non-compliant proxies on the internet, company firewalls. Commercial applications often use competing technologies for this reason. HTTP.jl lets you access the network without the restriction of structured messages. +- Security. Julia's Http(s) servers are currently not working to our knowledge. +- Non-compliant proxies on the internet, company firewalls. Commercial applications often use competing technologies for this reason, according to some old articles at least. HTTP.jl lets you use such techniques. - 'Warm-up', i.e. compilation when a method is first used. These are excluded from current benchmarks. - Garbage collection, which increases message latency at semi-random intervals. See benchmark plots. -- TCP. If a connection is broken, the underlying protocol will throw ECONNRESET messages. +- If a connection is closed improperly, the connection task will throw uncaught ECONNRESET and similar messages. - TCP quirks, including 'warm-up' time with low transmission speed after a pause. Heartbeats can alleviate. - Neither HTTP.jl or HttpServer.jl are made just for connecting WebSockets. You may need strong points from both. - The optional dependencies increases load time compared to fixed dependencies. +- Since 'read' is a blocking function, you can easily end up reading indefinetely from both sides. +## Server side example -## Server side - -As a first example, we can create a WebSockets echo server: +As a first example, we can create a WebSockets echo server. We use named function arguments for more readable stacktraces while debugging. ```julia using HttpServer @@ -156,15 +176,7 @@ Some logging utilties for a running relay server are available in /logutils. -## Installation/Setup -WebSockets.jl must be used with either HttpServer.jl or HTTP.jl, but neither is a dependency of this package. You will need to first add one or both, i.e.: - -~~~julia -julia> Pkg.add("HttpServer") -julia> Pkg.add("HTTP") -julia> Pkg.add("WebSockets") -~~~ diff --git a/src/HTTP.jl b/src/HTTP.jl index 04441af..4e95a1d 100644 --- a/src/HTTP.jl +++ b/src/HTTP.jl @@ -225,13 +225,10 @@ end """ -WebSockets.ServerWS is a variant of HTTP.Server which -also includes a WebSockets.WebSocketHandler. -Most or all of the functionality can alternaively be accessed with -'listen' - ``` julia - se = WebSockets.ServerWS(::HTTP.HandlerFunction, WebSockets.WebsocketHandler(gatekeeper)) - ``` + WebSockets.ServerWS(::HTTP.HandlerFunction, ::WebSockets.WebsocketHandler(gatekeeper)) + +WebSockets.ServerWS is an argument type for WebSockets.serve. Instances +include .in and .out channels, see WebSockets.serve. """ mutable struct ServerWS{T <: HTTP.Servers.Scheme, H <: HTTP.Handler, W <: WebsocketHandler} handler::H @@ -264,22 +261,23 @@ function ServerWS(handler::H, end """ -A variant of HTTP.serve with the WebSockets.ServerWS type. -Puts any caught error and stacktrace on the serve.out channel. + WebSockets.serve(server::ServerWS{T, H, W}, host, port, verbose) + +A wrapper for HTTP.listen. +Puts any caught error and stacktrace on the server.out channel. +To stop a running server, put HTTP.Servers.KILL on the .in channel. ```julia - @shedule WebSockets.serve(myServerWS, "127.0.0.1", 8080, false) + @shedule WebSockets.serve(server, "127.0.0.1", 8080, false) ``` -After a suspected 'connection task' failure: +After a suspected connection task failure: ```julia - if isready(myserver_WS.out) + if isready(server.out) err = take!(myserver_WS.out) - @test typeof(err) <: WebSocketClosedError end if isready(myserver_WS.out) stack_trace = take!(server_WS.out) end ``` - """ function serve(server::ServerWS{T, H, W}, host, port, verbose) where {T, H, W} diff --git a/src/WebSockets.jl b/src/WebSockets.jl index 3893824..2c8122f 100644 --- a/src/WebSockets.jl +++ b/src/WebSockets.jl @@ -15,11 +15,12 @@ includes another Julia session, parallel process or task. Future improvements: 1. Logging of refused requests and closures due to bad behavior of client. -2. Allow users to receive control messages if they want to. +2. Allow users to receive control messages or metadata if they want to. + For example RSV1 (compressed message) would be interesting. 3. Check rsv1 to rsv3 values. This will reduce bandwidth. 4. Optimize maskswitch!, possibly threaded above a certain limit. 5. Split messages over several frames. -6. Allow customizable output (e.g. 'ping'). See HttpServer listen. +6. Allow customizable console output (e.g. 'ping'). See HttpServer listen. """ module WebSockets import MbedTLS: digest, MD_SHA1 @@ -286,7 +287,7 @@ Base.eof(ws::WebSocket) = (ws.state == CLOSED) || eof(ws.socket) """ Represents one (received) message frame.""" mutable struct WebSocketFragment is_last::Bool - rsv1::Bool + rsv1::Bool # Set for compressed messages. rsv2::Bool rsv3::Bool opcode::UInt8 # This is actually a UInt4 value.