WIP: Parsing Buildkite logs to get build stats

buildkite_logs.jl

@@ -0,0 +1,84 @@
+using HTTP, JSON3
+
+function get_binaryurl(page)
+    url = "https://buildkite.com/julialang/julia-master/builds?branch=master&page=$page"
+
+    r = HTTP.get(url)
+    html = String(r.body)
+
+    build_urls = map(x -> x.match, eachmatch(r"julialang/julia-\w*/builds/(\d+)", html)) |> unique
+
+    sha_to_logs = Dict{String,String}()
+
+    for build_url in build_urls
+        details_url = "https://buildkite.com/" * build_url * ".json"
+        details_json = HTTP.get(details_url).body |> JSON3.read
+        idx = findfirst(x -> x.name == ":linux: build x86_64-linux-gnu", details_json.jobs)
+
+        logs_url = "https://buildkite.com/" * details_json.jobs[idx].base_path * "/raw_log"
+
+        log = HTTP.get(logs_url).body |> String
+        sha = details_json.commit_id
+
+        sha_to_logs[sha] = log
+    end
+
+    return sha_to_logs
+end
+
+function parse_logs!(sha_to_timings, sha_to_binary_sizes, sha_to_logs)
+    @views for (sha, log) in sha_to_logs
+        binary_size_start_idx = (findfirst("==> ./julia binary sizes", log) |> last) + 1
+        binary_size_end_idx = (findfirst("==> ./julia launch speedtest", log) |> first) - 1
+
+        binaries = eachmatch(r"/([a-zA-Z.]+)[[:blank:]]+:", log[binary_size_start_idx:binary_size_end_idx]) |> collect
+        for (i, binary) in pairs(binaries)
+            binary_name = binary.captures[1]
+            sha_to_binary_sizes[sha][binary_name] = Dict{String,UInt64}()
+
+            next_binary = i == lastindex(binaries) ? binary_size_end_idx : binaries[i+1].match.offset + binary_size_start_idx
+
+            for m in eachmatch(r"([a-zA-Z.]+)[[:blank:]]*(\d+)", log[binary_size_start_idx+binary.match.offset:next_binary])
+                sha_to_binary_sizes[sha][binary_name][m.captures[1]] = parse(UInt64, m.captures[2])
+            end
+        end
+
+        speedtest_start_idx = (findfirst("==> ./julia launch speedtest", log) |> last) + 1
+        speedtest_end_idx = (findfirst("Create build artifacts", log) |> first) - 1
+
+        for m in eachmatch(r"[^\[]([\d.]+)[[:blank:]]*([a-zA-Z]+)", log[speedtest_start_idx:speedtest_end_idx])
+            if !haskey(sha_to_timings[sha], m.captures[2])
+                sha_to_timings[sha][m.captures[2]] = Float64[]
+            end
+            push!(sha_to_timings[sha][m.captures[2]], parse(Float64, m.captures[1]))
+        end
+    end
+end
+
+sha_to_logs = get_binaryurl(1)
+sha_to_timings = Dict{String,Dict{String,Vector{Float64}}}(sha => Dict{String,Vector{Float64}}() for sha in keys(sha_to_logs))
+sha_to_binary_sizes = Dict{String,Dict{String,Dict{String,UInt64}}}(sha => Dict{String,Dict{String,UInt64}}() for sha in keys(sha_to_logs))
+
+parse_logs!(sha_to_timings, sha_to_binary_sizes, sha_to_logs)
+
+# for (sha, timing_series) in sha_to_timings, (name, timings) in timing_series
+#     println(name, "\n ", sum(timings) / length(timings))
+# end
+
+for (sha, binary_sizes) in sha_to_binary_sizes
+    println(sha, ": ", binary_sizes["sys.so"]["Total"] |> Int)
+end
+
+@testset "parsing logs" begin
+    sha_to_logs_test = JSON3.read("sha_to_logs_test.json", Dict{String,String})
+    sha_to_timings = Dict{String,Dict{String,Vector{Float64}}}(sha => Dict{String,Vector{Float64}}() for sha in keys(sha_to_logs))
+    sha_to_binary_sizes = Dict{String,Dict{String,Dict{String,UInt64}}}(sha => Dict{String,Dict{String,UInt64}}() for sha in keys(sha_to_logs))
+
+    parse_logs!(sha_to_timings, sha_to_binary_sizes, sha_to_logs_test)
+
+    sha_to_timings_test = JSON3.read("sha_to_timings_test.json", Dict{String,Dict{String,Vector{Float64}}})
+    sha_to_binary_sizes_test = JSON3.read("sha_to_binary_sizes_test.json", Dict{String,Dict{String,Dict{String,UInt64}}})
+
+    @test sha_to_timings == sha_to_timings_test
+    @test sha_to_binary_sizes == sha_to_binary_sizes_test
+end

sha_to_binary_sizes_test.json

@@ -0,0 +1 @@
+{"7c5c724ed93015d2e9c52c7fcd8f5c4ebbefff03":{"julia":{".got.plt":48,".gnu.hash":104,".dynsym":480,".plt.got":8,".ctors":16,"Total":9478,".tbss":8,".rodata":4,".plt":64,".init":36,".got":8,".dynstr":331,".bss":24,".gnu.version":40,".hash":156,".rela.plt":72,".rela.dyn":24,".comment":34,".dtors":16,".dynamic":512,".data":16,".fini":14,".text":545,".interp":28},"sys.so":{".got.plt":696,".gnu.hash":60,".dynsym":2568,".plt.got":16,".ctors":16,".lrodata":261680,"Total":197825262,".rodata":91602,".plt":1360,".init":36,".got":128,".dynstr":1839,".bss":16,".gnu.version":214,".hash":824,".rela.plt":2016,".rela.dyn":1113000,".comment":17,".dtors":16,".dynamic":480,".data":8,".fini":14,".lbss":107104,".text":29836129,".ldata":116973936},"libjulia.so":{".got.plt":264,".data.rel.ro":9688,".gnu.hash":5568,".dynsym":18600,".plt.got":16,".ctors":16,"Total":199055,".rodata":15066,".plt":496,".init":36,".got":64,".dynstr":14236,".bss":7144,".gnu.version":1550,".hash":5192,".rela.plt":720,".rela.dyn":29160,".comment":17,".dtors":16,".dynamic":560,".data":1056,".fini":14,".text":10577}},"a14cc38512b6daab6b8417ebb8a64fc794ff89cc":{"julia":{".got.plt":48,".gnu.hash":104,".dynsym":480,".plt.got":8,".ctors":16,"Total":9478,".tbss":8,".rodata":4,".plt":64,".init":36,".got":8,".dynstr":331,".bss":24,".gnu.version":40,".hash":156,".rela.plt":72,".rela.dyn":24,".comment":34,".dtors":16,".dynamic":512,".data":16,".fini":14,".text":545,".interp":28},"sys.so":{".got.plt":696,".gnu.hash":60,".dynsym":2568,".plt.got":16,".ctors":16,".lrodata":269968,"Total":197848157,".rodata":91875,".plt":1360,".init":36,".got":128,".dynstr":1839,".bss":16,".gnu.version":214,".hash":824,".rela.plt":2016,".rela.dyn":1132248,".comment":17,".dtors":16,".dynamic":480,".data":8,".fini":14,".lbss":106944,".text":29839873,".ldata":116969560},"libjulia.so":{".got.plt":264,".data.rel.ro":9688,".gnu.hash":5568,".dynsym":18600,".plt.got":16,".ctors":16,"Total":199055,".rodata":15066,".plt":496,".init":36,".got":64,".dynstr":14236,".bss":7144,".gnu.version":1550,".hash":5192,".rela.plt":720,".rela.dyn":29160,".comment":17,".dtors":16,".dynamic":560,".data":1056,".fini":14,".text":10577}},"ded0b28f7f94c2785a6330786fe5efea2a440c86":{"julia":{".got.plt":48,".gnu.hash":104,".dynsym":480,".plt.got":8,".ctors":16,"Total":9478,".tbss":8,".rodata":4,".plt":64,".init":36,".got":8,".dynstr":331,".bss":24,".gnu.version":40,".hash":156,".rela.plt":72,".rela.dyn":24,".comment":34,".dtors":16,".dynamic":512,".data":16,".fini":14,".text":545,".interp":28},"sys.so":{".got.plt":696,".gnu.hash":60,".dynsym":2568,".plt.got":16,".ctors":16,".lrodata":270272,"Total":197956044,".rodata":91584,".plt":1360,".init":36,".got":128,".dynstr":1839,".bss":16,".gnu.version":214,".hash":824,".rela.plt":2016,".rela.dyn":1134408,".comment":17,".dtors":16,".dynamic":480,".data":8,".fini":14,".lbss":107104,".text":29869233,".ldata":116970480},"libjulia.so":{".got.plt":264,".data.rel.ro":9688,".gnu.hash":5568,".dynsym":18600,".plt.got":16,".ctors":16,"Total":199055,".rodata":15066,".plt":496,".init":36,".got":64,".dynstr":14236,".bss":7144,".gnu.version":1550,".hash":5192,".rela.plt":720,".rela.dyn":29160,".comment":17,".dtors":16,".dynamic":560,".data":1056,".fini":14,".text":10577}},"c5994e4b2b4277ed56887a51f523149ed0c87bf5":{"julia":{".got.plt":48,".gnu.hash":104,".dynsym":480,".plt.got":8,".ctors":16,"Total":9478,".tbss":8,".rodata":4,".plt":64,".init":36,".got":8,".dynstr":331,".bss":24,".gnu.version":40,".hash":156,".rela.plt":72,".rela.dyn":24,".comment":34,".dtors":16,".dynamic":512,".data":16,".fini":14,".text":545,".interp":28},"sys.so":{".got.plt":696,".gnu.hash":60,".dynsym":2568,".plt.got":16,".ctors":16,".lrodata":261600,"Total":197849309,".rodata":91710,".plt":1360,".init":36,".got":128,".dynstr":1839,".bss":16,".gnu.version":214,".hash":824,".rela.plt":2016,".rela.dyn":1112760,".comment":17,".dtors":16,".dynamic":480,".data":8,".fini":14,".lbss":107088,".text":29814209,".ldata":117029288},"libjulia.so":{".got.plt":264,".data.rel.ro":9688,".gnu.hash":5568,".dynsym":18600,".plt.got":16,".ctors":16,"Total":199055,".rodata":15066,".plt":496,".init":36,".got":64,".dynstr":14236,".bss":7144,".gnu.version":1550,".hash":5192,".rela.plt":720,".rela.dyn":29160,".comment":17,".dtors":16,".dynamic":560,".data":1056,".fini":14,".text":10577}},"dd1ed17ae3c2e4e257f2444bbafd705326b4bbbd":{"julia":{".got.plt":48,".gnu.hash":104,".dynsym":480,".plt.got":8,".ctors":16,"Total":9474,".tbss":8,".rodata":4,".plt":64,".init":36,".got":8,".dynstr":331,".bss":24,".gnu.version":40,".hash":156,".rela.plt":72,".rela.dyn":24,".comment":34,".dtors":16,".dynamic":512,".data":16,".fini":14,".text":545,".interp":28},"sys.so":{".got.plt":696,".gnu.hash":60,".dynsym":2568,".plt.got":16,".ctors":16,".lrodata":261776,"Total":197891298,".rodata":91250,".plt":1360,".init":36,".got":128,".dynstr":1839,".bss":16,".gnu.version":214,".hash":824,".rela.plt":2016,".rela.dyn":1113528,".comment":17,".dtors":16,".dynamic":480,".data":8,".fini":14,".lbss":107120,".text":29823569,".ldata":117042336},"libjulia.so":{".got.plt":264,".data.rel.ro":9688,".gnu.hash":5568,".dynsym":18600,".plt.got":16,".ctors":16,"Total":199049,".rodata":15066,".plt":496,".init":36,".got":64,".dynstr":14236,".bss":7144,".gnu.version":1550,".hash":5192,".rela.plt":720,".rela.dyn":29160,".comment":17,".dtors":16,".dynamic":560,".data":1056,".fini":14,".text":10577}},"0d30be8d74a921ace3b405c5d47367d50872176b":{"julia":{".got.plt":48,".gnu.hash":104,".dynsym":480,".plt.got":8,".ctors":16,"Total":9478,".tbss":8,".rodata":4,".plt":64,".init":36,".got":8,".dynstr":331,".bss":24,".gnu.version":40,".hash":156,".rela.plt":72,".rela.dyn":24,".comment":34,".dtors":16,".dynamic":512,".data":16,".fini":14,".text":545,".interp":28},"sys.so":{".got.plt":696,".gnu.hash":60,".dynsym":2568,".plt.got":16,".ctors":16,".lrodata":270624,"Total":198011461,".rodata":91892,".plt":1360,".init":36,".got":128,".dynstr":1839,".bss":16,".gnu.version":214,".hash":824,".rela.plt":2016,".rela.dyn":1134456,".comment":17,".dtors":16,".dynamic":480,".data":8,".fini":14,".lbss":107120,".text":29880513,".ldata":117008888},"libjulia.so":{".got.plt":264,".data.rel.ro":9688,".gnu.hash":5568,".dynsym":18600,".plt.got":16,".ctors":16,"Total":199055,".rodata":15066,".plt":496,".init":36,".got":64,".dynstr":14236,".bss":7144,".gnu.version":1550,".hash":5192,".rela.plt":720,".rela.dyn":29160,".comment":17,".dtors":16,".dynamic":560,".data":1056,".fini":14,".text":10577}},"6f39acb9fb2b34b43d6b455d196b40f1759c9031":{"julia":{".got.plt":48,".gnu.hash":104,".dynsym":480,".plt.got":8,".ctors":16,"Total":9478,".tbss":8,".rodata":4,".plt":64,".init":36,".got":8,".dynstr":331,".bss":24,".gnu.version":40,".hash":156,".rela.plt":72,".rela.dyn":24,".comment":34,".dtors":16,".dynamic":512,".data":16,".fini":14,".text":545,".interp":28},"sys.so":{".got.plt":696,".gnu.hash":60,".dynsym":2568,".plt.got":16,".ctors":16,".lrodata":270608,"Total":198020677,".rodata":91712,".plt":1360,".init":36,".got":128,".dynstr":1839,".bss":16,".gnu.version":214,".hash":824,".rela.plt":2016,".rela.dyn":1134744,".comment":17,".dtors":16,".dynamic":480,".data":8,".fini":14,".lbss":107088,".text":29893361,".ldata":116985600},"libjulia.so":{".got.plt":264,".data.rel.ro":9688,".gnu.hash":5568,".dynsym":18600,".plt.got":16,".ctors":16,"Total":199055,".rodata":15066,".plt":496,".init":36,".got":64,".dynstr":14236,".bss":7144,".gnu.version":1550,".hash":5192,".rela.plt":720,".rela.dyn":29160,".comment":17,".dtors":16,".dynamic":560,".data":1056,".fini":14,".text":10577}},"320366bbc00331671b38f7aba15812ffabcf1fcb":{"julia":{".got.plt":48,".gnu.hash":104,".dynsym":480,".plt.got":8,".ctors":16,"Total":9478,".tbss":8,".rodata":4,".plt":64,".init":36,".got":8,".dynstr":331,".bss":24,".gnu.version":40,".hash":156,".rela.plt":72,".rela.dyn":24,".comment":34,".dtors":16,".dynamic":512,".data":16,".fini":14,".text":545,".interp":28},"sys.so":{".got.plt":696,".gnu.hash":60,".dynsym":2568,".plt.got":16,".ctors":16,".lrodata":270416,"Total":197962840,".rodata":91608,".plt":1360,".init":36,".got":128,".dynstr":1839,".bss":16,".gnu.version":214,".hash":824,".rela.plt":2016,".rela.dyn":1134984,".comment":17,".dtors":16,".dynamic":480,".data":8,".fini":14,".lbss":107120,".text":29884545,".ldata":116943888},"libjulia.so":{".got.plt":264,".data.rel.ro":9688,".gnu.hash":5568,".dynsym":18600,".plt.got":16,".ctors":16,"Total":199055,".rodata":15066,".plt":496,".init":36,".got":64,".dynstr":14236,".bss":7144,".gnu.version":1550,".hash":5192,".rela.plt":720,".rela.dyn":29160,".comment":17,".dtors":16,".dynamic":560,".data":1056,".fini":14,".text":10577}},"a9b48697deff2e53515d3c7611168466ce81f078":{"julia":{".got.plt":48,".gnu.hash":104,".dynsym":480,".plt.got":8,".ctors":16,"Total":9478,".tbss":8,".rodata":4,".plt":64,".init":36,".got":8,".dynstr":331,".bss":24,".gnu.version":40,".hash":156,".rela.plt":72,".rela.dyn":24,".comment":34,".dtors":16,".dynamic":512,".data":16,".fini":14,".text":545,".interp":28},"sys.so":{".got.plt":696,".gnu.hash":60,".dynsym":2568,".plt.got":16,".ctors":16,".lrodata":270592,"Total":197961214,".rodata":91892,".plt":1360,".init":36,".got":128,".dynstr":1839,".bss":16,".gnu.version":214,".hash":824,".rela.plt":2016,".rela.dyn":1134408,".comment":17,".dtors":16,".dynamic":480,".data":8,".fini":14,".lbss":107104,".text":29877457,".ldata":116961600},"libjulia.so":{".got.plt":264,".data.rel.ro":9688,".gnu.hash":5568,".dynsym":18600,".plt.got":16,".ctors":16,"Total":199055,".rodata":15066,".plt":496,".init":36,".got":64,".dynstr":14236,".bss":7144,".gnu.version":1550,".hash":5192,".rela.plt":720,".rela.dyn":29160,".comment":17,".dtors":16,".dynamic":560,".data":1056,".fini":14,".text":10577}}}

sha_to_timings_test.json

@@ -0,0 +1 @@
+{"7c5c724ed93015d2e9c52c7fcd8f5c4ebbefff03":{"elapsed":[0.13,0.12,0.12],"system":[0.04,0.04,0.03],"user":[0.09,0.08,0.09],"outputs":[0.0,0.0,0.0],"minor":[20351.0,20348.0,20354.0],"swaps":[0.0,0.0,0.0],"maxresident":[175020.0,174656.0,175380.0],"major":[0.0,0.0,0.0],"avgtext":[0.0,0.0,0.0],"avgdata":[0.0,0.0,0.0],"inputs":[0.0,0.0,0.0]},"a14cc38512b6daab6b8417ebb8a64fc794ff89cc":{"elapsed":[0.2,0.15,0.17],"system":[0.07,0.08,0.06],"user":[1.59,0.19,0.48],"outputs":[0.0,0.0,0.0],"minor":[20574.0,20577.0,20570.0],"swaps":[0.0,0.0,0.0],"maxresident":[180876.0,180804.0,180720.0],"major":[0.0,0.0,0.0],"avgtext":[0.0,0.0,0.0],"avgdata":[0.0,0.0,0.0],"inputs":[0.0,0.0,0.0]},"ded0b28f7f94c2785a6330786fe5efea2a440c86":{"elapsed":[0.14,0.14,0.13],"system":[0.04,0.08,0.06],"user":[0.35,0.31,0.32],"outputs":[0.0,0.0,0.0],"minor":[20574.0,20576.0,20558.0],"swaps":[0.0,0.0,0.0],"maxresident":[181184.0,181504.0,180168.0],"major":[0.0,0.0,0.0],"avgtext":[0.0,0.0,0.0],"avgdata":[0.0,0.0,0.0],"inputs":[0.0,0.0,0.0]},"c5994e4b2b4277ed56887a51f523149ed0c87bf5":{"elapsed":[0.13,0.12,0.12],"system":[0.05,0.06,0.05],"user":[0.08,0.06,0.07],"outputs":[0.0,0.0,0.0],"minor":[20361.0,20367.0,20363.0],"swaps":[0.0,0.0,0.0],"maxresident":[174888.0,174940.0,174920.0],"major":[0.0,0.0,0.0],"avgtext":[0.0,0.0,0.0],"avgdata":[0.0,0.0,0.0],"inputs":[0.0,0.0,0.0]},"dd1ed17ae3c2e4e257f2444bbafd705326b4bbbd":{"elapsed":[0.12,0.12,0.12],"system":[0.05,0.05,0.03],"user":[0.07,0.08,0.09],"outputs":[0.0,0.0,0.0],"minor":[20391.0,20383.0,20376.0],"swaps":[0.0,0.0,0.0],"maxresident":[176080.0,175684.0,175252.0],"major":[0.0,0.0,0.0],"avgtext":[0.0,0.0,0.0],"avgdata":[0.0,0.0,0.0],"inputs":[0.0,0.0,0.0]},"0d30be8d74a921ace3b405c5d47367d50872176b":{"elapsed":[0.14,0.13,0.15],"system":[0.09,0.05,0.07],"user":[0.33,0.33,0.32],"outputs":[0.0,0.0,0.0],"minor":[20566.0,20569.0,20569.0],"swaps":[0.0,0.0,0.0],"maxresident":[181036.0,181220.0,181040.0],"major":[0.0,0.0,0.0],"avgtext":[0.0,0.0,0.0],"avgdata":[0.0,0.0,0.0],"inputs":[0.0,0.0,0.0]},"6f39acb9fb2b34b43d6b455d196b40f1759c9031":{"elapsed":[0.17,0.13,0.13],"system":[0.07,0.04,0.04],"user":[1.47,0.25,0.25],"outputs":[0.0,0.0,0.0],"minor":[20543.0,20547.0,20531.0],"swaps":[0.0,0.0,0.0],"maxresident":[180384.0,180280.0,179632.0],"major":[0.0,0.0,0.0],"avgtext":[0.0,0.0,0.0],"avgdata":[0.0,0.0,0.0],"inputs":[0.0,0.0,0.0]},"320366bbc00331671b38f7aba15812ffabcf1fcb":{"elapsed":[0.14,0.13,0.13],"system":[0.06,0.04,0.06],"user":[0.34,0.32,0.32],"outputs":[0.0,0.0,0.0],"minor":[20569.0,20545.0,20606.0],"swaps":[0.0,0.0,0.0],"maxresident":[181532.0,180300.0,180552.0],"major":[0.0,0.0,0.0],"avgtext":[0.0,0.0,0.0],"avgdata":[0.0,0.0,0.0],"inputs":[0.0,0.0,0.0]},"a9b48697deff2e53515d3c7611168466ce81f078":{"elapsed":[0.15,0.15,0.14],"system":[0.22,0.04,0.07],"user":[0.22,0.38,0.31],"outputs":[0.0,0.0,0.0],"minor":[20558.0,20562.0,20556.0],"swaps":[0.0,0.0,0.0],"maxresident":[180684.0,181420.0,181100.0],"major":[0.0,0.0,0.0],"avgtext":[0.0,0.0,0.0],"avgdata":[0.0,0.0,0.0],"inputs":[0.0,0.0,0.0]}}