diff --git a/reports/css/d3-style.css b/reports/css/d3-style.css index 33b1552..124f4e1 100644 --- a/reports/css/d3-style.css +++ b/reports/css/d3-style.css @@ -25,8 +25,9 @@ fill: steelblue; .d3-tip { line-height: 1; - font-weight: bold; - padding: 12px; + border: 2px solid; + font-size: 12px; + padding: 6px; background: rgba(0, 0, 0, 0.9); color: #fff; border-radius: 2px; @@ -35,7 +36,7 @@ fill: steelblue; .d3-tip:after { box-sizing: border-box; display: inline; - font-size: 10px; + font-size: 8px; width: 100%; line-height: 1; color: rgba(0, 0, 0, 0.9); @@ -69,6 +70,7 @@ svg text { } .library-select-div { + clear: both; min-width: 140px; float: left; height: 20px; @@ -156,14 +158,15 @@ svg text { padding: 10px; } -.method-select-label, .param-select-label, .main-dataset-select-label { +.method-select-label, .param-select-label, .main-dataset-select-label, +.sweep-select-label, .metric-select-label { float: left; font-size: 75%; color: #ffffff; margin-right: 8px; } -#method_select, #param_select, #main_dataset_select, +#method_select, #param_select, #sweep_select, #main_dataset_select, #metric_select, #option_select { float: left; margin-right: 20px; diff --git a/reports/index.html b/reports/index.html index b88bd19..013d01a 100644 --- a/reports/index.html +++ b/reports/index.html @@ -43,7 +43,23 @@ - +
+ + +
+
+ + +
+
+ + +
@@ -67,5 +83,8 @@ + + + diff --git a/reports/js/benchmarks.js b/reports/js/benchmarks.js index d6b2fc6..47e36ee 100644 --- a/reports/js/benchmarks.js +++ b/reports/js/benchmarks.js @@ -74,7 +74,8 @@ var color = d3.scale.ordinal().range(["#98abc5", "#8a89a6", "#7b6888", "#6b486b" */ function mapRuntime(runtime, max) { - if (runtime == ">9000") { return max; } + if (String(runtime).indexOf(">") != -1) { return max; } + else if (runtime == "timeout") { return max; } else if (runtime == "failure") { return 0; } else { return runtime; } } @@ -140,6 +141,9 @@ function chartTypeSelect() else if (chartType == "dataset-comparison") { activeChartType = dc; } else if (chartType == "metric-comparison") { activeChartType = mc; } else if (chartType == "highest-metric-comparison") { activeChartType = hmc; } + else if (chartType == "sweep-runtime-comparison") { activeChartType = sc; } + else if (chartType == "sweep-metric-comparison") { activeChartType = smc; } + else if (chartType == "all-sweep-metric-comparison") { activeChartType = apmc; } activeChartType.onTypeSelect(); } diff --git a/reports/js/benchmarks/all-params-metric-comparison-view.js b/reports/js/benchmarks/all-params-metric-comparison-view.js new file mode 100644 index 0000000..b3b8f26 --- /dev/null +++ b/reports/js/benchmarks/all-params-metric-comparison-view.js @@ -0,0 +1,618 @@ +// Define namespace: apmc = all parameters metric comparison. +var apmc = apmc = apmc || {}; + +apmc.methodName = ""; +apmc.dataset = ""; +apmc.libraries = []; +apmc.activeLibraries = []; +apmc.results = []; +apmc.sweepId = -1; + +// This chart type has been selected. +apmc.onTypeSelect = function() +{ + // The user needs to be able to select a method, then parameters, then a + // dataset. + var selectHolder = d3.select(".selectholder"); + selectHolder.append("label") + .attr("for", "method_select") + .attr("class", "method-select-label") + .text("Select method:"); + selectHolder.append("select") + .attr("id", "method_select") + .attr("onchange", "apmc.methodSelect()"); + selectHolder.append("label") + .attr("for", "dataset_select") + .attr("class", "dataset-select-label") + .text("Select dataset:"); + selectHolder.append("select") + .attr("id", "dataset_select") + .attr("onchange", "apmc.datasetSelect()"); + selectHolder.append("label") + .attr("for", "metric_select") + .attr("class", "metric-select-label") + .text("Select metric:"); + selectHolder.append("select") + .attr("id", "metric_select") + .attr("onchange", "apmc.metricSelect()"); + + apmc.listMethods(); +} + +apmc.clear = function() +{ + apmc.clearChart(); +} + +apmc.process_library_name = function(l, p) +{ + // We want to turn the library name and parameters dict into, e.g., + // mlpack: max_iterations=sweep(10, 10, 100), algorithm=lbfgs + var output = l; + var i = 0; + var d = jQuery.parseJSON(p); + if (Object.keys(d).length > 0) + { + output += ": "; + for (pp in d) + { + output += pp + "=" + d[pp]; + i++; + if (i != Object.keys(d).length) + output += ";"; + } + } + + return output; +} + +// List the available methods where there is a sweep. +apmc.listMethods = function() +{ + var methods = dbExec("SELECT DISTINCT methods.name FROM methods, results " + + "WHERE methods.id = results.method_id ORDER BY name;"); + var methodSelectBox = document.getElementById("method_select"); + + // Remove old things. + clearSelectBox(methodSelectBox); + + // Add new things. + var length = dbType === "sqlite" ? methods[0].values.length : methods.length; + for (i = 0; i < length; i++) + { + var newOption = document.createElement("option"); + newOption.text = dbType === "sqlite" ? methods[0].values[i] : + methods[i].name; + methodSelectBox.add(newOption); + } + methodSelectBox.selectedIndex = -1; + + // Clear dataset box. + clearSelectBox(document.getElementById("dataset_select")); +} + +// Called when the user selects a method. +apmc.methodSelect = function() +{ + // Extract the name of the method we selected. + var methodSelectBox = document.getElementById("method_select"); + apmc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; // At higher scope. + + // Now we need to get the datasets. + var sqlstr = "SELECT DISTINCT datasets.name FROM datasets, results, methods " + + "WHERE datasets.id = results.dataset_id AND methods.name = '" + + apmc.methodName + "' AND results.method_id = methods.id;"; + var datasets = dbExec(sqlstr); + + // Loop through results and fill the second list box. + var datasetSelectBox = document.getElementById("dataset_select"); + clearSelectBox(datasetSelectBox); + + var newOption = document.createElement("option"); + datasetSelectBox.add(newOption); + + datasetSelectBox.selectedIndex = 0; + if ((dbType === "sqlite" && datasets[0]) || (dbType === "mysql" && datasets)) + { + // Put in the datasets. + var length = dbType === "sqlite" ? datasets[0].values.length : datasets.length; + for (i = 0; i < length; i++) + { + newOption = document.createElement("option"); + + var datasetName = dbType === "sqlite" ? datasets[0].values[i][0] : datasets[i].name; + newOption.text = datasetName; + datasetSelectBox.add(newOption); + } + } + + datasetSelectBox.selectedIndex = 0; +} + +apmc.datasetSelect = function() +{ + // The user selected a dataset, so now we can ask them to select the metric + // they want. + var methodSelectBox = document.getElementById("method_select"); + apmc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; + var datasetSelectBox = document.getElementById("dataset_select"); + var datasetName = + datasetSelectBox.options[datasetSelectBox.selectedIndex].text; + + // What metrics do we have available? We can do our actual query for results + // here, then we need to parse it. We want all of the most recent results. + var sqlstr = "SELECT metrics.metric, metrics.build_id, metrics.sweep_id, " + + "metrics.sweep_elem_id, libraries.name, methods.parameters " + + "FROM datasets, methods, metrics, libraries " + + "WHERE metrics.libary_id = libraries.id" + + " AND datasets.id = metrics.dataset_id" + + " AND datasets.name = '" + datasetName + "'" + + " AND methods.name = '" + apmc.methodName + "'" + + " AND metrics.method_id = methods.id;"; + apmc.results = dbExec(sqlstr); + apmc.results = dbType === "sqlite" ? apmc.results[0].values : apmc.results; + + // Also determine the maximum build number for a given library. + apmc.libraryVersions = {} + addLibrary = function(p, c) + { + var lib = (dbType === "sqlite") ? c[4] : c.name; + var params = (dbType === "sqlite") ? c[5] : c.parameters; + var bid = (dbType === "sqlite") ? c[1] : c.build_id; + + if (lib in apmc.libraryVersions) + { + if (params in apmc.libraryVersions[lib]) + { + if (apmc.libraryVersions[lib][params] < bid) + apmc.libraryVersions[lib][params] = bid; + } + else + { + apmc.libraryVersions[lib][params] = bid; + } + } + else + { + apmc.libraryVersions[lib] = {} + apmc.libraryVersions[lib][params] = bid; + } + + return; + } + apmc.results.reduce(addLibrary, []); + + // Filter out rows that weren't the newest. + apmc.results = apmc.results.filter(function(c) + { + return (((dbType === "sqlite") ? c[1] : c.build_id) == + apmc.libraryVersions[((dbType === "sqlite") ? c[4] : c.name)][((dbType === "sqlite") ? c[5] : c.params)]); + }); + + // Lastly, we need to gather all of the sweeps we might have. + apmc.sweeps = {} + addSweep = function(p, c) + { + var params = jQuery.parseJSON((dbType === "sqlite") ? c[5] : c.parameters); + for (e in params) + { + if (String(params[e]).indexOf("sweep(") != -1) + { + // Do we already know about this sweep? + if (String(params[e]) in apmc.sweeps) + continue; + + // Great, we have a sweep---now we need to get the information. + sqlstr = "SELECT type, begin, step, end FROM sweeps WHERE id = " + + ((dbType === "sqlite") ? c[2] : c.sweep_id) + ";"; + var sweepResults = dbExec(sqlstr); + // There should only be one result... + sweepResults = (dbType === "sqlite" ? sweepResults[0].values[0] : sweepResults[0]); + + // Insert these results into the dict. + apmc.sweeps[String(params[e])] = + { "type": (dbType === "sqlite" ? sweepResults[0] : sweepResults.type), + "begin": (dbType === "sqlite" ? sweepResults[1] : sweepResults.begin), + "step": (dbType === "sqlite" ? sweepResults[2] : sweepResults.step), + "end": (dbType === "sqlite" ? sweepResults[3] : sweepResults.end) }; + } + } + } + apmc.results.reduce(addSweep, []); + + // Now we have to parse through the metrics and see what we find. + addMetric = function(p, c) + { + var json = jQuery.parseJSON(dbType === "sqlite" ? c[0] : c.metric); + for(var k in json) + if(p.indexOf(k) < 0) + p.push(k); + return p; + }; + metrics = apmc.results.reduce(addMetric, []); + + var metric_select_box = document.getElementById("metric_select"); + clearSelectBox(metric_select_box); + for (i = 0; i < metrics.length; i++) + { + var new_option = document.createElement("option"); + new_option.text = metrics[i]; + metric_select_box.add(new_option); + } + metric_select_box.selectedIndex = -1; +} + +apmc.metricSelect = function() +{ + // We've already got the results, and now the user has specified the metric + // they want plotted. + var metricSelectBox = document.getElementById("metric_select"); + apmc.metricName = metricSelectBox.options[metricSelectBox.selectedIndex].text; + + // Obtain unique list of libraries. + apmc.libraries = apmc.results.map( + function(d) { + return dbType === "sqlite" ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.lib, d.parameters); + }).reduce( + function(p, c) { + if (p.indexOf(c) < 0) p.push(c); return p; + }, []); + console.log(apmc.libraries); + + // By default, all libraries are active. + apmc.activeLibraries = {}; + for (i = 0; i < apmc.libraries.length; ++i) + { + apmc.activeLibraries[apmc.libraries[i]] = true; + } + + clearChart(); + buildChart(); +} + +apmc.clearChart = function() +{ + d3.select("svg").remove(); + d3.selectAll(".d3-tip").remove(); + d3.selectAll(".library-select-title").remove(); + d3.selectAll(".library_select_div").remove(); + d3.selectAll(".legendholder").selectAll("*").remove(); +} + +apmc.extractRuntime = function(d) +{ + var json = jQuery.parseJSON(d); + for (var m in json) + { + if (m == "Runtime") + { + if (json[m] == -2) + return "failure"; + else if (json[m] == -1) + return "timeout"; + else if (String(json[m]).indexOf(">") != -1) + return "timeout"; + else + return json[m]; + } + } + + return "failure"; +} + +apmc.extractMetric = function(d, metricName, notFoundValue) +{ + var json = jQuery.parseJSON(d); + for (var m in json) + if (m == metricName) + return json[m]; + + return notFoundValue; +} + +apmc.buildChart = function() +{ + // Get lists of active library/param combinations. + var activeLibraryList = apmc.libraries.map(function(d) { return d; }).reduce( + function(p, c) + { + if (apmc.activeLibraries[c] == true) + p.push(c); + return p; + }, []); + + var maxRuntime = d3.max(apmc.results, + function(d) + { + if (apmc.activeLibraries[dbType === "sqlite" ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.name, d.parameters)] == false) + { + return 0; + } + else + { + x = apmc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric); + if (x === "failure" || x === "timeout") + return 0; + else + return x; + } + }); + + var maxMetric = d3.max(apmc.results, + function(d) + { + if (apmc.activeLibraries[dbType === "sqlite" ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.name, d.parameters)] == false) + return 0; + else + return apmc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, + apmc.metricName, 0); + }); + + // Increase so we have 16 spare pixels at the top. + maxMetric *= ((height + 16) / height); + + console.log(maxRuntime); + var runtimeScale = d3.scale.linear() + .domain([0, maxRuntime]) + .range([0, width]); + + // We need to find out how big the sweep is. +// var sweepSql = "SELECT type, begin, step, end FROM sweeps where id = " + apmc.sweepId; +// var sweepResults = dbExec(sweepSql); +// sweepResults = dbType === "sqlite" ? sweepResults[0].values : sweepResults.results; + +// var func = (dbType === "sqlite" ? sweepResults[0][0] : sweepResults[0].type) === "int" ? parseInt : parseFloat; +// var start = func(dbType === "sqlite" ? sweepResults[0][1] : sweepResults[0].start); +// var step = func(dbType === "sqlite" ? sweepResults[0][2] : sweepResults[0].step); +// var end = func(dbType === "sqlite" ? sweepResults[0][3] : sweepResults[0].end); + + var metricScale = d3.scale.linear() + .domain([0, maxMetric]) + .range([height, 0]); + + var xAxis = d3.svg.axis().scale(runtimeScale).orient("bottom"); + var yAxis = d3.svg.axis().scale(metricScale).orient("left"); + + // Create svg object. + var svg = d3.select(".svgholder").append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + // Add x axis. + svg.append("g").attr("id", "xaxis") + .attr("class", "x axis") + .attr("transform", "translate(0, " + height + ")") + .call(xAxis) + .append("text") + .style("text-anchor", "end") + .attr("dx", 500) + .attr("dy", "3em") + .text("Runtime (s)"); + + // Add y axis. + svg.append("g").attr("id", "yaxis") + .attr("class", "y axis") + .call(yAxis) + .append("text") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text(apmc.metricName); + + // Create tooltips. + var tip = d3.tip() + .attr("class", "d3-tip") + .offset([-10, 0]) + .html(function(d) { + var runtime = apmc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric); + if (runtime != "timeout" && runtime != "failure") { + runtime = runtime.toFixed(3); + } + var metricValue = apmc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, apmc.metricName, ""); + if (metricValue != "timeout" && metricValue != "failure" && +String(metricValue).indexOf(">") == -1 && metricValue != "") + { + metricValue = metricValue.toFixed(3); + } + else if (metricValue == "") + { + metricValue = "failure"; + } + + var libName = (dbType === "sqlite" ? d[4] : d.name); + var paramName = (dbType === "sqlite" ? d[5] : d.parameters); + var start = 0; + var step = 0; + // Goal: output, e.g., + // mlpack: + // non_sweep_param_1=x, non_sweep_param_2=x, sweep_param=x + // metric: + // runtime: + // All centered with a border in the desired color. + output = "" + libName + "
"; + // Loop over parameters. + params = jQuery.parseJSON(paramName); + for (p in params) + { + output += p + ": "; + // Is it a sweep option? + if (String(params[p]).indexOf("sweep(") != -1) + { + // Get sweep element. + var sweepElemId = (dbType === "sqlite" ? d[3] : d.sweep_elem_id); + + var func = (apmc.sweeps[params[p]]['type'] === "int" ? parseInt : parseFloat); + var start = func(apmc.sweeps[params[p]]['begin']); + var step = func(apmc.sweeps[params[p]]['step']); + + output += (start + step * sweepElemId); + } + else + { + output += String(params[p]); + } + output += "; "; + } + output += "
"; + output += apmc.metricName + ": " + metricValue + "
"; + output += "Runtime: " + runtime; + if (runtime != "failure" && runtime != "timeout") + { + output += "s
"; + } + + return output; + }); + svg.call(tip); + + // Add all of the data points. + var lineFunc = d3.svg.line() + .x(function(d) { return runtimeScale(mapRuntime(apmc.extractRuntime( + dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .y(function(d) { return metricScale(apmc.extractMetric( + dbType === "sqlite" ? d[0] : d.metric, apmc.metricName, 0)); }) + .interpolate("linear"); + + // Add all of the data points. + for (var l in apmc.libraryVersions) + { + for (var p in apmc.libraryVersions[l]) + { + if (!apmc.activeLibraries[apmc.process_library_name(l, p)]) + continue; + + // Now we have a library/parameters combination. + // Collect all the data and let's see if we have a sweep or just a single + // point. + var res = apmc.results.filter(function(c) + { + var ln = (dbType === "sqlite") ? c[4] : c.name; + var pn = (dbType === "sqlite") ? c[5] : c.parameters; + return ((ln === l) && (pn === p)); + }); + + if (res.length > 0) + { + svg.append('svg:path') + .attr('d', lineFunc(res)) + .attr('stroke', color(l + ': ' + p)) + .attr('stroke-width', 2) + .attr('fill', 'none'); + + // Colored circle enclosed in white circle enclosed in background color + // circle; looks kind of nice. + svg.selectAll("dot").data(res).enter().append("circle") + .attr("r", 6) + .attr("cx", function(d) { return runtimeScale(mapRuntime(apmc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .attr("cy", function(d) { return metricScale(apmc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, apmc.metricName, 0)); }) + .attr('fill', '#222222') + .on('mouseover', function(d, i) + { + d3.select('.d3-tip').style("border-color", color((dbType === "sqlite") ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.name, d.parameters))); + tip.show(d, i); + }) + .on('mouseout', tip.hide); + svg.selectAll("dot").data(res).enter().append("circle") + .attr("r", 4) + .attr("cx", function(d) { return runtimeScale(mapRuntime(apmc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .attr("cy", function(d) { return metricScale(apmc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, apmc.metricName, 0)); }) + .attr('fill', '#ffffff') + .on('mouseover', function(d, i) + { + d3.select('.d3-tip').style("border-color", color((dbType === "sqlite") ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.name, d.parameters))); + tip.show(d, i); + }) + .on('mouseout', tip.hide); + svg.selectAll("dot").data(res).enter().append("circle") + .attr("r", 3) + .attr("cx", function(d) { return runtimeScale(mapRuntime(apmc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .attr("cy", function(d) { return metricScale(apmc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, apmc.metricName, 0)); }) + .attr('fill', function(d) { return color((dbType === "sqlite") ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.name, d.parameters)) }) + .on('mouseover', function(d, i) + { + d3.select('.d3-tip').style("border-color", color((dbType === "sqlite") ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.name, d.parameters))); + tip.show(d, i); + }) + .on('mouseout', tip.hide); + } + } + } + + // Create the library selector. + var librarySelectTitle = d3.select(".legendholder").append("div") + .attr("class", "library-select-title"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-text") + .text("Libraries:"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-open-paren") + .text("("); + librarySelectTitle.append("div") + .attr("class", "library-select-title-enable-all") + .text("enable all") + .on('click', function() { apmc.enableAllLibraries(); }); + librarySelectTitle.append("div") + .attr("class", "library-select-title-bar") + .text("|"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-disable-all") + .text("disable all") + .on('click', function() { apmc.disableAllLibraries(); }); + librarySelectTitle.append("div") + .attr("class", "library-select-title-close-paren") + .text(")"); + + var libraryDivs = d3.select(".legendholder").selectAll("input") + .data(apmc.libraries) + .enter() + .append("div") + .attr("class", "library-select-div") + .attr("id", function(d) { return d + '-library-checkbox-div'; }); + + libraryDivs.append("label") + .attr('for', function(d) { return d + '-library-checkbox'; }) + .style('background', color) + .attr('class', 'library-select-color'); + + libraryDivs.append("input") + .property("checked", function(d) { return apmc.activeLibraries[d]; }) + .attr("type", "checkbox") + .attr("id", function(d) { return d + '-library-checkbox'; }) + .attr('class', 'library-select-box') + .attr("onClick", function(d, i) { return "apmc.toggleLibrary('" + d + "');"; }); + + libraryDivs.append("label") + .attr('for', function(d) { return d + '-library-checkbox'; }) + .attr('class', 'library-select-label') + .text(function(d) { return d; }); +} + +// Toggle a library to on or off. +apmc.toggleLibrary = function(library) +{ + apmc.activeLibraries[library] = !apmc.activeLibraries[library]; + + clearChart(); + buildChart(); +} + +// Set all libraries on. +apmc.enableAllLibraries = function() +{ + for (v in apmc.activeLibraries) { apmc.activeLibraries[v] = true; } + + clearChart(); + buildChart(); +} + +// Set all libraries off. +apmc.disableAllLibraries = function() +{ + for (v in apmc.activeLibraries) { apmc.activeLibraries[v] = false; } + + clearChart(); + buildChart(); +} diff --git a/reports/js/benchmarks/historical-comparison-view.js b/reports/js/benchmarks/historical-comparison-view.js index c38e74c..ce03534 100644 --- a/reports/js/benchmarks/historical-comparison-view.js +++ b/reports/js/benchmarks/historical-comparison-view.js @@ -100,7 +100,7 @@ hc.paramSelect = function() var param_select_box = document.getElementById("param_select"); var param_name_full = param_select_box.options[param_select_box.selectedIndex].text; // Parse out actual parameters. - hc.param_name = param_name_full.split("(")[0].replace(/^\s+|\s+$/g, ''); // At higher scope. + hc.param_name = param_name_full.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. if (hc.param_name == "[no parameters]") { hc.param_name = ""; } // Given a method name and parameters, query the SQLite database for all of diff --git a/reports/js/benchmarks/metric-multiple-parameter-comparison-view.js b/reports/js/benchmarks/metric-multiple-parameter-comparison-view.js index 52cf937..1601d2d 100644 --- a/reports/js/benchmarks/metric-multiple-parameter-comparison-view.js +++ b/reports/js/benchmarks/metric-multiple-parameter-comparison-view.js @@ -97,11 +97,13 @@ mmpc.listOptions = function() "AND methods.name = '" + mmpc.method_name + "' " + "AND datasets.name = '" + mmpc.dataset_name + "';"; var results = dbExec(sqlstr); + console.log(JSON.stringify(results)); results = dbType === "sqlite" ? results[0].values : results; var addOption = function(p, c) { var options = mmpc.getOptionList(dbType === "sqlite" ? c.toString() : c.parameter); + console.log(JSON.stringify(options)); for (i = 0; i < options.length; i++) if(p.indexOf(options[i]) < 0) p.push(options[i]); return p; @@ -131,6 +133,7 @@ mmpc.listMetrics = function() "AND datasets.name = '" + mmpc.dataset_name + "';"; var results = dbExec(sqlstr); results = dbType === "sqlite" ? results[0].values : results; + console.log(results); addMetric = function(p, c) { var json = jQuery.parseJSON(dbType === "sqlite" ? c : c.metric); @@ -198,9 +201,10 @@ mmpc.metricSelect = function() "AND methods.name = '" + mmpc.method_name + "' " + "AND datasets.name = '" + mmpc.dataset_name + "';"; mmpc.results = dbExec(sqlstr); + console.log(sqlstr); mmpc.results = dbType === "sqlite" ? mmpc.results[0].values : mmpc.results; - console.log(mmpc.results) + console.log(JSON.stringify(mmpc.results)); var filterAndSet = function(p, d) { var metrics = jQuery.parseJSON(dbType === "sqlite" ? d[0] : d.metric); @@ -358,13 +362,15 @@ mmpc.clearChart = function() // Return a param's value from a list of parameters. mmpc.getOptionValue = function(str, opt) { - var list = str.split("-" + opt) - if (list.length < 2) - return ""; - list = list[1].split(/[\s=]+/); - if (list.length < 2) - return ""; - return list[1]; + var options = JSON.parse(str); + console.log("getOptionValue: " + str); + if (opt in options) + { + // Then we know the value. + return options[opt]; + } + + return ""; } // Remove a parameter from the list of parameters. @@ -376,11 +382,12 @@ mmpc.removeOption = function(str, opt) // Returns a list of parameters. mmpc.getOptionList = function(str) { - optList = str.split(/-+/) - .map(function (d) {return d.replace(/^\s+|\s+$/g, '').split(/[\s=]+/); }) - .filter(function (d) {return (d.length >= 1);}) - .map(function (d) {return d[0];}); - optList.shift(); + var opts = JSON.parse(str); + var optList = []; + for (var k in opts) + { + optList.push(k); + } return optList; } diff --git a/reports/js/benchmarks/metric-parameter-comparison-view.js b/reports/js/benchmarks/metric-parameter-comparison-view.js index e8f3e6b..03f685c 100644 --- a/reports/js/benchmarks/metric-parameter-comparison-view.js +++ b/reports/js/benchmarks/metric-parameter-comparison-view.js @@ -113,7 +113,7 @@ mpc.paramSelect = function() var param_select_box = document.getElementById("param_select"); var param_name_full = param_select_box.options[param_select_box.selectedIndex].text; // Parse out actual parameters. - mpc.param_name = param_name_full.split("(")[0].replace(/^\s+|\s+$/g, ''); // At higher scope. + mpc.param_name = param_name_full.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. if (mpc.param_name == "[no parameters]") { mpc.param_name = ""; } if (mpc.param_name == "-") diff --git a/reports/js/benchmarks/runtime-comparison-view.js b/reports/js/benchmarks/runtime-comparison-view.js index f782756..b667bfe 100644 --- a/reports/js/benchmarks/runtime-comparison-view.js +++ b/reports/js/benchmarks/runtime-comparison-view.js @@ -19,22 +19,22 @@ rc.onTypeSelect = function() .attr("for", "method_select") .attr("class", "method-select-label") .text("Select method:"); - selectHolder.append("select") + selectHolder.append("select") .attr("id", "method_select") .attr("onchange", "rc.methodSelect()"); - selectHolder.append("label") + selectHolder.append("label") .attr("for", "param_select") .attr("class", "param-select-label") .text("Select parameters:"); - selectHolder.append("select") + selectHolder.append("select") .attr("id", "param_select") .attr("onchange", "rc.paramSelect()"); - selectHolder.append("br"); - selectHolder.append("label") + selectHolder.append("br"); + selectHolder.append("label") .attr("for", "main_dataset_select") .attr("class", "main-dataset-select-label") .text("Sort results by:"); - selectHolder.append("select") + selectHolder.append("select") .attr("id", "main_dataset_select") .attr("onchange", "rc.orderSelect()"); @@ -94,8 +94,9 @@ rc.methodSelect = function() var method_select_box = document.getElementById("method_select"); rc.method_name = method_select_box.options[method_select_box.selectedIndex].text; // At higher scope. - var sqlstr = "SELECT DISTINCT methods.parameters, metrics.libary_id, COUNT(DISTINCT metrics.libary_id) AS count FROM methods, metrics WHERE methods.name = '" + rc.method_name + "' AND methods.id = metrics.method_id GROUP BY methods.parameters;"; + var sqlstr = "SELECT DISTINCT methods.parameters, methods.sweep_id, metrics.libary_id, COUNT(DISTINCT metrics.libary_id) AS count FROM methods, metrics WHERE methods.name = '" + rc.method_name + "' AND methods.id = metrics.method_id GROUP BY methods.parameters;"; var params = dbExec(sqlstr); + console.log(JSON.stringify(params)); // Loop through results and fill the second list box. var param_select_box = document.getElementById("param_select"); @@ -113,7 +114,8 @@ rc.methodSelect = function() var new_option = document.createElement("option"); var parameters = dbType === "sqlite" ? params[0].values[i][0] : params[i].parameters; - var libraries = dbType === "sqlite" ? params[0].values[i][2] : params[i].count; + var sweepId = dbType === "sqlite" ? params[0].values[i][1] : params[i].sweep_id; + var libraries = dbType === "sqlite" ? params[0].values[i][3] : params[i].count; if (parameters) { @@ -123,6 +125,13 @@ rc.methodSelect = function() { new_option.text = "[no parameters] (" + libraries + " libraries)"; } + + if (sweepId !== -1) + { + new_option.text += " [sweep]"; + } + new_option.id = sweepId; + param_select_box.add(new_option); } } @@ -139,14 +148,80 @@ rc.paramSelect = function() rc.method_name = method_select_box.options[method_select_box.selectedIndex].text; var param_select_box = document.getElementById("param_select"); var param_name_full = param_select_box.options[param_select_box.selectedIndex].text; + var sweep_id = param_select_box.options[param_select_box.selectedIndex].id; + + // Remove the sweep box. + var ss = document.getElementById("sweep_select") + if (ss !== null) + { + ss.parentNode.removeChild(ss); + } + ss = document.getElementById("sweep_select_label") + if (ss !== null) + { + ss.parentNode.removeChild(ss); + } // Parse out actual parameters. - rc.param_name = param_name_full.split("(")[0].replace(/^\s+|\s+$/g, ''); // At higher scope. + rc.param_name = param_name_full.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. if (rc.param_name == "[no parameters]") { rc.param_name = ""; } + // If the user selected a sweep, we can't actually plot anything yet. + if (sweep_id !== -1) + { + // Instead we have to create an extra selector to know what element of the + // sweep they want to look at. + var selectHolder = d3.select(".selectholder"); + selectHolder.insert("label", "#param_select + *") + .attr("for", "sweep_select") + .attr("class", "sweep-select-label") + .attr("id", "sweep_select_label") + .text("Select sweep element:"); + selectHolder.insert("select", "#sweep_select_label + *") + .attr("id", "sweep_select") + .attr("onchange", "rc.sweepSelect()"); + + // Now populate the sweep elements. + var sweepsql = "SELECT type, begin, step, end FROM sweeps where id = " + sweep_id; + rc.results = dbExec(sweepsql); + console.log(sweepsql); + rc.results = dbType === "sqlite" ? rc.results[0].values : rc.results; + + // Get the parameter name we are sweeping. + var params = JSON.parse(rc.param_name); + var name = ""; + for (var i in params) + { + if (params[i].search(/sweep\(/) !== -1) + { + name = i; + break; + } + } + + // sweep_info: [start, step, end] + var sweepSelectBox = document.getElementById("sweep_select"); + sweepSelectBox.add(document.createElement("option")); + var func = (dbType === "sqlite" ? rc.results[0][0] : rc.results[0].type) === "int" ? parseInt : parseFloat; + var start = func(dbType === "sqlite" ? rc.results[0][1] : rc.results[0].start); + var step = func(dbType === "sqlite" ? rc.results[0][2] : rc.results[0].step); + var end = func(dbType === "sqlite" ? rc.results[0][3] : rc.results[0].end); + var elem = 0; + for (var i = start; i <= end; i += step, elem++) + { + var new_option = document.createElement("option"); + new_option.text = name + ": " + i; + new_option.id = elem; + sweepSelectBox.add(new_option); + } + sweepSelectBox.selectedIndex = 0; + + return; + } + // Extract the name of the method we selected. var order_select_box = document.getElementById("main_dataset_select"); rc.groupBy = "datasets." + order_select_box.options[order_select_box.selectedIndex].text; // At higher scope. @@ -169,7 +244,76 @@ rc.paramSelect = function() "FROM results, datasets, methods, libraries WHERE " + "results.dataset_id = datasets.id AND results.method_id = methods.id AND methods.name = '" + rc.method_name + "' AND methods.parameters = '" + rc.param_name + "' AND libraries.id = results.libary_id ORDER BY bid DESC " + ") tmp GROUP BY lid, " + rc.groupBy + ", did;"; + console.log(sqlstr); + rc.results = dbExec(sqlstr); + console.log(JSON.stringify(rc.results)); + rc.results = dbType === "sqlite" ? rc.results[0].values : rc.results; + + // Obtain unique list of datasets. + rc.datasets = rc.results.map(function(d) { return dbType === "sqlite" ? d[4] : d.dataset; }).reduce(function(p, c) { if(p.indexOf(c) < 0) p.push(c); return p; }, []); + // Obtain unique list of libraries. + rc.libraries = rc.results.map(function(d) { return dbType === "sqlite" ? d[3] : d.lib; }).reduce(function(p, c) { if(p.indexOf(c) < 0) p.push(c); return p; }, []); + + // By default, everything is active. + rc.active_datasets = {}; + for (i = 0; i < rc.datasets.length; i++) + { + rc.active_datasets[rc.datasets[i]] = true; + } + + rc.active_libraries = {}; + for (i = 0; i < rc.libraries.length; i++) + { + rc.active_libraries[rc.libraries[i]] = true; + } + + clearChart(); + buildChart(); +} + +rc.sweepSelect = function() +{ + // Now we have selected a sweep ID and we can plot it. + var method_select_box = document.getElementById("method_select"); + rc.method_name = method_select_box.options[method_select_box.selectedIndex].text; + var param_select_box = document.getElementById("param_select"); + var param_name_full = param_select_box.options[param_select_box.selectedIndex].text; + var sweep_id = param_select_box.options[param_select_box.selectedIndex].id; + var sweep_select_box = document.getElementById("sweep_select"); + var sweep_element_id = sweep_select_box.options[sweep_select_box.selectedIndex].id; + + // Parse out actual parameters. + rc.param_name = param_name_full.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. + if (rc.param_name == "[no parameters]") + { + rc.param_name = ""; + } + + // Extract the name of the method we selected. + var order_select_box = document.getElementById("main_dataset_select"); + rc.groupBy = "datasets." + order_select_box.options[order_select_box.selectedIndex].text; // At higher scope. + + if (rc.groupBy == "datasets.instances") + { + rc.groupBy = "di"; + } + else if (rc.groupBy == "datasets.attributes") + { + rc.groupBy = "da"; + } + else + { + rc.groupBy = "ds"; + } + + var sqlstr = "SELECT DISTINCT * FROM " + + "(SELECT results.time as time, results.var as var, libraries.id, libraries.name as lib, datasets.name as dataset, datasets.id as did, libraries.id as lid, results.build_id as bid, datasets.instances as di, datasets.attributes as da, datasets.size as ds " + + "FROM results, datasets, methods, libraries WHERE " + + "results.dataset_id = datasets.id AND results.method_id = methods.id AND methods.name = '" + rc.method_name + "' AND methods.parameters = '" + rc.param_name + "' AND libraries.id = results.libary_id AND results.sweep_elem_id = " + sweep_element_id + " ORDER BY bid DESC " + + ") tmp GROUP BY lid, " + rc.groupBy + ", did;"; + console.log(sqlstr); rc.results = dbExec(sqlstr); + console.log(JSON.stringify(rc.results)); rc.results = dbType === "sqlite" ? rc.results[0].values : rc.results; // Obtain unique list of datasets. diff --git a/reports/js/benchmarks/sweep-comparison-view.js b/reports/js/benchmarks/sweep-comparison-view.js new file mode 100644 index 0000000..551a959 --- /dev/null +++ b/reports/js/benchmarks/sweep-comparison-view.js @@ -0,0 +1,488 @@ +// Define namespace: sc = sweep comparison. +var sc = sc = sc || {}; + +sc.methodName = ""; +sc.paramName = ""; +sc.dataset = ""; +sc.libraries = []; +sc.activeLibraries = []; +sc.results = []; +sc.sweepId = -1; + +// This chart type has been selected. +sc.onTypeSelect = function() +{ + // The user needs to be able to select a method, then parameters, then a + // dataset. + var selectHolder = d3.select(".selectholder"); + selectHolder.append("label") + .attr("for", "method_select") + .attr("class", "method-select-label") + .text("Select method:"); + selectHolder.append("select") + .attr("id", "method_select") + .attr("onchange", "sc.methodSelect()"); + selectHolder.append("label") + .attr("for", "param_select") + .attr("class", "param-select-label") + .text("Select parameters:"); + selectHolder.append("select") + .attr("id", "param_select") + .attr("onchange", "sc.paramSelect()"); + selectHolder.append("label") + .attr("for", "main_dataset_select") + .attr("class", "main-dataset-select-label") + .text("Select dataset:"); + selectHolder.append("select") + .attr("id", "main_dataset_select") + .attr("onchange", "sc.datasetSelect()"); + + sc.listMethods(); +} + +sc.clear = function() +{ + sc.clearChart(); +} + +// List the available methods where there is a sweep. +sc.listMethods = function() +{ + var methods = dbExec("SELECT DISTINCT methods.name FROM methods, results " + + "WHERE methods.id = results.method_id AND methods.sweep_id != -1 " + + "ORDER BY name;"); + var methodSelectBox = document.getElementById("method_select"); + + // Remove old things. + clearSelectBox(methodSelectBox); + + // Add new things. + var length = dbType === "sqlite" ? methods[0].values.length : methods.length; + for (i = 0; i < length; i++) + { + var newOption = document.createElement("option"); + newOption.text = dbType === "sqlite" ? methods[0].values[i] : + methods[i].name; + methodSelectBox.add(newOption); + } + methodSelectBox.selectedIndex = -1; + + // Clear parameters box. + clearSelectBox(document.getElementById("param_select")); +} + +// Called when the user selects a method. +sc.methodSelect = function() +{ + // Extract the name of the method we selected. + var methodSelectBox = document.getElementById("method_select"); + sc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; // At higher scope. + + var sqlstr = "SELECT DISTINCT methods.parameters, methods.sweep_id, " + + "metrics.libary_id, COUNT(DISTINCT metrics.libary_id) AS count FROM " + + "methods, metrics WHERE methods.name = '" + sc.methodName + + "' AND methods.id = metrics.method_id AND methods.sweep_id != -1 " + + "GROUP BY methods.parameters;"; + var params = dbExec(sqlstr); + console.log(JSON.stringify(params)); + + // Loop through results and fill the second list box. + var paramSelectBox = document.getElementById("param_select"); + clearSelectBox(paramSelectBox); + + var newOption = document.createElement("option"); + paramSelectBox.add(newOption); + + if ((dbType === "sqlite" && params[0]) || (dbType === "mysql" && params)) + { + // Put in the new options. + var length = dbType === "sqlite" ? params[0].values.length : params.length; + for (i = 0; i < length; i++) + { + var newOption = document.createElement("option"); + + var parameters = dbType === "sqlite" ? params[0].values[i][0] : params[i].parameters; + var sweepId = dbType === "sqlite" ? params[0].values[i][1] : params[i].sweep_id; + var libraries = dbType === "sqlite" ? params[0].values[i][3] : params[i].count; + + if (parameters) + { + newOption.text = parameters + " (" + libraries + " libraries)"; + } + else + { + newOption.text = "[no parameters] (" + libraries + " libraries)"; + } + newOption.id = sweepId; + + paramSelectBox.add(newOption); + } + } + + paramSelectBox.selectedIndex = 0; +} + +// Called when the user selects parameters. +// Called when a set of parameters is selected. Now we are ready to draw the +// chart. +sc.paramSelect = function() +{ + // The user has selected a library and parameters. Now we need to generate + // the list of datasets. + var methodSelectBox = document.getElementById("method_select"); + sc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; + var paramSelectBox = document.getElementById("param_select"); + var paramNameFull = paramSelectBox.options[paramSelectBox.selectedIndex].text; + var sweepId = paramSelectBox.options[paramSelectBox.selectedIndex].id; + + sc.paramName = paramNameFull.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. + + var sqlstr = "SELECT DISTINCT datasets.name FROM datasets, results, methods " + + "WHERE datasets.id = results.dataset_id AND methods.name = '" + + sc.methodName + "' AND methods.sweep_id = " + sweepId + " AND " + + "results.method_id = methods.id AND methods.parameters = '" + sc.paramName + + "';"; + var datasets = dbExec(sqlstr); + console.log(JSON.stringify(datasets)); + + // Loop through the results and fill the third list box. + var datasetSelectBox = document.getElementById("main_dataset_select"); + clearSelectBox(datasetSelectBox); + + var newOption = document.createElement("option"); + datasetSelectBox.add(newOption); + + if ((dbType === "sqlite" && datasets[0]) || (dbType === "mysql" && datasets)) + { + // Put in the datasets. + var length = dbType === "sqlite" ? datasets[0].values.length : datasets.length; + for (i = 0; i < length; i++) + { + newOption = document.createElement("option"); + + var datasetName = dbType === "sqlite" ? datasets[0].values[i][0] : datasets[i].name; + newOption.text = datasetName; + datasetSelectBox.add(newOption); + } + } + + datasetSelectBox.selectedIndex = 0; +} + +sc.datasetSelect = function() +{ + // The user selected a dataset, so now we can plot. + var methodSelectBox = document.getElementById("method_select"); + sc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; + var paramSelectBox = document.getElementById("param_select"); + var paramNameFull = paramSelectBox.options[paramSelectBox.selectedIndex].text; + sc.sweepId = paramSelectBox.options[paramSelectBox.selectedIndex].id; + var datasetSelectBox = document.getElementById("main_dataset_select"); + var datasetName = + datasetSelectBox.options[datasetSelectBox.selectedIndex].text; + + sc.paramName = paramNameFull.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); + + var sqlstr = "SELECT DISTINCT * FROM " + + "(SELECT results.time as time, results.var as var, " + + " results.sweep_elem_id as sweep_elem_id, libraries.name as lib," + + " max(results.build_id) as bid, datasets.instances as di, " + + " datasets.attributes as da, datasets.size as ds" + + " FROM results, datasets, methods, libraries" + + " WHERE results.dataset_id = datasets.id" + + " AND results.method_id = methods.id" + + " AND methods.name = '" + sc.methodName + "'" + + " AND methods.parameters = '" + sc.paramName + "'" + + " AND libraries.id = results.libary_id" + + " AND datasets.name = '" + datasetName + "'" + + " GROUP BY lib, sweep_elem_id)" + + "tmp GROUP BY sweep_elem_id, lib;"; + + sc.results = dbExec(sqlstr); + sc.results = dbType === "sqlite" ? sc.results[0].values : sc.results; + + // Obtain unique list of libraries. + sc.libraries = sc.results.map( + function(d) { + return dbType === "sqlite" ? d[3] : d.lib; + }).reduce( + function(p, c) { + if (p.indexOf(c) < 0) p.push(c); return p; + }, []); + + // By default, all libraries are active. + sc.activeLibraries = {}; + for (i = 0; i < sc.libraries.length; ++i) + { + sc.activeLibraries[sc.libraries[i]] = true; + } + + clearChart(); + buildChart(); +} + +sc.clearChart = function() +{ + d3.select("svg").remove(); + d3.selectAll(".d3-tip").remove(); + d3.selectAll(".library-select-title").remove(); + d3.selectAll(".library_select_div").remove(); + d3.selectAll(".legendholder").selectAll("*").remove(); +} + +sc.buildChart = function() +{ + sc.results = sc.results.map( + function(d) + { + var runtime = dbType === "sqlite" ? d[0] : d.time; + if (runtime == -2) + { + if (dbType === "sqlite") + d[0] = "failure"; + else + d.time = "failure"; + } + else if (runtime == -1) + { + if (dbType === "sqlite") + d[0] = ">9000"; + else + d.time = "failure"; + } + + return d; + }); + + // Get the parameter name we are sweeping. + var params = JSON.parse(sc.paramName); + var name = ""; + for (var i in params) + { + if (params[i].search(/sweep\(/) !== -1) + { + name = i; + break; + } + } + + // Get lists of active libraries. + var activeLibraryList = sc.libraries.map(function(d) { return d; }).reduce( + function(p, c) + { + if (sc.activeLibraries[c] == true) + p.push(c); + return p; + }, []); + + var maxRuntime = d3.max(sc.results, + function(d) + { + if (sc.activeLibraries[dbType === "sqlite" ? d[3] : d.lib] == false) + return 0; + else + return mapRuntime(dbType === "sqlite" ? d[0] : d.time, 0); + }); + // Increase so we have 16 spare pixels at the top. + maxRuntime *= ((height + 16) / height); + + var runtimeScale = d3.scale.linear() + .domain([0, maxRuntime]) + .range([height, 0]); + + // We need to find out how big the sweep is. + var sweepSql = "SELECT type, begin, step, end FROM sweeps where id = " + sc.sweepId; + var sweepResults = dbExec(sweepSql); + sweepResults = dbType === "sqlite" ? sweepResults[0].values : sweepResults.results; + + var func = (dbType === "sqlite" ? sweepResults[0][0] : sweepResults[0].type) === "int" ? parseInt : parseFloat; + var start = func(dbType === "sqlite" ? sweepResults[0][1] : sweepResults[0].start); + var step = func(dbType === "sqlite" ? sweepResults[0][2] : sweepResults[0].step); + var end = func(dbType === "sqlite" ? sweepResults[0][3] : sweepResults[0].end); + + var sweepScale = d3.scale.linear() + .domain([start, end]) + .range([0, width]); + + var xAxis = d3.svg.axis().scale(sweepScale).orient("bottom"); + var yAxis = d3.svg.axis().scale(runtimeScale).orient("left").tickFormat(d3.format(".2f")); + + // Create svg object. + var svg = d3.select(".svgholder").append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + // Add x axis. + svg.append("g").attr("id", "xaxis") + .attr("class", "x axis") + .attr("transform", "translate(0, " + height + ")") + .call(xAxis) + .append("text") + .style("text-anchor", "end") + .attr("dx", 500) + .attr("dy", "3em") + .text("Value of parameter '" + name + "'"); + + // Add y axis. + svg.append("g").attr("id", "yaxis") + .attr("class", "y axis") + .call(yAxis) + .append("text") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text("Runtime (s)"); + + // Create tooltips. + var tip = d3.tip() + .attr("class", "d3-tip") + .offset([-10, 0]) + .html(function(d) { + var runtime = d[0]; + if (d[0] != ">9000" && d[0] != "failure") { + runtime = d[0].toFixed(2); + } + return "" + d[3] + "; " + name + ": " + (start + step * d[2]) + ": " + runtime + "s"; }); + svg.call(tip); + + // Add all of the data points. + var lineFunc = d3.svg.line() + .x(function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) + .y(function(d) { return runtimeScale(mapRuntime( + dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .interpolate("linear"); + + var lineResults = [] + for (var l in sc.libraries) + { + if (sc.activeLibraries[sc.libraries[l]] == true) + { + lineResults.push(sc.results.map(function(d) { return d; }).reduce(function(p, c) { if(c[3] == sc.libraries[l]) { p.push(c); } return p; }, [])); + } + else + { + lineResults.push([]); + } + } + + for (i = 0; i < lineResults.length; ++i) + { + console.log(JSON.stringify(lineResults[i])); + if (lineFunc(lineResults[i]) != null) + { + svg.append('svg:path') + .attr('d', lineFunc(lineResults[i])) + .attr('stroke', color(sc.libraries[i])) + .attr('stroke-width', 2) + .attr('fill', 'none'); + } + } + + for (i = 0; i < lineResults.length; i++) + { + if (lineFunc(lineResults[i]) == null) + continue; + + // Colored circle enclosed in white circle enclosed in background color + // circle; looks kind of nice. + svg.selectAll("dot").data(lineResults[i]).enter().append("circle") + .attr("r", 6) + .attr("cx", function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) + .attr("cy", function(d) { return runtimeScale(mapRuntime(dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .attr('fill', '#222222') + .on('mouseover', tip.show) + .on('mouseout', tip.hide); + svg.selectAll("dot").data(lineResults[i]).enter().append("circle") + .attr("r", 4) + .attr("cx", function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) + .attr("cy", function(d) { return runtimeScale(mapRuntime(dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .attr('fill', '#ffffff') + .on('mouseover', tip.show) + .on('mouseout', tip.hide); + svg.selectAll("dot").data(lineResults[i]).enter().append("circle") + .attr("r", 3) + .attr("cx", function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) + .attr("cy", function(d) { return runtimeScale(mapRuntime(dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .attr('fill', function(d) { return color(d[4]) }) + .on('mouseover', tip.show) + .on('mouseout', tip.hide); + } + + // Create the library selector. + var librarySelectTitle = d3.select(".legendholder").append("div") + .attr("class", "library-select-title"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-text") + .text("Libraries:"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-open-paren") + .text("("); + librarySelectTitle.append("div") + .attr("class", "library-select-title-enable-all") + .text("enable all") + .on('click', function() { sc.enableAllLibraries(); }); + librarySelectTitle.append("div") + .attr("class", "library-select-title-bar") + .text("|"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-disable-all") + .text("disable all") + .on('click', function() { sc.disableAllLibraries(); }); + librarySelectTitle.append("div") + .attr("class", "library-select-title-close-paren") + .text(")"); + + var libraryDivs = d3.select(".legendholder").selectAll("input") + .data(sc.libraries) + .enter() + .append("div") + .attr("class", "library-select-div") + .attr("id", function(d) { return d + '-library-checkbox-div'; }); + + libraryDivs.append("label") + .attr('for', function(d) { return d + '-library-checkbox'; }) + .style('background', color) + .attr('class', 'library-select-color'); + + libraryDivs.append("input") + .property("checked", function(d) { return sc.activeLibraries[d]; }) + .attr("type", "checkbox") + .attr("id", function(d) { return d + '-library-checkbox'; }) + .attr('class', 'library-select-box') + .attr("onClick", function(d, i) { return "sc.toggleLibrary(\"" + d + "\");"; }); + + libraryDivs.append("label") + .attr('for', function(d) { return d + '-library-checkbox'; }) + .attr('class', 'library-select-label') + .text(function(d) { return d; }); +} + +// Toggle a library to on or off. +sc.toggleLibrary = function(library) +{ + sc.activeLibraries[library] = !sc.activeLibraries[library]; + + clearChart(); + buildChart(); +} + +// Set all libraries on. +sc.enableAllLibraries = function() +{ + for (v in sc.activeLibraries) { sc.activeLibraries[v] = true; } + + clearChart(); + buildChart(); +} + +// Set all libraries off. +sc.disableAllLibraries = function() +{ + for (v in sc.activeLibraries) { sc.activeLibraries[v] = false; } + + clearChart(); + buildChart(); +} diff --git a/reports/js/benchmarks/sweep-metric-comparison-view.js b/reports/js/benchmarks/sweep-metric-comparison-view.js new file mode 100644 index 0000000..6b16666 --- /dev/null +++ b/reports/js/benchmarks/sweep-metric-comparison-view.js @@ -0,0 +1,540 @@ +// Define namespace: smc = sweep metric comparison. +var smc = smc = smc || {}; + +smc.methodName = ""; +smc.paramName = ""; +smc.dataset = ""; +smc.libraries = []; +smc.activeLibraries = []; +smc.results = []; +smc.sweepId = -1; + +// This chart type has been selected. +smc.onTypeSelect = function() +{ + // The user needs to be able to select a method, then parameters, then a + // dataset. + var selectHolder = d3.select(".selectholder"); + selectHolder.append("label") + .attr("for", "method_select") + .attr("class", "method-select-label") + .text("Select method:"); + selectHolder.append("select") + .attr("id", "method_select") + .attr("onchange", "smc.methodSelect()"); + selectHolder.append("label") + .attr("for", "param_select") + .attr("class", "param-select-label") + .text("Select parameters:"); + selectHolder.append("select") + .attr("id", "param_select") + .attr("onchange", "smc.paramSelect()"); + selectHolder.append("label") + .attr("for", "main_dataset_select") + .attr("class", "main-dataset-select-label") + .text("Select dataset:"); + selectHolder.append("select") + .attr("id", "main_dataset_select") + .attr("onchange", "smc.datasetSelect()"); + selectHolder.append("label") + .attr("for", "metric_select") + .attr("class", "metric-select-label") + .text("Select metric:"); + selectHolder.append("select") + .attr("id", "metric_select") + .attr("onchange", "smc.metricSelect()"); + + smc.listMethods(); +} + +smc.clear = function() +{ + smc.clearChart(); +} + +// List the available methods where there is a sweep. +smc.listMethods = function() +{ + var methods = dbExec("SELECT DISTINCT methods.name FROM methods, results " + + "WHERE methods.id = results.method_id AND results.sweep_id != -1 " + + "ORDER BY name;"); + var methodSelectBox = document.getElementById("method_select"); + + // Remove old things. + clearSelectBox(methodSelectBox); + + // Add new things. + var length = dbType === "sqlite" ? methods[0].values.length : methods.length; + for (i = 0; i < length; i++) + { + var newOption = document.createElement("option"); + newOption.text = dbType === "sqlite" ? methods[0].values[i] : + methods[i].name; + methodSelectBox.add(newOption); + } + methodSelectBox.selectedIndex = -1; + + // Clear parameters box. + clearSelectBox(document.getElementById("param_select")); +} + +// Called when the user selects a method. +smc.methodSelect = function() +{ + // Extract the name of the method we selected. + var methodSelectBox = document.getElementById("method_select"); + smc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; // At higher scope. + + var sqlstr = "SELECT DISTINCT methods.parameters, metrics.sweep_id, " + + "metrics.libary_id, COUNT(DISTINCT metrics.libary_id) AS count FROM " + + "methods, metrics WHERE methods.name = '" + smc.methodName + + "' AND methods.id = metrics.method_id AND metrics.sweep_id != -1 " + + "GROUP BY methods.parameters;"; + var params = dbExec(sqlstr); + + // Loop through results and fill the second list box. + var paramSelectBox = document.getElementById("param_select"); + clearSelectBox(paramSelectBox); + + var newOption = document.createElement("option"); + paramSelectBox.add(newOption); + + if ((dbType === "sqlite" && params[0]) || (dbType === "mysql" && params)) + { + // Put in the new options. + var length = dbType === "sqlite" ? params[0].values.length : params.length; + for (i = 0; i < length; i++) + { + var newOption = document.createElement("option"); + + var parameters = dbType === "sqlite" ? params[0].values[i][0] : params[i].parameters; + var sweepId = dbType === "sqlite" ? params[0].values[i][1] : params[i].sweep_id; + var libraries = dbType === "sqlite" ? params[0].values[i][3] : params[i].count; + + if (parameters) + { + newOption.text = parameters + " (" + libraries + " libraries)"; + } + else + { + newOption.text = "[no parameters] (" + libraries + " libraries)"; + } + newOption.id = sweepId; + + paramSelectBox.add(newOption); + } + } + + paramSelectBox.selectedIndex = 0; +} + +// Called when the user selects parameters. +// Called when a set of parameters is selected. Now we are ready to draw the +// chart. +smc.paramSelect = function() +{ + // The user has selected a library and parameters. Now we need to generate + // the list of datasets. + var methodSelectBox = document.getElementById("method_select"); + smc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; + var paramSelectBox = document.getElementById("param_select"); + var paramNameFull = paramSelectBox.options[paramSelectBox.selectedIndex].text; + var sweepId = paramSelectBox.options[paramSelectBox.selectedIndex].id; + + smc.paramName = paramNameFull.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. + + var sqlstr = "SELECT DISTINCT datasets.name FROM datasets, results, methods " + + "WHERE datasets.id = results.dataset_id AND methods.name = '" + + smc.methodName + "' AND results.sweep_id = " + sweepId + " AND " + + "results.method_id = methods.id AND methods.parameters = '" + smc.paramName + + "';"; + var datasets = dbExec(sqlstr); + + // Loop through the results and fill the third list box. + var datasetSelectBox = document.getElementById("main_dataset_select"); + clearSelectBox(datasetSelectBox); + + var newOption = document.createElement("option"); + datasetSelectBox.add(newOption); + + if ((dbType === "sqlite" && datasets[0]) || (dbType === "mysql" && datasets)) + { + // Put in the datasets. + var length = dbType === "sqlite" ? datasets[0].values.length : datasets.length; + for (i = 0; i < length; i++) + { + newOption = document.createElement("option"); + + var datasetName = dbType === "sqlite" ? datasets[0].values[i][0] : datasets[i].name; + newOption.text = datasetName; + datasetSelectBox.add(newOption); + } + } + + datasetSelectBox.selectedIndex = 0; +} + +smc.datasetSelect = function() +{ + // The user selected a dataset, so now we can plot. + var methodSelectBox = document.getElementById("method_select"); + smc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; + var paramSelectBox = document.getElementById("param_select"); + var paramNameFull = paramSelectBox.options[paramSelectBox.selectedIndex].text; + smc.sweepId = paramSelectBox.options[paramSelectBox.selectedIndex].id; + var datasetSelectBox = document.getElementById("main_dataset_select"); + var datasetName = + datasetSelectBox.options[datasetSelectBox.selectedIndex].text; + + smc.paramName = paramNameFull.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); + + // What metrics do we have available? We can do our actual query for results + // here, then we need to parse it. + var sqlstr = "SELECT DISTINCT * FROM " + + "(SELECT metrics.metric as metric, " + + " metrics.sweep_elem_id as sweep_elem_id, libraries.name as lib," + + " max(metrics.build_id) as bid, datasets.instances as di, " + + " datasets.attributes as da, datasets.size as ds" + + " FROM metrics, datasets, methods, libraries" + + " WHERE metrics.dataset_id = datasets.id" + + " AND metrics.method_id = methods.id" + + " AND methods.name = '" + smc.methodName + "'" + + " AND methods.parameters = '" + smc.paramName + "'" + + " AND libraries.id = metrics.libary_id" + + " AND datasets.name = '" + datasetName + "'" + + " GROUP BY lib, metrics.sweep_elem_id)" + + "tmp GROUP BY sweep_elem_id, lib;"; + smc.results = dbExec(sqlstr); + smc.results = dbType === "sqlite" ? smc.results[0].values : smc.results; + + // Now we have to parse through the metrics and see what we find. + addMetric = function(p, c) + { + var json = jQuery.parseJSON(dbType === "sqlite" ? c[0] : c.metric); + for(var k in json) + if(p.indexOf(k) < 0) + p.push(k); + return p; + }; + metrics = smc.results.reduce(addMetric, []); + + var metric_select_box = document.getElementById("metric_select"); + clearSelectBox(metric_select_box); + for (i = 0; i < metrics.length; i++) + { + var new_option = document.createElement("option"); + new_option.text = metrics[i]; + metric_select_box.add(new_option); + } + metric_select_box.selectedIndex = -1; +} + +smc.metricSelect = function() +{ + // We've already got the results, and now the user has specified the metric + // they want plotted. + var metricSelectBox = document.getElementById("metric_select"); + smc.metricName = metricSelectBox.options[metricSelectBox.selectedIndex].text; + + // Obtain unique list of libraries. + smc.libraries = smc.results.map( + function(d) { + return dbType === "sqlite" ? d[2] : d.lib; + }).reduce( + function(p, c) { + if (p.indexOf(c) < 0) p.push(c); return p; + }, []); + + // By default, all libraries are active. + smc.activeLibraries = {}; + for (i = 0; i < smc.libraries.length; ++i) + { + smc.activeLibraries[smc.libraries[i]] = true; + } + + clearChart(); + buildChart(); +} + +smc.clearChart = function() +{ + d3.select("svg").remove(); + d3.selectAll(".d3-tip").remove(); + d3.selectAll(".library-select-title").remove(); + d3.selectAll(".library_select_div").remove(); + d3.selectAll(".legendholder").selectAll("*").remove(); +} + +smc.extractRuntime = function(d) +{ + var json = jQuery.parseJSON(d); + for (var m in json) + { + if (m == "Runtime") + { + if (json[m] == -2) + return "failure"; + else if (json[m] == -1) + return ">9000"; + else + return json[m]; + } + } + + return "failure"; +} + +smc.extractMetric = function(d, metricName, notFoundValue) +{ + var json = jQuery.parseJSON(d); + for (var m in json) + if (m == metricName) + return json[m]; + + return notFoundValue; +} + +smc.buildChart = function() +{ + // Get the parameter name we are sweeping. + var params = JSON.parse(smc.paramName); + var name = ""; + for (var i in params) + { + if (params[i].search(/sweep\(/) !== -1) + { + name = i; + break; + } + } + + // Get lists of active libraries. + var activeLibraryList = smc.libraries.map(function(d) { return d; }).reduce( + function(p, c) + { + if (smc.activeLibraries[c] == true) + p.push(c); + return p; + }, []); + + var maxRuntime = d3.max(smc.results, + function(d) + { + if (smc.activeLibraries[dbType === "sqlite" ? d[2] : d.lib] == false) + return 0; + else + return smc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric); + }); + + var maxMetric = d3.max(smc.results, + function(d) + { + if (smc.activeLibraries[dbType === "sqlite" ? d[2] : d.lib] == false) + return 0; + else + return smc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, + smc.metricName, 0); + }); + + // Increase so we have 16 spare pixels at the top. + maxMetric *= ((height + 16) / height); + + var runtimeScale = d3.scale.linear() + .domain([0, maxRuntime]) + .range([0, width]); + + // We need to find out how big the sweep is. + var sweepSql = "SELECT type, begin, step, end FROM sweeps where id = " + smc.sweepId; + var sweepResults = dbExec(sweepSql); + sweepResults = dbType === "sqlite" ? sweepResults[0].values : sweepResults.results; + + var func = (dbType === "sqlite" ? sweepResults[0][0] : sweepResults[0].type) === "int" ? parseInt : parseFloat; + var start = func(dbType === "sqlite" ? sweepResults[0][1] : sweepResults[0].start); + var step = func(dbType === "sqlite" ? sweepResults[0][2] : sweepResults[0].step); + var end = func(dbType === "sqlite" ? sweepResults[0][3] : sweepResults[0].end); + + var metricScale = d3.scale.linear() + .domain([0, maxMetric]) + .range([height, 0]); + + var xAxis = d3.svg.axis().scale(runtimeScale).orient("bottom"); + var yAxis = d3.svg.axis().scale(metricScale).orient("left"); + + // Create svg object. + var svg = d3.select(".svgholder").append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + // Add x axis. + svg.append("g").attr("id", "xaxis") + .attr("class", "x axis") + .attr("transform", "translate(0, " + height + ")") + .call(xAxis) + .append("text") + .style("text-anchor", "end") + .attr("dx", 500) + .attr("dy", "3em") + .text("Runtime (s)"); + + // Add y axis. + svg.append("g").attr("id", "yaxis") + .attr("class", "y axis") + .call(yAxis) + .append("text") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text(smc.metricName); + + // Create tooltips. + var tip = d3.tip() + .attr("class", "d3-tip") + .offset([-10, 0]) + .html(function(d) { + var runtime = smc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric); + if (runtime != ">9000" && runtime != "failure") { + runtime = runtime.toFixed(2); + } + var metricValue = smc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, smc.metricName, ""); + return "" + d[2] + "; " + name + ":
" + (start + step * d[1]) + ": " + smc.metricName + " " + metricValue + ", " + runtime + "s"; }); + svg.call(tip); + + // Add all of the data points. + var lineFunc = d3.svg.line() + .x(function(d) { return runtimeScale(mapRuntime(smc.extractRuntime( + dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .y(function(d) { return metricScale(smc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, smc.metricName, 0)); }) + .interpolate("linear"); + + var lineResults = [] + for (var l in smc.libraries) + { + if (smc.activeLibraries[smc.libraries[l]] == true) + { + lineResults.push(smc.results.map(function(d) { return d; }).reduce(function(p, c) { if(c[2] == smc.libraries[l]) { p.push(c); } return p; }, [])); + } + else + { + lineResults.push([]); + } + } + for (i = 0; i < lineResults.length; ++i) + { + if (lineFunc(lineResults[i]) != null) + { + svg.append('svg:path') + .attr('d', lineFunc(lineResults[i])) + .attr('stroke', color(smc.libraries[i])) + .attr('stroke-width', 2) + .attr('fill', 'none'); + } + } + + for (i = 0; i < lineResults.length; i++) + { + if (lineFunc(lineResults[i]) == null) + continue; + + // Colored circle enclosed in white circle enclosed in background color + // circle; looks kind of nice. + svg.selectAll("dot").data(lineResults[i]).enter().append("circle") + .attr("r", 6) + .attr("cx", function(d) { return runtimeScale(mapRuntime(smc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .attr("cy", function(d) { return metricScale(smc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, smc.metricName, 0)); }) + .attr('fill', '#222222') + .on('mouseover', tip.show) + .on('mouseout', tip.hide); + svg.selectAll("dot").data(lineResults[i]).enter().append("circle") + .attr("r", 4) + .attr("cx", function(d) { return runtimeScale(mapRuntime(smc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .attr("cy", function(d) { return metricScale(smc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, smc.metricName, 0)); }) + .attr('fill', '#ffffff') + .on('mouseover', tip.show) + .on('mouseout', tip.hide); + svg.selectAll("dot").data(lineResults[i]).enter().append("circle") + .attr("r", 3) + .attr("cx", function(d) { return runtimeScale(mapRuntime(smc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .attr("cy", function(d) { return metricScale(smc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, smc.metricName, 0)); }) + .attr('fill', function(d) { return color(d[4]) }) + .on('mouseover', tip.show) + .on('mouseout', tip.hide); + } + + // Create the library selector. + var librarySelectTitle = d3.select(".legendholder").append("div") + .attr("class", "library-select-title"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-text") + .text("Libraries:"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-open-paren") + .text("("); + librarySelectTitle.append("div") + .attr("class", "library-select-title-enable-all") + .text("enable all") + .on('click', function() { smc.enableAllLibraries(); }); + librarySelectTitle.append("div") + .attr("class", "library-select-title-bar") + .text("|"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-disable-all") + .text("disable all") + .on('click', function() { smc.disableAllLibraries(); }); + librarySelectTitle.append("div") + .attr("class", "library-select-title-close-paren") + .text(")"); + + var libraryDivs = d3.select(".legendholder").selectAll("input") + .data(smc.libraries) + .enter() + .append("div") + .attr("class", "library-select-div") + .attr("id", function(d) { return d + '-library-checkbox-div'; }); + + libraryDivs.append("label") + .attr('for', function(d) { return d + '-library-checkbox'; }) + .style('background', color) + .attr('class', 'library-select-color'); + + libraryDivs.append("input") + .property("checked", function(d) { return smc.activeLibraries[d]; }) + .attr("type", "checkbox") + .attr("id", function(d) { return d + '-library-checkbox'; }) + .attr('class', 'library-select-box') + .attr("onClick", function(d, i) { return "smc.toggleLibrary(\"" + d + "\");"; }); + + libraryDivs.append("label") + .attr('for', function(d) { return d + '-library-checkbox'; }) + .attr('class', 'library-select-label') + .text(function(d) { return d; }); +} + +// Toggle a library to on or off. +smc.toggleLibrary = function(library) +{ + smc.activeLibraries[library] = !smc.activeLibraries[library]; + + clearChart(); + buildChart(); +} + +// Set all libraries on. +smc.enableAllLibraries = function() +{ + for (v in smc.activeLibraries) { smc.activeLibraries[v] = true; } + + clearChart(); + buildChart(); +} + +// Set all libraries off. +smc.disableAllLibraries = function() +{ + for (v in smc.activeLibraries) { smc.activeLibraries[v] = false; } + + clearChart(); + buildChart(); +}