From efe8508b1af7215d0357fc3ea397a047752cc1a5 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 19 Jul 2017 11:08:16 -0400 Subject: [PATCH 1/9] Better displays when we have a sweep. --- reports/css/d3-style.css | 5 +- .../benchmarks/historical-comparison-view.js | 2 +- ...tric-multiple-parameter-comparison-view.js | 33 ++-- .../metric-parameter-comparison-view.js | 2 +- .../js/benchmarks/runtime-comparison-view.js | 162 ++++++++++++++++- .../js/benchmarks/sweep-comparison-view.js | 164 ++++++++++++++++++ 6 files changed, 342 insertions(+), 26 deletions(-) create mode 100644 reports/js/benchmarks/sweep-comparison-view.js diff --git a/reports/css/d3-style.css b/reports/css/d3-style.css index 33b1552..ce955a9 100644 --- a/reports/css/d3-style.css +++ b/reports/css/d3-style.css @@ -156,14 +156,15 @@ svg text { padding: 10px; } -.method-select-label, .param-select-label, .main-dataset-select-label { +.method-select-label, .param-select-label, .main-dataset-select-label, +.sweep-select-label { float: left; font-size: 75%; color: #ffffff; margin-right: 8px; } -#method_select, #param_select, #main_dataset_select, +#method_select, #param_select, #sweep_select, #main_dataset_select, #metric_select, #option_select { float: left; margin-right: 20px; diff --git a/reports/js/benchmarks/historical-comparison-view.js b/reports/js/benchmarks/historical-comparison-view.js index c38e74c..ce03534 100644 --- a/reports/js/benchmarks/historical-comparison-view.js +++ b/reports/js/benchmarks/historical-comparison-view.js @@ -100,7 +100,7 @@ hc.paramSelect = function() var param_select_box = document.getElementById("param_select"); var param_name_full = param_select_box.options[param_select_box.selectedIndex].text; // Parse out actual parameters. - hc.param_name = param_name_full.split("(")[0].replace(/^\s+|\s+$/g, ''); // At higher scope. + hc.param_name = param_name_full.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. if (hc.param_name == "[no parameters]") { hc.param_name = ""; } // Given a method name and parameters, query the SQLite database for all of diff --git a/reports/js/benchmarks/metric-multiple-parameter-comparison-view.js b/reports/js/benchmarks/metric-multiple-parameter-comparison-view.js index 52cf937..1601d2d 100644 --- a/reports/js/benchmarks/metric-multiple-parameter-comparison-view.js +++ b/reports/js/benchmarks/metric-multiple-parameter-comparison-view.js @@ -97,11 +97,13 @@ mmpc.listOptions = function() "AND methods.name = '" + mmpc.method_name + "' " + "AND datasets.name = '" + mmpc.dataset_name + "';"; var results = dbExec(sqlstr); + console.log(JSON.stringify(results)); results = dbType === "sqlite" ? results[0].values : results; var addOption = function(p, c) { var options = mmpc.getOptionList(dbType === "sqlite" ? c.toString() : c.parameter); + console.log(JSON.stringify(options)); for (i = 0; i < options.length; i++) if(p.indexOf(options[i]) < 0) p.push(options[i]); return p; @@ -131,6 +133,7 @@ mmpc.listMetrics = function() "AND datasets.name = '" + mmpc.dataset_name + "';"; var results = dbExec(sqlstr); results = dbType === "sqlite" ? results[0].values : results; + console.log(results); addMetric = function(p, c) { var json = jQuery.parseJSON(dbType === "sqlite" ? c : c.metric); @@ -198,9 +201,10 @@ mmpc.metricSelect = function() "AND methods.name = '" + mmpc.method_name + "' " + "AND datasets.name = '" + mmpc.dataset_name + "';"; mmpc.results = dbExec(sqlstr); + console.log(sqlstr); mmpc.results = dbType === "sqlite" ? mmpc.results[0].values : mmpc.results; - console.log(mmpc.results) + console.log(JSON.stringify(mmpc.results)); var filterAndSet = function(p, d) { var metrics = jQuery.parseJSON(dbType === "sqlite" ? d[0] : d.metric); @@ -358,13 +362,15 @@ mmpc.clearChart = function() // Return a param's value from a list of parameters. mmpc.getOptionValue = function(str, opt) { - var list = str.split("-" + opt) - if (list.length < 2) - return ""; - list = list[1].split(/[\s=]+/); - if (list.length < 2) - return ""; - return list[1]; + var options = JSON.parse(str); + console.log("getOptionValue: " + str); + if (opt in options) + { + // Then we know the value. + return options[opt]; + } + + return ""; } // Remove a parameter from the list of parameters. @@ -376,11 +382,12 @@ mmpc.removeOption = function(str, opt) // Returns a list of parameters. mmpc.getOptionList = function(str) { - optList = str.split(/-+/) - .map(function (d) {return d.replace(/^\s+|\s+$/g, '').split(/[\s=]+/); }) - .filter(function (d) {return (d.length >= 1);}) - .map(function (d) {return d[0];}); - optList.shift(); + var opts = JSON.parse(str); + var optList = []; + for (var k in opts) + { + optList.push(k); + } return optList; } diff --git a/reports/js/benchmarks/metric-parameter-comparison-view.js b/reports/js/benchmarks/metric-parameter-comparison-view.js index e8f3e6b..03f685c 100644 --- a/reports/js/benchmarks/metric-parameter-comparison-view.js +++ b/reports/js/benchmarks/metric-parameter-comparison-view.js @@ -113,7 +113,7 @@ mpc.paramSelect = function() var param_select_box = document.getElementById("param_select"); var param_name_full = param_select_box.options[param_select_box.selectedIndex].text; // Parse out actual parameters. - mpc.param_name = param_name_full.split("(")[0].replace(/^\s+|\s+$/g, ''); // At higher scope. + mpc.param_name = param_name_full.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. if (mpc.param_name == "[no parameters]") { mpc.param_name = ""; } if (mpc.param_name == "-") diff --git a/reports/js/benchmarks/runtime-comparison-view.js b/reports/js/benchmarks/runtime-comparison-view.js index f782756..0de52ae 100644 --- a/reports/js/benchmarks/runtime-comparison-view.js +++ b/reports/js/benchmarks/runtime-comparison-view.js @@ -19,22 +19,22 @@ rc.onTypeSelect = function() .attr("for", "method_select") .attr("class", "method-select-label") .text("Select method:"); - selectHolder.append("select") + selectHolder.append("select") .attr("id", "method_select") .attr("onchange", "rc.methodSelect()"); - selectHolder.append("label") + selectHolder.append("label") .attr("for", "param_select") .attr("class", "param-select-label") .text("Select parameters:"); - selectHolder.append("select") + selectHolder.append("select") .attr("id", "param_select") .attr("onchange", "rc.paramSelect()"); - selectHolder.append("br"); - selectHolder.append("label") + selectHolder.append("br"); + selectHolder.append("label") .attr("for", "main_dataset_select") .attr("class", "main-dataset-select-label") .text("Sort results by:"); - selectHolder.append("select") + selectHolder.append("select") .attr("id", "main_dataset_select") .attr("onchange", "rc.orderSelect()"); @@ -94,8 +94,9 @@ rc.methodSelect = function() var method_select_box = document.getElementById("method_select"); rc.method_name = method_select_box.options[method_select_box.selectedIndex].text; // At higher scope. - var sqlstr = "SELECT DISTINCT methods.parameters, metrics.libary_id, COUNT(DISTINCT metrics.libary_id) AS count FROM methods, metrics WHERE methods.name = '" + rc.method_name + "' AND methods.id = metrics.method_id GROUP BY methods.parameters;"; + var sqlstr = "SELECT DISTINCT methods.parameters, methods.sweep_id, metrics.libary_id, COUNT(DISTINCT metrics.libary_id) AS count FROM methods, metrics WHERE methods.name = '" + rc.method_name + "' AND methods.id = metrics.method_id GROUP BY methods.parameters;"; var params = dbExec(sqlstr); + console.log(JSON.stringify(params)); // Loop through results and fill the second list box. var param_select_box = document.getElementById("param_select"); @@ -113,7 +114,8 @@ rc.methodSelect = function() var new_option = document.createElement("option"); var parameters = dbType === "sqlite" ? params[0].values[i][0] : params[i].parameters; - var libraries = dbType === "sqlite" ? params[0].values[i][2] : params[i].count; + var sweepId = dbType === "sqlite" ? params[0].values[i][1] : params[i].sweep_id; + var libraries = dbType === "sqlite" ? params[0].values[i][3] : params[i].count; if (parameters) { @@ -123,6 +125,13 @@ rc.methodSelect = function() { new_option.text = "[no parameters] (" + libraries + " libraries)"; } + + if (sweepId !== -1) + { + new_option.text += " [sweep]"; + } + new_option.id = sweepId; + param_select_box.add(new_option); } } @@ -139,14 +148,80 @@ rc.paramSelect = function() rc.method_name = method_select_box.options[method_select_box.selectedIndex].text; var param_select_box = document.getElementById("param_select"); var param_name_full = param_select_box.options[param_select_box.selectedIndex].text; + var sweep_id = param_select_box.options[param_select_box.selectedIndex].id; + + // Remove the sweep box. + var ss = document.getElementById("sweep_select") + if (ss !== null) + { + ss.parentNode.removeChild(ss); + } + ss = document.getElementById("sweep_select_label") + if (ss !== null) + { + ss.parentNode.removeChild(ss); + } // Parse out actual parameters. - rc.param_name = param_name_full.split("(")[0].replace(/^\s+|\s+$/g, ''); // At higher scope. + rc.param_name = param_name_full.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. if (rc.param_name == "[no parameters]") { rc.param_name = ""; } + // If the user selected a sweep, we can't actually plot anything yet. + if (sweep_id !== -1) + { + // Instead we have to create an extra selector to know what element of the + // sweep they want to look at. + var selectHolder = d3.select(".selectholder"); + selectHolder.insert("label", "#param_select + *") + .attr("for", "sweep_select") + .attr("class", "sweep-select-label") + .attr("id", "sweep_select_label") + .text("Select sweep element:"); + selectHolder.insert("select", "#sweep_select_label + *") + .attr("id", "sweep_select") + .attr("onchange", "rc.sweepSelect()"); + + // Now populate the sweep elements. + var sweepsql = "SELECT type, begin, step, end FROM sweeps where id = " + sweep_id; + rc.results = dbExec(sweepsql); + console.log(sweepsql); + rc.results = dbType === "sqlite" ? rc.results[0].values : rc.results; + + // Get the parameter name we are sweeping. + var params = JSON.parse(rc.param_name); + var name = ""; + for (var i in params) + { + if (params[i].search(/sweep\(/) !== -1) + { + name = i; + break; + } + } + + // sweep_info: [start, step, end] + var sweepSelectBox = document.getElementById("sweep_select"); + sweepSelectBox.add(document.createElement("option")); + var func = (dbType === "sqlite" ? rc.results[0][0] : rc.results[0].type) === "int" ? parseInt : parseFloat; + var start = func(dbType === "sqlite" ? rc.results[0][1] : rc.results[0].start); + var step = func(dbType === "sqlite" ? rc.results[0][2] : rc.results[0].step); + var end = func(dbType === "sqlite" ? rc.results[0][3] : rc.results[0].end); + var elem = 0; + for (var i = start; i < end; i += step, elem++) + { + var new_option = document.createElement("option"); + new_option.text = name + ": " + i; + new_option.id = elem; + sweepSelectBox.add(new_option); + } + sweepSelectBox.selectedIndex = 0; + + return; + } + // Extract the name of the method we selected. var order_select_box = document.getElementById("main_dataset_select"); rc.groupBy = "datasets." + order_select_box.options[order_select_box.selectedIndex].text; // At higher scope. @@ -169,7 +244,76 @@ rc.paramSelect = function() "FROM results, datasets, methods, libraries WHERE " + "results.dataset_id = datasets.id AND results.method_id = methods.id AND methods.name = '" + rc.method_name + "' AND methods.parameters = '" + rc.param_name + "' AND libraries.id = results.libary_id ORDER BY bid DESC " + ") tmp GROUP BY lid, " + rc.groupBy + ", did;"; + console.log(sqlstr); + rc.results = dbExec(sqlstr); + console.log(JSON.stringify(rc.results)); + rc.results = dbType === "sqlite" ? rc.results[0].values : rc.results; + + // Obtain unique list of datasets. + rc.datasets = rc.results.map(function(d) { return dbType === "sqlite" ? d[4] : d.dataset; }).reduce(function(p, c) { if(p.indexOf(c) < 0) p.push(c); return p; }, []); + // Obtain unique list of libraries. + rc.libraries = rc.results.map(function(d) { return dbType === "sqlite" ? d[3] : d.lib; }).reduce(function(p, c) { if(p.indexOf(c) < 0) p.push(c); return p; }, []); + + // By default, everything is active. + rc.active_datasets = {}; + for (i = 0; i < rc.datasets.length; i++) + { + rc.active_datasets[rc.datasets[i]] = true; + } + + rc.active_libraries = {}; + for (i = 0; i < rc.libraries.length; i++) + { + rc.active_libraries[rc.libraries[i]] = true; + } + + clearChart(); + buildChart(); +} + +rc.sweepSelect = function() +{ + // Now we have selected a sweep ID and we can plot it. + var method_select_box = document.getElementById("method_select"); + rc.method_name = method_select_box.options[method_select_box.selectedIndex].text; + var param_select_box = document.getElementById("param_select"); + var param_name_full = param_select_box.options[param_select_box.selectedIndex].text; + var sweep_id = param_select_box.options[param_select_box.selectedIndex].id; + var sweep_select_box = document.getElementById("sweep_select"); + var sweep_element_id = sweep_select_box.options[sweep_select_box.selectedIndex].id; + + // Parse out actual parameters. + rc.param_name = param_name_full.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. + if (rc.param_name == "[no parameters]") + { + rc.param_name = ""; + } + + // Extract the name of the method we selected. + var order_select_box = document.getElementById("main_dataset_select"); + rc.groupBy = "datasets." + order_select_box.options[order_select_box.selectedIndex].text; // At higher scope. + + if (rc.groupBy == "datasets.instances") + { + rc.groupBy = "di"; + } + else if (rc.groupBy == "datasets.attributes") + { + rc.groupBy = "da"; + } + else + { + rc.groupBy = "ds"; + } + + var sqlstr = "SELECT DISTINCT * FROM " + + "(SELECT results.time as time, results.var as var, libraries.id, libraries.name as lib, datasets.name as dataset, datasets.id as did, libraries.id as lid, results.build_id as bid, datasets.instances as di, datasets.attributes as da, datasets.size as ds " + + "FROM results, datasets, methods, libraries WHERE " + + "results.dataset_id = datasets.id AND results.method_id = methods.id AND methods.name = '" + rc.method_name + "' AND methods.parameters = '" + rc.param_name + "' AND libraries.id = results.libary_id AND results.sweep_elem_id = " + sweep_element_id + " ORDER BY bid DESC " + + ") tmp GROUP BY lid, " + rc.groupBy + ", did;"; + console.log(sqlstr); rc.results = dbExec(sqlstr); + console.log(JSON.stringify(rc.results)); rc.results = dbType === "sqlite" ? rc.results[0].values : rc.results; // Obtain unique list of datasets. diff --git a/reports/js/benchmarks/sweep-comparison-view.js b/reports/js/benchmarks/sweep-comparison-view.js new file mode 100644 index 0000000..65fc641 --- /dev/null +++ b/reports/js/benchmarks/sweep-comparison-view.js @@ -0,0 +1,164 @@ +// Define namespace: sc = sweep comparison. +var sc = sc = sc || {}; + +sc.method_name = ""; +sc.param_name = ""; +sc.dataset = ""; +sc.libraries = []; +sc.active_libraries = []; +sc.results = []; + +// This chart type has been selected. +sc.onTypeSelect = function() +{ + // The user needs to be able to select a method, then parameters, then a + // dataset. + var selectHolder = d3.select(".selectholder"); + selectHolder.append("label") + .attr("for", "method_select") + .attr("class", "method-select-label") + .text("Select method:"); + selectHolder.append("select") + .attr("id", "method_select") + .attr("onchange", "sc.methodSelect()"); + selectHolder.append("label") + .attr("for", "param_select") + .attr("class", "param-select-label") + .text("Select parameters:"); + selectHolder.append("select") + .attr("id", "param_select") + .attr("onchange", "sc.paramSelect()"); + selectHolder.append("label") + .attr("for", "main_dataset_select") + .attr("class", "main-dataset-select-label") + .text("Select dataset:"); + selectHolder.append("select") + .attr("id", "main_dataset_select") + .attr("onchange", "sc.datasetSelect()"); + + sc.listMethods(); +} + +// List the available methods where there is a sweep. +sc.listMethods = function() +{ + var methods = dbExec("SELECT DISTINCT methods.name FROM methods, results " + + "WHERE methods.id = results.method_id AND methods.sweep_id != -1 " + + "ORDER BY name;"); + var methodSelectBox = document.getElementById("method_select"); + + // Remove old things. + clearSelectBox(methodSelectBox); + + // Add new things. + var length = dbType === "sqlite" ? methods[0].values.length : methods.length; + for (i = 0; i < length; i++) + { + var newOption = document.createElement("option"); + newOption.text = dbType === "sqlite" ? methods[0].values[i] : + methods[i].name; + methodSelectBox.add(newOption); + } + methodSelectBox.selectedIndex = -1; + + // Clear parameters box. + clearSelectBox(document.getElementById("param_select")); +} + +// Called when the user selects a method. +sc.methodSelect = function() +{ + // Extract the name of the method we selected. + var methodSelectBox = document.getElementById("method_select"); + sc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; // At higher scope. + + var sqlstr = "SELECT DISTINCT methods.parameters, methods.sweep_id, " + + "metrics.libary_id, COUNT(DISTINCT metrics.libary_id) AS count FROM " + + "methods, metrics WHERE methods.name = '" + sc.method_name + + "' AND methods.id = metrics.method_id AND methods.sweep_id != -1 " + + "GROUP BY methods.parameters;"; + var params = dbExec(sqlstr); + console.log(JSON.stringify(params)); + + // Loop through results and fill the second list box. + var paramSelectBox = document.getElementById("param_select"); + clearSelectBox(paramSelectBox); + + var newOption = document.createElement("option"); + param_select_box.add(newOption); + + if ((dbType === "sqlite" && params[0]) || (dbType === "mysql" && params)) + { + // Put in the new options. + var length = dbType === "sqlite" ? params[0].values.length : params.length; + for (i = 0; i < length; i++) + { + var newOption = document.createElement("option"); + + var parameters = dbType === "sqlite" ? params[0].values[i][0] : params[i].parameters; + var sweepId = dbType === "sqlite" ? params[0].values[i][1] : params[i].sweep_id; + var libraries = dbType === "sqlite" ? params[0].values[i][3] : params[i].count; + + if (parameters) + { + newOption.text = parameters + " (" + libraries + " libraries)"; + } + else + { + newOption.text = "[no parameters] (" + libraries + " libraries)"; + } + newOption.id = sweepId; + + paramSelectBox.add(newOption); + } + } + + paramSelectBox.selectedIndex = 0; +} + +// Called when the user selects parameters. +// Called when a set of parameters is selected. Now we are ready to draw the +// chart. +rc.paramSelect = function() +{ + // The user has selected a library and parameters. Now we need to generate + // the list of datasets. + var methodSelectBox = document.getElementById("method_select"); + rc.method_name = methodSelectBox.options[methodSelectBox.selectedIndex].text; + var paramSelectBox = document.getElementById("param_select"); + var paramNameFull = paramSelectBox.options[paramSelectBox.selectedIndex].text; + var sweepId = paramSelectBox.options[paramSelectBox.selectedIndex].id; + + sc.param_name = param_name_full.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. + + var sqlstr = "SELECT DISTINCT datasets.name FROM datasets, results, methods " + + "WHERE datasets.id = results.dataset_id AND methods.name = '" + + sc.method_name + "' AND methods.sweep_id = " + sweepId + " AND " + + "results.method_id = methods.id AND methods.parameters = " + sc.paramName + + ";"; + var datasets = dbExec(sqlstr); + console.log(JSON.stringify(params)); + + // Loop through the results and fill the third list box. + var datasetSelectBox = document.getElementById("dataset_select"); + clearSelectBox(datasetSelectBox); + + var newOption = document.createElement("option"); + datasetSelectBox.add(newOption); + + if ((dbType === "sqlite" && datasets[0]) || (dbType === "mysql" && datasets)) + { + // Put in the datasets. + var length = dbType === "sqlite" ? datasets[0].values.length : datasets.length; + for (i = 0; i < length; i++) + { + newOption = document.createElement("option"); + + var datasetName = dbType === "sqlite" ? datasets[0].values[i][0] : datasets[i].name; + newOption.text = datasetName; + datasetSelectBox.add(newOption); + } + } + + datasetSelectBox.selectedIndex = 0; +} From 4e3541f43be701c602a08a0cbc6e2fb6697e9949 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 16 Aug 2017 17:35:02 -0400 Subject: [PATCH 2/9] Add sweep comparison view. --- reports/index.html | 9 +- reports/js/benchmarks.js | 1 + .../js/benchmarks/runtime-comparison-view.js | 2 +- .../js/benchmarks/sweep-comparison-view.js | 341 +++++++++++++++++- 4 files changed, 338 insertions(+), 15 deletions(-) diff --git a/reports/index.html b/reports/index.html index b88bd19..ebc276d 100644 --- a/reports/index.html +++ b/reports/index.html @@ -43,7 +43,13 @@ - +
+ + +
@@ -67,5 +73,6 @@ + diff --git a/reports/js/benchmarks.js b/reports/js/benchmarks.js index d6b2fc6..3d7d87d 100644 --- a/reports/js/benchmarks.js +++ b/reports/js/benchmarks.js @@ -140,6 +140,7 @@ function chartTypeSelect() else if (chartType == "dataset-comparison") { activeChartType = dc; } else if (chartType == "metric-comparison") { activeChartType = mc; } else if (chartType == "highest-metric-comparison") { activeChartType = hmc; } + else if (chartType == "sweep-runtime-comparison") { activeChartType = sc; } activeChartType.onTypeSelect(); } diff --git a/reports/js/benchmarks/runtime-comparison-view.js b/reports/js/benchmarks/runtime-comparison-view.js index 0de52ae..b667bfe 100644 --- a/reports/js/benchmarks/runtime-comparison-view.js +++ b/reports/js/benchmarks/runtime-comparison-view.js @@ -210,7 +210,7 @@ rc.paramSelect = function() var step = func(dbType === "sqlite" ? rc.results[0][2] : rc.results[0].step); var end = func(dbType === "sqlite" ? rc.results[0][3] : rc.results[0].end); var elem = 0; - for (var i = start; i < end; i += step, elem++) + for (var i = start; i <= end; i += step, elem++) { var new_option = document.createElement("option"); new_option.text = name + ": " + i; diff --git a/reports/js/benchmarks/sweep-comparison-view.js b/reports/js/benchmarks/sweep-comparison-view.js index 65fc641..60589e0 100644 --- a/reports/js/benchmarks/sweep-comparison-view.js +++ b/reports/js/benchmarks/sweep-comparison-view.js @@ -1,12 +1,13 @@ // Define namespace: sc = sweep comparison. var sc = sc = sc || {}; -sc.method_name = ""; -sc.param_name = ""; +sc.methodName = ""; +sc.paramName = ""; sc.dataset = ""; sc.libraries = []; -sc.active_libraries = []; +sc.activeLibraries = []; sc.results = []; +sc.sweepId = -1; // This chart type has been selected. sc.onTypeSelect = function() @@ -74,7 +75,7 @@ sc.methodSelect = function() var sqlstr = "SELECT DISTINCT methods.parameters, methods.sweep_id, " + "metrics.libary_id, COUNT(DISTINCT metrics.libary_id) AS count FROM " + - "methods, metrics WHERE methods.name = '" + sc.method_name + + "methods, metrics WHERE methods.name = '" + sc.methodName + "' AND methods.id = metrics.method_id AND methods.sweep_id != -1 " + "GROUP BY methods.parameters;"; var params = dbExec(sqlstr); @@ -85,7 +86,7 @@ sc.methodSelect = function() clearSelectBox(paramSelectBox); var newOption = document.createElement("option"); - param_select_box.add(newOption); + paramSelectBox.add(newOption); if ((dbType === "sqlite" && params[0]) || (dbType === "mysql" && params)) { @@ -119,28 +120,28 @@ sc.methodSelect = function() // Called when the user selects parameters. // Called when a set of parameters is selected. Now we are ready to draw the // chart. -rc.paramSelect = function() +sc.paramSelect = function() { // The user has selected a library and parameters. Now we need to generate // the list of datasets. var methodSelectBox = document.getElementById("method_select"); - rc.method_name = methodSelectBox.options[methodSelectBox.selectedIndex].text; + sc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; var paramSelectBox = document.getElementById("param_select"); var paramNameFull = paramSelectBox.options[paramSelectBox.selectedIndex].text; var sweepId = paramSelectBox.options[paramSelectBox.selectedIndex].id; - sc.param_name = param_name_full.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. + sc.paramName = paramNameFull.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. var sqlstr = "SELECT DISTINCT datasets.name FROM datasets, results, methods " + "WHERE datasets.id = results.dataset_id AND methods.name = '" - + sc.method_name + "' AND methods.sweep_id = " + sweepId + " AND " - + "results.method_id = methods.id AND methods.parameters = " + sc.paramName - + ";"; + + sc.methodName + "' AND methods.sweep_id = " + sweepId + " AND " + + "results.method_id = methods.id AND methods.parameters = '" + sc.paramName + + "';"; var datasets = dbExec(sqlstr); - console.log(JSON.stringify(params)); + console.log(JSON.stringify(datasets)); // Loop through the results and fill the third list box. - var datasetSelectBox = document.getElementById("dataset_select"); + var datasetSelectBox = document.getElementById("main_dataset_select"); clearSelectBox(datasetSelectBox); var newOption = document.createElement("option"); @@ -162,3 +163,317 @@ rc.paramSelect = function() datasetSelectBox.selectedIndex = 0; } + +sc.datasetSelect = function() +{ + // The user selected a dataset, so now we can plot. + var methodSelectBox = document.getElementById("method_select"); + sc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; + var paramSelectBox = document.getElementById("param_select"); + var paramNameFull = paramSelectBox.options[paramSelectBox.selectedIndex].text; + sc.sweepId = paramSelectBox.options[paramSelectBox.selectedIndex].id; + var datasetSelectBox = document.getElementById("main_dataset_select"); + var datasetName = + datasetSelectBox.options[datasetSelectBox.selectedIndex].text; + + sc.paramName = paramNameFull.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); + + var sqlstr = "SELECT DISTINCT * FROM " + + "(SELECT results.time as time, results.var as var, " + + " results.sweep_elem_id as sweep_elem_id, libraries.name as lib," + + " max(results.build_id) as bid, datasets.instances as di, " + + " datasets.attributes as da, datasets.size as ds" + + " FROM results, datasets, methods, libraries" + + " WHERE results.dataset_id = datasets.id" + + " AND results.method_id = methods.id" + + " AND methods.name = '" + sc.methodName + "'" + + " AND methods.parameters = '" + sc.paramName + "'" + + " AND libraries.id = results.libary_id" + + " AND datasets.name = '" + datasetName + "'" + + " GROUP BY lib, sweep_elem_id)" + + "tmp GROUP BY sweep_elem_id, lib;"; + + sc.results = dbExec(sqlstr); + sc.results = dbType === "sqlite" ? sc.results[0].values : sc.results; + + // Obtain unique list of libraries. + sc.libraries = sc.results.map( + function(d) { + return dbType === "sqlite" ? d[3] : d.lib; + }).reduce( + function(p, c) { + if (p.indexOf(c) < 0) p.push(c); return p; + }, []); + + // By default, all libraries are active. + sc.activeLibraries = {}; + for (i = 0; i < sc.libraries.length; ++i) + { + sc.activeLibraries[sc.libraries[i]] = true; + } + + clearChart(); + buildChart(); +} + +sc.clearChart = function() +{ + d3.select("svg").remove(); +} + +sc.buildChart = function() +{ + sc.results = sc.results.map( + function(d) + { + var runtime = dbType === "sqlite" ? d[0] : d.time; + if (runtime == -2) + { + if (dbType === "sqlite") + d[0] = "failure"; + else + d.time = "failure"; + } + else if (runtime == -1) + { + if (dbType === "sqlite") + d[0] = ">9000"; + else + d.time = "failure"; + } + + return d; + }); + + // Get the parameter name we are sweeping. + var params = JSON.parse(sc.paramName); + var name = ""; + for (var i in params) + { + if (params[i].search(/sweep\(/) !== -1) + { + name = i; + break; + } + } + + // Get lists of active libraries. + var activeLibraryList = sc.libraries.map(function(d) { return d; }).reduce( + function(p, c) + { + if (sc.activeLibraries[c] == true) + p.push(c); + return p; + }, []); + + var maxRuntime = d3.max(sc.results, + function(d) + { + if (sc.activeLibraries[dbType === "sqlite" ? d[3] : d.lib] == false) + return 0; + else + return mapRuntime(dbType === "sqlite" ? d[0] : d.time, 0); + }); + // Increase so we have 16 spare pixels at the top. + maxRuntime *= ((height + 16) / height); + + var runtimeScale = d3.scale.linear() + .domain([0, maxRuntime]) + .range([height, 0]); + + // We need to find out how big the sweep is. + var sweepSql = "SELECT type, begin, step, end FROM sweeps where id = " + sc.sweepId; + var sweepResults = dbExec(sweepSql); + sweepResults = dbType === "sqlite" ? sweepResults[0].values : sweepResults.results; + + var func = (dbType === "sqlite" ? sweepResults[0][0] : sweepResults[0].type) === "int" ? parseInt : parseFloat; + var start = func(dbType === "sqlite" ? sweepResults[0][1] : sweepResults[0].start); + var step = func(dbType === "sqlite" ? sweepResults[0][2] : sweepResults[0].step); + var end = func(dbType === "sqlite" ? sweepResults[0][3] : sweepResults[0].end); + + var sweepScale = d3.scale.linear() + .domain([start, end]) + .range([0, width]); + + var xAxis = d3.svg.axis().scale(sweepScale).orient("bottom"); + var yAxis = d3.svg.axis().scale(runtimeScale).orient("left").tickFormat(d3.format(".2f")); + + // Create svg object. + var svg = d3.select(".svgholder").append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + // Add x axis. + svg.append("g").attr("id", "xaxis") + .attr("class", "x axis") + .attr("transform", "translate(0, " + height + ")") + .call(xAxis) + .append("text") + .style("text-anchor", "end") + .attr("dx", 500) + .attr("dy", "3em") + .text("Value of parameter '" + name + "'"); + + // Add y axis. + svg.append("g").attr("id", "yaxis") + .attr("class", "y axis") + .call(yAxis) + .append("text") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text("Runtime (s)"); + + // Create tooltips. + var tip = d3.tip() + .attr("class", "d3-tip") + .offset([-10, 0]) + .html(function(d) { + var runtime = d[0]; + if (d[0] != ">9000" && d[0] != "failure") { + runtime = d[0].toFixed(2); + } + return "" + d[3] + "; " + name + ": " + (start + step * d[2]) + ": " + runtime + "s"; }); + svg.call(tip); + + // Add all of the data points. + var lineFunc = d3.svg.line() + .x(function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) + .y(function(d) { return runtimeScale(mapRuntime( + dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .interpolate("linear"); + + var lineResults = [] + for (var l in sc.libraries) + { + if (sc.activeLibraries[sc.libraries[l]] == true) + { + lineResults.push(sc.results.map(function(d) { return d; }).reduce(function(p, c) { if(c[3] == sc.libraries[l]) { p.push(c); } return p; }, [])); + } + else + { + lineResults.push([]); + } + } + + for (i = 0; i < lineResults.length; ++i) + { + console.log(JSON.stringify(lineResults[i])); + if (lineFunc(lineResults[i]) != null) + { + svg.append('svg:path') + .attr('d', lineFunc(lineResults[i])) + .attr('stroke', color(sc.libraries[i])) + .attr('stroke-width', 2) + .attr('fill', 'none'); + } + } + + for (i = 0; i < lineResults.length; i++) + { + if (lineFunc(lineResults[i]) == null) + continue; + + // Colored circle enclosed in white circle enclosed in background color + // circle; looks kind of nice. + svg.selectAll("dot").data(lineResults[i]).enter().append("circle") + .attr("r", 6) + .attr("cx", function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) + .attr("cy", function(d) { return runtimeScale(mapRuntime(dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .attr('fill', '#222222') + .on('mouseover', tip.show) + .on('mouseout', tip.hide); + svg.selectAll("dot").data(lineResults[i]).enter().append("circle") + .attr("r", 4) + .attr("cx", function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) + .attr("cy", function(d) { return runtimeScale(mapRuntime(dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .attr('fill', '#ffffff') + .on('mouseover', tip.show) + .on('mouseout', tip.hide); + svg.selectAll("dot").data(lineResults[i]).enter().append("circle") + .attr("r", 3) + .attr("cx", function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) + .attr("cy", function(d) { return runtimeScale(mapRuntime(dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .attr('fill', function(d) { return color(d[4]) }) + .on('mouseover', tip.show) + .on('mouseout', tip.hide); + } + + // Create the library selector. + var librarySelectTitle = d3.select(".legendholder").append("div") + .attr("class", "library-select-title"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-text") + .text("Libraries:"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-open-paren") + .text("("); + librarySelectTitle.append("div") + .attr("class", "library-select-title-enable-all") + .text("enable all") + .on('click', function() { sc.enableAllLibraries(); }); + librarySelectTitle.append("div") + .attr("class", "library-select-title-bar") + .text("|"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-disable-all") + .text("disable all") + .on('click', function() { sc.disableAllLibraries(); }); + librarySelectTitle.append("div") + .attr("class", "library-select-title-close-paren") + .text(")"); + + var libraryDivs = d3.select(".legendholder").selectAll("input") + .data(sc.libraries) + .enter() + .append("div") + .attr("class", "library-select-div") + .attr("id", function(d) { return d + '-library-checkbox-div'; }); + + libraryDivs.append("label") + .attr('for', function(d) { return d + '-library-checkbox'; }) + .style('background', color) + .attr('class', 'library-select-color'); + + libraryDivs.append("input") + .property("checked", function(d) { return sc.activeLibraries[d]; }) + .attr("type", "checkbox") + .attr("id", function(d) { return d + '-library-checkbox'; }) + .attr('class', 'library-select-box') + .attr("onClick", function(d, i) { return "sc.toggleLibrary(\"" + d + "\");"; }); + + libraryDivs.append("label") + .attr('for', function(d) { return d + '-library-checkbox'; }) + .attr('class', 'library-select-label') + .text(function(d) { return d; }); +} + +// Toggle a library to on or off. +sc.toggleLibrary = function(library) +{ + sc.activeLibraries[library] = !sc.activeLibraries[library]; + + clearChart(); + buildChart(); +} + +// Set all libraries on. +sc.enableAllLibraries = function() +{ + for (v in sc.activeLibraries) { sc.activeLibraries[v] = true; } + + clearChart(); + buildChart(); +} + +// Set all libraries off. +sc.disableAllLibraries = function() +{ + for (v in sc.activeLibraries) { sc.activeLibraries[v] = false; } + + clearChart(); + buildChart(); +} From 11b96bf8ae1d5f401fa3b91f26758532929733f2 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 16 Aug 2017 17:35:21 -0400 Subject: [PATCH 3/9] Partially-finished metric sweep. --- .../sweep-metric-comparison-view.js | 492 ++++++++++++++++++ 1 file changed, 492 insertions(+) create mode 100644 reports/js/benchmarks/sweep-metric-comparison-view.js diff --git a/reports/js/benchmarks/sweep-metric-comparison-view.js b/reports/js/benchmarks/sweep-metric-comparison-view.js new file mode 100644 index 0000000..395f20b --- /dev/null +++ b/reports/js/benchmarks/sweep-metric-comparison-view.js @@ -0,0 +1,492 @@ +// Define namespace: smc = sweep metric comparison. +var smc = smc = smc || {}; + +smc.methodName = ""; +smc.paramName = ""; +smc.dataset = ""; +smc.libraries = []; +smc.activeLibraries = []; +smc.results = []; +smc.sweepId = -1; + +// This chart type has been selected. +smc.onTypeSelect = function() +{ + // The user needs to be able to select a method, then parameters, then a + // dataset. + var selectHolder = d3.select(".selectholder"); + selectHolder.append("label") + .attr("for", "method_select") + .attr("class", "method-select-label") + .text("Select method:"); + selectHolder.append("select") + .attr("id", "method_select") + .attr("onchange", "smc.methodSelect()"); + selectHolder.append("label") + .attr("for", "param_select") + .attr("class", "param-select-label") + .text("Select parameters:"); + selectHolder.append("select") + .attr("id", "param_select") + .attr("onchange", "smc.paramSelect()"); + selectHolder.append("label") + .attr("for", "main_dataset_select") + .attr("class", "main-dataset-select-label") + .text("Select dataset:"); + selectHolder.append("select") + .attr("id", "main_dataset_select") + .attr("onchange", "smc.datasetSelect()"); + selectHolder.append("label") + .attr("for", "metric_select") + .attr("class", "metric-select-label") + .text("Select metric:"); + selectHolder.append("select") + .attr("id", "metric_select") + .attr("onchange", "smc.metricSelect()"); + + smc.listMethods(); +} + +// List the available methods where there is a sweep. +smc.listMethods = function() +{ + var methods = dbExec("SELECT DISTINCT methods.name FROM methods, results " + + "WHERE methods.id = results.method_id AND methods.sweep_id != -1 " + + "ORDER BY name;"); + var methodSelectBox = document.getElementById("method_select"); + + // Remove old things. + clearSelectBox(methodSelectBox); + + // Add new things. + var length = dbType === "sqlite" ? methods[0].values.length : methods.length; + for (i = 0; i < length; i++) + { + var newOption = document.createElement("option"); + newOption.text = dbType === "sqlite" ? methods[0].values[i] : + methods[i].name; + methodSelectBox.add(newOption); + } + methodSelectBox.selectedIndex = -1; + + // Clear parameters box. + clearSelectBox(document.getElementById("param_select")); +} + +// Called when the user selects a method. +smc.methodSelect = function() +{ + // Extract the name of the method we selected. + var methodSelectBox = document.getElementById("method_select"); + smc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; // At higher scope. + + var sqlstr = "SELECT DISTINCT methods.parameters, methods.sweep_id, " + + "metrics.libary_id, COUNT(DISTINCT metrics.libary_id) AS count FROM " + + "methods, metrics WHERE methods.name = '" + smc.methodName + + "' AND methods.id = metrics.method_id AND methods.sweep_id != -1 " + + "GROUP BY methods.parameters;"; + var params = dbExec(sqlstr); + console.log(JSON.stringify(params)); + + // Loop through results and fill the second list box. + var paramSelectBox = document.getElementById("param_select"); + clearSelectBox(paramSelectBox); + + var newOption = document.createElement("option"); + paramSelectBox.add(newOption); + + if ((dbType === "sqlite" && params[0]) || (dbType === "mysql" && params)) + { + // Put in the new options. + var length = dbType === "sqlite" ? params[0].values.length : params.length; + for (i = 0; i < length; i++) + { + var newOption = document.createElement("option"); + + var parameters = dbType === "sqlite" ? params[0].values[i][0] : params[i].parameters; + var sweepId = dbType === "sqlite" ? params[0].values[i][1] : params[i].sweep_id; + var libraries = dbType === "sqlite" ? params[0].values[i][3] : params[i].count; + + if (parameters) + { + newOption.text = parameters + " (" + libraries + " libraries)"; + } + else + { + newOption.text = "[no parameters] (" + libraries + " libraries)"; + } + newOption.id = sweepId; + + paramSelectBox.add(newOption); + } + } + + paramSelectBox.selectedIndex = 0; +} + +// Called when the user selects parameters. +// Called when a set of parameters is selected. Now we are ready to draw the +// chart. +smc.paramSelect = function() +{ + // The user has selected a library and parameters. Now we need to generate + // the list of datasets. + var methodSelectBox = document.getElementById("method_select"); + smc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; + var paramSelectBox = document.getElementById("param_select"); + var paramNameFull = paramSelectBox.options[paramSelectBox.selectedIndex].text; + var sweepId = paramSelectBox.options[paramSelectBox.selectedIndex].id; + + smc.paramName = paramNameFull.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // At higher scope. + + var sqlstr = "SELECT DISTINCT datasets.name FROM datasets, results, methods " + + "WHERE datasets.id = results.dataset_id AND methods.name = '" + + smc.methodName + "' AND methods.sweep_id = " + sweepId + " AND " + + "results.method_id = methods.id AND methods.parameters = '" + smc.paramName + + "';"; + var datasets = dbExec(sqlstr); + console.log(JSON.stringify(datasets)); + + // Loop through the results and fill the third list box. + var datasetSelectBox = document.getElementById("main_dataset_select"); + clearSelectBox(datasetSelectBox); + + var newOption = document.createElement("option"); + datasetSelectBox.add(newOption); + + if ((dbType === "sqlite" && datasets[0]) || (dbType === "mysql" && datasets)) + { + // Put in the datasets. + var length = dbType === "sqlite" ? datasets[0].values.length : datasets.length; + for (i = 0; i < length; i++) + { + newOption = document.createElement("option"); + + var datasetName = dbType === "sqlite" ? datasets[0].values[i][0] : datasets[i].name; + newOption.text = datasetName; + datasetSelectBox.add(newOption); + } + } + + datasetSelectBox.selectedIndex = 0; +} + +smc.datasetSelect = function() +{ + // The user selected a dataset, so now we can plot. + var methodSelectBox = document.getElementById("method_select"); + smc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; + var paramSelectBox = document.getElementById("param_select"); + var paramNameFull = paramSelectBox.options[paramSelectBox.selectedIndex].text; + smc.sweepId = paramSelectBox.options[paramSelectBox.selectedIndex].id; + var datasetSelectBox = document.getElementById("main_dataset_select"); + var datasetName = + datasetSelectBox.options[datasetSelectBox.selectedIndex].text; + + smc.paramName = paramNameFull.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); + + // What metrics do we have available? + // TODO: from here +} + +smc.metricSelect = function() + + var sqlstr = "SELECT DISTINCT * FROM " + + "(SELECT results.time as time, results.var as var, " + + " results.sweep_elem_id as sweep_elem_id, libraries.name as lib," + + " max(results.build_id) as bid, datasets.instances as di, " + + " datasets.attributes as da, datasets.size as ds" + + " FROM results, datasets, methods, libraries" + + " WHERE results.dataset_id = datasets.id" + + " AND results.method_id = methods.id" + + " AND methods.name = '" + smc.methodName + "'" + + " AND methods.parameters = '" + smc.paramName + "'" + + " AND libraries.id = results.libary_id" + + " AND datasets.name = '" + datasetName + "'" + + " GROUP BY lib, sweep_elem_id)" + + "tmp GROUP BY sweep_elem_id, lib;"; + + smc.results = dbExec(sqlstr); + smc.results = dbType === "sqlite" ? sc.results[0].values : sc.results; + + // Obtain unique list of libraries. + smc.libraries = sc.results.map( + function(d) { + return dbType === "sqlite" ? d[3] : d.lib; + }).reduce( + function(p, c) { + if (p.indexOf(c) < 0) p.push(c); return p; + }, []); + + // By default, all libraries are active. + smc.activeLibraries = {}; + for (i = 0; i < smc.libraries.length; ++i) + { + smc.activeLibraries[sc.libraries[i]] = true; + } + + clearChart(); + buildChart(); +} + +smc.clearChart = function() +{ + d3.select("svg").remove(); +} + +smc.buildChart = function() +{ + smc.results = sc.results.map( + function(d) + { + var runtime = dbType === "sqlite" ? d[0] : d.time; + if (runtime == -2) + { + if (dbType === "sqlite") + d[0] = "failure"; + else + d.time = "failure"; + } + else if (runtime == -1) + { + if (dbType === "sqlite") + d[0] = ">9000"; + else + d.time = "failure"; + } + + return d; + }); + + // Get the parameter name we are sweeping. + var params = JSON.parse(smc.paramName); + var name = ""; + for (var i in params) + { + if (params[i].search(/sweep\(/) !== -1) + { + name = i; + break; + } + } + + // Get lists of active libraries. + var activeLibraryList = smc.libraries.map(function(d) { return d; }).reduce( + function(p, c) + { + if (smc.activeLibraries[c] == true) + p.push(c); + return p; + }, []); + + var maxRuntime = d3.max(smc.results, + function(d) + { + if (smc.activeLibraries[dbType === "sqlite" ? d[3] : d.lib] == false) + return 0; + else + return mapRuntime(dbType === "sqlite" ? d[0] : d.time, 0); + }); + // Increase so we have 16 spare pixels at the top. + maxRuntime *= ((height + 16) / height); + + var runtimeScale = d3.scale.linear() + .domain([0, maxRuntime]) + .range([height, 0]); + + // We need to find out how big the sweep is. + var sweepSql = "SELECT type, begin, step, end FROM sweeps where id = " + smc.sweepId; + var sweepResults = dbExec(sweepSql); + sweepResults = dbType === "sqlite" ? sweepResults[0].values : sweepResults.results; + + var func = (dbType === "sqlite" ? sweepResults[0][0] : sweepResults[0].type) === "int" ? parseInt : parseFloat; + var start = func(dbType === "sqlite" ? sweepResults[0][1] : sweepResults[0].start); + var step = func(dbType === "sqlite" ? sweepResults[0][2] : sweepResults[0].step); + var end = func(dbType === "sqlite" ? sweepResults[0][3] : sweepResults[0].end); + + var sweepScale = d3.scale.linear() + .domain([start, end]) + .range([0, width]); + + var xAxis = d3.svg.axis().scale(sweepScale).orient("bottom"); + var yAxis = d3.svg.axis().scale(runtimeScale).orient("left").tickFormat(d3.format(".2f")); + + // Create svg object. + var svg = d3.select(".svgholder").append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + // Add x axis. + svg.append("g").attr("id", "xaxis") + .attr("class", "x axis") + .attr("transform", "translate(0, " + height + ")") + .call(xAxis) + .append("text") + .style("text-anchor", "end") + .attr("dx", 500) + .attr("dy", "3em") + .text("Value of parameter '" + name + "'"); + + // Add y axis. + svg.append("g").attr("id", "yaxis") + .attr("class", "y axis") + .call(yAxis) + .append("text") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text("Runtime (s)"); + + // Create tooltips. + var tip = d3.tip() + .attr("class", "d3-tip") + .offset([-10, 0]) + .html(function(d) { + var runtime = d[0]; + if (d[0] != ">9000" && d[0] != "failure") { + runtime = d[0].toFixed(2); + } + return "" + d[3] + "; " + name + ": " + (start + step * d[2]) + ": " + runtime + "s"; }); + svg.call(tip); + + // Add all of the data points. + var lineFunc = d3.svg.line() + .x(function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) + .y(function(d) { return runtimeScale(mapRuntime( + dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .interpolate("linear"); + + var lineResults = [] + for (var l in smc.libraries) + { + if (smc.activeLibraries[sc.libraries[l]] == true) + { + lineResults.push(smc.results.map(function(d) { return d; }).reduce(function(p, c) { if(c[3] == sc.libraries[l]) { p.push(c); } return p; }, [])); + } + else + { + lineResults.push([]); + } + } + + for (i = 0; i < lineResults.length; ++i) + { + console.log(JSON.stringify(lineResults[i])); + if (lineFunc(lineResults[i]) != null) + { + svg.append('svg:path') + .attr('d', lineFunc(lineResults[i])) + .attr('stroke', color(smc.libraries[i])) + .attr('stroke-width', 2) + .attr('fill', 'none'); + } + } + + for (i = 0; i < lineResults.length; i++) + { + if (lineFunc(lineResults[i]) == null) + continue; + + // Colored circle enclosed in white circle enclosed in background color + // circle; looks kind of nice. + svg.selectAll("dot").data(lineResults[i]).enter().append("circle") + .attr("r", 6) + .attr("cx", function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) + .attr("cy", function(d) { return runtimeScale(mapRuntime(dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .attr('fill', '#222222') + .on('mouseover', tip.show) + .on('mouseout', tip.hide); + svg.selectAll("dot").data(lineResults[i]).enter().append("circle") + .attr("r", 4) + .attr("cx", function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) + .attr("cy", function(d) { return runtimeScale(mapRuntime(dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .attr('fill', '#ffffff') + .on('mouseover', tip.show) + .on('mouseout', tip.hide); + svg.selectAll("dot").data(lineResults[i]).enter().append("circle") + .attr("r", 3) + .attr("cx", function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) + .attr("cy", function(d) { return runtimeScale(mapRuntime(dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .attr('fill', function(d) { return color(d[4]) }) + .on('mouseover', tip.show) + .on('mouseout', tip.hide); + } + + // Create the library selector. + var librarySelectTitle = d3.select(".legendholder").append("div") + .attr("class", "library-select-title"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-text") + .text("Libraries:"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-open-paren") + .text("("); + librarySelectTitle.append("div") + .attr("class", "library-select-title-enable-all") + .text("enable all") + .on('click', function() { smc.enableAllLibraries(); }); + librarySelectTitle.append("div") + .attr("class", "library-select-title-bar") + .text("|"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-disable-all") + .text("disable all") + .on('click', function() { smc.disableAllLibraries(); }); + librarySelectTitle.append("div") + .attr("class", "library-select-title-close-paren") + .text(")"); + + var libraryDivs = d3.select(".legendholder").selectAll("input") + .data(smc.libraries) + .enter() + .append("div") + .attr("class", "library-select-div") + .attr("id", function(d) { return d + '-library-checkbox-div'; }); + + libraryDivs.append("label") + .attr('for', function(d) { return d + '-library-checkbox'; }) + .style('background', color) + .attr('class', 'library-select-color'); + + libraryDivs.append("input") + .property("checked", function(d) { return smc.activeLibraries[d]; }) + .attr("type", "checkbox") + .attr("id", function(d) { return d + '-library-checkbox'; }) + .attr('class', 'library-select-box') + .attr("onClick", function(d, i) { return "smc.toggleLibrary(\"" + d + "\");"; }); + + libraryDivs.append("label") + .attr('for', function(d) { return d + '-library-checkbox'; }) + .attr('class', 'library-select-label') + .text(function(d) { return d; }); +} + +// Toggle a library to on or off. +smc.toggleLibrary = function(library) +{ + smc.activeLibraries[library] = !hc.activeLibraries[library]; + + clearChart(); + buildChart(); +} + +// Set all libraries on. +smc.enableAllLibraries = function() +{ + for (v in smc.activeLibraries) { hc.activeLibraries[v] = true; } + + clearChart(); + buildChart(); +} + +// Set all libraries off. +smc.disableAllLibraries = function() +{ + for (v in hc.activeLibraries) { hc.activeLibraries[v] = false; } + + clearChart(); + buildChart(); +} From 91c74a088d4871a11b1b0e5c02aded8d20850e0a Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 16 Aug 2017 18:00:16 -0400 Subject: [PATCH 4/9] Updates on metric comparison view. --- reports/index.html | 6 ++++++ reports/js/benchmarks.js | 1 + reports/js/benchmarks/sweep-comparison-view.js | 4 ++++ reports/js/benchmarks/sweep-metric-comparison-view.js | 6 ++++-- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/reports/index.html b/reports/index.html index ebc276d..6b9b440 100644 --- a/reports/index.html +++ b/reports/index.html @@ -50,6 +50,11 @@ +
+ + +
@@ -74,5 +79,6 @@ + diff --git a/reports/js/benchmarks.js b/reports/js/benchmarks.js index 3d7d87d..fb0bb6d 100644 --- a/reports/js/benchmarks.js +++ b/reports/js/benchmarks.js @@ -141,6 +141,7 @@ function chartTypeSelect() else if (chartType == "metric-comparison") { activeChartType = mc; } else if (chartType == "highest-metric-comparison") { activeChartType = hmc; } else if (chartType == "sweep-runtime-comparison") { activeChartType = sc; } + else if (chartType == "sweep-metric-comparison") { activeChartType = smc; } activeChartType.onTypeSelect(); } diff --git a/reports/js/benchmarks/sweep-comparison-view.js b/reports/js/benchmarks/sweep-comparison-view.js index 60589e0..4a17a8c 100644 --- a/reports/js/benchmarks/sweep-comparison-view.js +++ b/reports/js/benchmarks/sweep-comparison-view.js @@ -219,6 +219,10 @@ sc.datasetSelect = function() sc.clearChart = function() { d3.select("svg").remove(); + d3.selectAll(".d3-tip").remove(); + d3.selectAll(".library-select-title").remove(); + d3.selectAll(".library_select_div").remove(); + d3.selectAll(".legendholder").selectAll("*").remove(); } sc.buildChart = function() diff --git a/reports/js/benchmarks/sweep-metric-comparison-view.js b/reports/js/benchmarks/sweep-metric-comparison-view.js index 395f20b..d91195c 100644 --- a/reports/js/benchmarks/sweep-metric-comparison-view.js +++ b/reports/js/benchmarks/sweep-metric-comparison-view.js @@ -12,6 +12,7 @@ smc.sweepId = -1; // This chart type has been selected. smc.onTypeSelect = function() { + console.log("onTypeSelect"); // The user needs to be able to select a method, then parameters, then a // dataset. var selectHolder = d3.select(".selectholder"); @@ -186,11 +187,12 @@ smc.datasetSelect = function() smc.paramName = paramNameFull.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); // What metrics do we have available? - // TODO: from here + // TODO: write sql query to do this. + var sqlstr = " "; } smc.metricSelect = function() - +{ var sqlstr = "SELECT DISTINCT * FROM " + "(SELECT results.time as time, results.var as var, " + " results.sweep_elem_id as sweep_elem_id, libraries.name as lib," From f8a36899c50bde7888f9cc5c513308a2c2bdfe36 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 18 Aug 2017 11:27:58 -0400 Subject: [PATCH 5/9] Finish sweep metric comparison view. --- reports/css/d3-style.css | 2 +- .../sweep-metric-comparison-view.js | 197 +++++++++++------- 2 files changed, 120 insertions(+), 79 deletions(-) diff --git a/reports/css/d3-style.css b/reports/css/d3-style.css index ce955a9..acbd43f 100644 --- a/reports/css/d3-style.css +++ b/reports/css/d3-style.css @@ -157,7 +157,7 @@ svg text { } .method-select-label, .param-select-label, .main-dataset-select-label, -.sweep-select-label { +.sweep-select-label, .metric-select-label { float: left; font-size: 75%; color: #ffffff; diff --git a/reports/js/benchmarks/sweep-metric-comparison-view.js b/reports/js/benchmarks/sweep-metric-comparison-view.js index d91195c..3b30bf5 100644 --- a/reports/js/benchmarks/sweep-metric-comparison-view.js +++ b/reports/js/benchmarks/sweep-metric-comparison-view.js @@ -12,7 +12,6 @@ smc.sweepId = -1; // This chart type has been selected. smc.onTypeSelect = function() { - console.log("onTypeSelect"); // The user needs to be able to select a method, then parameters, then a // dataset. var selectHolder = d3.select(".selectholder"); @@ -87,7 +86,6 @@ smc.methodSelect = function() "' AND methods.id = metrics.method_id AND methods.sweep_id != -1 " + "GROUP BY methods.parameters;"; var params = dbExec(sqlstr); - console.log(JSON.stringify(params)); // Loop through results and fill the second list box. var paramSelectBox = document.getElementById("param_select"); @@ -146,7 +144,6 @@ smc.paramSelect = function() + "results.method_id = methods.id AND methods.parameters = '" + smc.paramName + "';"; var datasets = dbExec(sqlstr); - console.log(JSON.stringify(datasets)); // Loop through the results and fill the third list box. var datasetSelectBox = document.getElementById("main_dataset_select"); @@ -186,35 +183,58 @@ smc.datasetSelect = function() smc.paramName = paramNameFull.split("(").slice(0, -1).join("(").replace(/^\s+|\s+$/g, ''); - // What metrics do we have available? - // TODO: write sql query to do this. - var sqlstr = " "; -} - -smc.metricSelect = function() -{ + // What metrics do we have available? We can do our actual query for results + // here, then we need to parse it. var sqlstr = "SELECT DISTINCT * FROM " - + "(SELECT results.time as time, results.var as var, " - + " results.sweep_elem_id as sweep_elem_id, libraries.name as lib," - + " max(results.build_id) as bid, datasets.instances as di, " + + "(SELECT metrics.metric as metric, " + + " metrics.sweep_elem_id as sweep_elem_id, libraries.name as lib," + + " max(metrics.build_id) as bid, datasets.instances as di, " + " datasets.attributes as da, datasets.size as ds" - + " FROM results, datasets, methods, libraries" - + " WHERE results.dataset_id = datasets.id" - + " AND results.method_id = methods.id" + + " FROM metrics, datasets, methods, libraries" + + " WHERE metrics.dataset_id = datasets.id" + + " AND metrics.method_id = methods.id" + " AND methods.name = '" + smc.methodName + "'" + " AND methods.parameters = '" + smc.paramName + "'" - + " AND libraries.id = results.libary_id" + + " AND libraries.id = metrics.libary_id" + " AND datasets.name = '" + datasetName + "'" - + " GROUP BY lib, sweep_elem_id)" + + " GROUP BY lib, metrics.sweep_elem_id)" + "tmp GROUP BY sweep_elem_id, lib;"; - smc.results = dbExec(sqlstr); - smc.results = dbType === "sqlite" ? sc.results[0].values : sc.results; + smc.results = dbType === "sqlite" ? smc.results[0].values : smc.results; + + // Now we have to parse through the metrics and see what we find. + addMetric = function(p, c) + { + var json = jQuery.parseJSON(dbType === "sqlite" ? c[0] : c.metric); + for(var k in json) + if(p.indexOf(k) < 0) + p.push(k); + return p; + }; + metrics = smc.results.reduce(addMetric, []); + + var metric_select_box = document.getElementById("metric_select"); + clearSelectBox(metric_select_box); + for (i = 0; i < metrics.length; i++) + { + var new_option = document.createElement("option"); + new_option.text = metrics[i]; + metric_select_box.add(new_option); + } + metric_select_box.selectedIndex = -1; +} + +smc.metricSelect = function() +{ + // We've already got the results, and now the user has specified the metric + // they want plotted. + var metricSelectBox = document.getElementById("metric_select"); + smc.metricName = metricSelectBox.options[metricSelectBox.selectedIndex].text; // Obtain unique list of libraries. - smc.libraries = sc.results.map( + smc.libraries = smc.results.map( function(d) { - return dbType === "sqlite" ? d[3] : d.lib; + return dbType === "sqlite" ? d[2] : d.lib; }).reduce( function(p, c) { if (p.indexOf(c) < 0) p.push(c); return p; @@ -224,7 +244,7 @@ smc.metricSelect = function() smc.activeLibraries = {}; for (i = 0; i < smc.libraries.length; ++i) { - smc.activeLibraries[sc.libraries[i]] = true; + smc.activeLibraries[smc.libraries[i]] = true; } clearChart(); @@ -234,33 +254,44 @@ smc.metricSelect = function() smc.clearChart = function() { d3.select("svg").remove(); + d3.selectAll(".d3-tip").remove(); + d3.selectAll(".library-select-title").remove(); + d3.selectAll(".library_select_div").remove(); + d3.selectAll(".legendholder").selectAll("*").remove(); } -smc.buildChart = function() +smc.extractRuntime = function(d) { - smc.results = sc.results.map( - function(d) - { - var runtime = dbType === "sqlite" ? d[0] : d.time; - if (runtime == -2) - { - if (dbType === "sqlite") - d[0] = "failure"; - else - d.time = "failure"; - } - else if (runtime == -1) - { - if (dbType === "sqlite") - d[0] = ">9000"; - else - d.time = "failure"; - } - - return d; - }); + var json = jQuery.parseJSON(d); + for (var m in json) + { + if (m == "Runtime") + { + if (json[m] == -2) + return "failure"; + else if (json[m] == -1) + return ">9000"; + else + return json[m]; + } + } - // Get the parameter name we are sweeping. + return "failure"; +} + +smc.extractMetric = function(d, metricName, notFoundValue) +{ + var json = jQuery.parseJSON(d); + for (var m in json) + if (m == metricName) + return json[m]; + + return notFoundValue; +} + +smc.buildChart = function() +{ + // Get the parameter name we are sweeping. var params = JSON.parse(smc.paramName); var name = ""; for (var i in params) @@ -276,7 +307,7 @@ smc.buildChart = function() var activeLibraryList = smc.libraries.map(function(d) { return d; }).reduce( function(p, c) { - if (smc.activeLibraries[c] == true) + if (smc.activeLibraries[c] == true) p.push(c); return p; }, []); @@ -284,17 +315,28 @@ smc.buildChart = function() var maxRuntime = d3.max(smc.results, function(d) { - if (smc.activeLibraries[dbType === "sqlite" ? d[3] : d.lib] == false) + if (smc.activeLibraries[dbType === "sqlite" ? d[2] : d.lib] == false) + return 0; + else + return smc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric); + }); + + var maxMetric = d3.max(smc.results, + function(d) + { + if (smc.activeLibraries[dbType === "sqlite" ? d[2] : d.lib] == false) return 0; else - return mapRuntime(dbType === "sqlite" ? d[0] : d.time, 0); + return smc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, + smc.metricName, 0); }); + // Increase so we have 16 spare pixels at the top. - maxRuntime *= ((height + 16) / height); + maxMetric *= ((height + 16) / height); var runtimeScale = d3.scale.linear() .domain([0, maxRuntime]) - .range([height, 0]); + .range([0, width]); // We need to find out how big the sweep is. var sweepSql = "SELECT type, begin, step, end FROM sweeps where id = " + smc.sweepId; @@ -306,12 +348,12 @@ smc.buildChart = function() var step = func(dbType === "sqlite" ? sweepResults[0][2] : sweepResults[0].step); var end = func(dbType === "sqlite" ? sweepResults[0][3] : sweepResults[0].end); - var sweepScale = d3.scale.linear() - .domain([start, end]) - .range([0, width]); + var metricScale = d3.scale.linear() + .domain([0, maxMetric]) + .range([height, 0]); - var xAxis = d3.svg.axis().scale(sweepScale).orient("bottom"); - var yAxis = d3.svg.axis().scale(runtimeScale).orient("left").tickFormat(d3.format(".2f")); + var xAxis = d3.svg.axis().scale(runtimeScale).orient("bottom"); + var yAxis = d3.svg.axis().scale(metricScale).orient("left"); // Create svg object. var svg = d3.select(".svgholder").append("svg") @@ -329,7 +371,7 @@ smc.buildChart = function() .style("text-anchor", "end") .attr("dx", 500) .attr("dy", "3em") - .text("Value of parameter '" + name + "'"); + .text("Runtime (s)"); // Add y axis. svg.append("g").attr("id", "yaxis") @@ -340,43 +382,42 @@ smc.buildChart = function() .attr("y", 6) .attr("dy", ".71em") .style("text-anchor", "end") - .text("Runtime (s)"); + .text(smc.metricName); // Create tooltips. var tip = d3.tip() .attr("class", "d3-tip") .offset([-10, 0]) .html(function(d) { - var runtime = d[0]; - if (d[0] != ">9000" && d[0] != "failure") { - runtime = d[0].toFixed(2); + var runtime = smc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric); + if (runtime != ">9000" && runtime != "failure") { + runtime = runtime.toFixed(2); } - return "" + d[3] + "; " + name + ": " + (start + step * d[2]) + ": " + runtime + "s"; }); + var metricValue = smc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, smc.metricName, ""); + return "" + d[2] + "; " + name + ": " + (start + step * d[1]) + ": " + smc.metricName + " " + metricValue + ", " + runtime + "s"; }); svg.call(tip); // Add all of the data points. var lineFunc = d3.svg.line() - .x(function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) - .y(function(d) { return runtimeScale(mapRuntime( - dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .x(function(d) { return runtimeScale(mapRuntime(smc.extractRuntime( + dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .y(function(d) { return metricScale(smc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, smc.metricName, 0)); }) .interpolate("linear"); var lineResults = [] for (var l in smc.libraries) { - if (smc.activeLibraries[sc.libraries[l]] == true) + if (smc.activeLibraries[smc.libraries[l]] == true) { - lineResults.push(smc.results.map(function(d) { return d; }).reduce(function(p, c) { if(c[3] == sc.libraries[l]) { p.push(c); } return p; }, [])); + lineResults.push(smc.results.map(function(d) { return d; }).reduce(function(p, c) { if(c[2] == smc.libraries[l]) { p.push(c); } return p; }, [])); } else { lineResults.push([]); } } - for (i = 0; i < lineResults.length; ++i) { - console.log(JSON.stringify(lineResults[i])); if (lineFunc(lineResults[i]) != null) { svg.append('svg:path') @@ -396,22 +437,22 @@ smc.buildChart = function() // circle; looks kind of nice. svg.selectAll("dot").data(lineResults[i]).enter().append("circle") .attr("r", 6) - .attr("cx", function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) - .attr("cy", function(d) { return runtimeScale(mapRuntime(dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .attr("cx", function(d) { return runtimeScale(mapRuntime(smc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .attr("cy", function(d) { return metricScale(smc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, smc.metricName, 0)); }) .attr('fill', '#222222') .on('mouseover', tip.show) .on('mouseout', tip.hide); svg.selectAll("dot").data(lineResults[i]).enter().append("circle") .attr("r", 4) - .attr("cx", function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) - .attr("cy", function(d) { return runtimeScale(mapRuntime(dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .attr("cx", function(d) { return runtimeScale(mapRuntime(smc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .attr("cy", function(d) { return metricScale(smc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, smc.metricName, 0)); }) .attr('fill', '#ffffff') .on('mouseover', tip.show) .on('mouseout', tip.hide); svg.selectAll("dot").data(lineResults[i]).enter().append("circle") .attr("r", 3) - .attr("cx", function(d) { return sweepScale(dbType === "sqlite" ? start + step * d[2] : start + step * d.sweep_elem_id); }) - .attr("cy", function(d) { return runtimeScale(mapRuntime(dbType === "sqlite" ? d[0] : d.time, maxRuntime)); }) + .attr("cx", function(d) { return runtimeScale(mapRuntime(smc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .attr("cy", function(d) { return metricScale(smc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, smc.metricName, 0)); }) .attr('fill', function(d) { return color(d[4]) }) .on('mouseover', tip.show) .on('mouseout', tip.hide); @@ -469,7 +510,7 @@ smc.buildChart = function() // Toggle a library to on or off. smc.toggleLibrary = function(library) { - smc.activeLibraries[library] = !hc.activeLibraries[library]; + smc.activeLibraries[library] = !smc.activeLibraries[library]; clearChart(); buildChart(); @@ -478,7 +519,7 @@ smc.toggleLibrary = function(library) // Set all libraries on. smc.enableAllLibraries = function() { - for (v in smc.activeLibraries) { hc.activeLibraries[v] = true; } + for (v in smc.activeLibraries) { smc.activeLibraries[v] = true; } clearChart(); buildChart(); @@ -487,7 +528,7 @@ smc.enableAllLibraries = function() // Set all libraries off. smc.disableAllLibraries = function() { - for (v in hc.activeLibraries) { hc.activeLibraries[v] = false; } + for (v in smc.activeLibraries) { smc.activeLibraries[v] = false; } clearChart(); buildChart(); From f0e1191ef0a07c6436e5d6244174b3fc59ea2579 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 18 Aug 2017 11:29:28 -0400 Subject: [PATCH 6/9] Add clear() function. --- reports/js/benchmarks/sweep-comparison-view.js | 5 +++++ reports/js/benchmarks/sweep-metric-comparison-view.js | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/reports/js/benchmarks/sweep-comparison-view.js b/reports/js/benchmarks/sweep-comparison-view.js index 4a17a8c..551a959 100644 --- a/reports/js/benchmarks/sweep-comparison-view.js +++ b/reports/js/benchmarks/sweep-comparison-view.js @@ -40,6 +40,11 @@ sc.onTypeSelect = function() sc.listMethods(); } +sc.clear = function() +{ + sc.clearChart(); +} + // List the available methods where there is a sweep. sc.listMethods = function() { diff --git a/reports/js/benchmarks/sweep-metric-comparison-view.js b/reports/js/benchmarks/sweep-metric-comparison-view.js index 3b30bf5..f7f87cf 100644 --- a/reports/js/benchmarks/sweep-metric-comparison-view.js +++ b/reports/js/benchmarks/sweep-metric-comparison-view.js @@ -47,6 +47,11 @@ smc.onTypeSelect = function() smc.listMethods(); } +smc.clear = function() +{ + smc.clearChart(); +} + // List the available methods where there is a sweep. smc.listMethods = function() { From 84df40a4d595c4d79a69405bbd5bd660bdf8bd73 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 21 Sep 2017 10:29:39 -0400 Subject: [PATCH 7/9] Add the actual view I wanted to make. --- reports/css/d3-style.css | 7 +- reports/index.html | 6 + reports/js/benchmarks.js | 1 + .../all-params-metric-comparison-view.js | 582 ++++++++++++++++++ .../sweep-metric-comparison-view.js | 10 +- 5 files changed, 598 insertions(+), 8 deletions(-) create mode 100644 reports/js/benchmarks/all-params-metric-comparison-view.js diff --git a/reports/css/d3-style.css b/reports/css/d3-style.css index acbd43f..8344647 100644 --- a/reports/css/d3-style.css +++ b/reports/css/d3-style.css @@ -25,8 +25,9 @@ fill: steelblue; .d3-tip { line-height: 1; - font-weight: bold; - padding: 12px; + border: 2px solid; + font-size: 12px; + padding: 6px; background: rgba(0, 0, 0, 0.9); color: #fff; border-radius: 2px; @@ -35,7 +36,7 @@ fill: steelblue; .d3-tip:after { box-sizing: border-box; display: inline; - font-size: 10px; + font-size: 8px; width: 100%; line-height: 1; color: rgba(0, 0, 0, 0.9); diff --git a/reports/index.html b/reports/index.html index 6b9b440..013d01a 100644 --- a/reports/index.html +++ b/reports/index.html @@ -55,6 +55,11 @@ +
+ + +
@@ -80,5 +85,6 @@ + diff --git a/reports/js/benchmarks.js b/reports/js/benchmarks.js index fb0bb6d..af9bc8a 100644 --- a/reports/js/benchmarks.js +++ b/reports/js/benchmarks.js @@ -142,6 +142,7 @@ function chartTypeSelect() else if (chartType == "highest-metric-comparison") { activeChartType = hmc; } else if (chartType == "sweep-runtime-comparison") { activeChartType = sc; } else if (chartType == "sweep-metric-comparison") { activeChartType = smc; } + else if (chartType == "all-sweep-metric-comparison") { activeChartType = apmc; } activeChartType.onTypeSelect(); } diff --git a/reports/js/benchmarks/all-params-metric-comparison-view.js b/reports/js/benchmarks/all-params-metric-comparison-view.js new file mode 100644 index 0000000..a4751be --- /dev/null +++ b/reports/js/benchmarks/all-params-metric-comparison-view.js @@ -0,0 +1,582 @@ +// Define namespace: apmc = all parameters metric comparison. +var apmc = apmc = apmc || {}; + +apmc.methodName = ""; +apmc.dataset = ""; +apmc.libraries = []; +apmc.activeLibraries = []; +apmc.results = []; +apmc.sweepId = -1; + +// This chart type has been selected. +apmc.onTypeSelect = function() +{ + // The user needs to be able to select a method, then parameters, then a + // dataset. + var selectHolder = d3.select(".selectholder"); + selectHolder.append("label") + .attr("for", "method_select") + .attr("class", "method-select-label") + .text("Select method:"); + selectHolder.append("select") + .attr("id", "method_select") + .attr("onchange", "apmc.methodSelect()"); + selectHolder.append("label") + .attr("for", "dataset_select") + .attr("class", "dataset-select-label") + .text("Select dataset:"); + selectHolder.append("select") + .attr("id", "dataset_select") + .attr("onchange", "apmc.datasetSelect()"); + selectHolder.append("label") + .attr("for", "metric_select") + .attr("class", "metric-select-label") + .text("Select metric:"); + selectHolder.append("select") + .attr("id", "metric_select") + .attr("onchange", "apmc.metricSelect()"); + + apmc.listMethods(); +} + +apmc.clear = function() +{ + apmc.clearChart(); +} + +// List the available methods where there is a sweep. +apmc.listMethods = function() +{ + var methods = dbExec("SELECT DISTINCT methods.name FROM methods, results " + + "WHERE methods.id = results.method_id ORDER BY name;"); + var methodSelectBox = document.getElementById("method_select"); + + // Remove old things. + clearSelectBox(methodSelectBox); + + // Add new things. + var length = dbType === "sqlite" ? methods[0].values.length : methods.length; + for (i = 0; i < length; i++) + { + var newOption = document.createElement("option"); + newOption.text = dbType === "sqlite" ? methods[0].values[i] : + methods[i].name; + methodSelectBox.add(newOption); + } + methodSelectBox.selectedIndex = -1; + + // Clear dataset box. + clearSelectBox(document.getElementById("dataset_select")); +} + +// Called when the user selects a method. +apmc.methodSelect = function() +{ + // Extract the name of the method we selected. + var methodSelectBox = document.getElementById("method_select"); + apmc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; // At higher scope. + + // Now we need to get the datasets. + var sqlstr = "SELECT DISTINCT datasets.name FROM datasets, results, methods " + + "WHERE datasets.id = results.dataset_id AND methods.name = '" + + apmc.methodName + "' AND results.method_id = methods.id;"; + var datasets = dbExec(sqlstr); + + // Loop through results and fill the second list box. + var datasetSelectBox = document.getElementById("dataset_select"); + clearSelectBox(datasetSelectBox); + + var newOption = document.createElement("option"); + datasetSelectBox.add(newOption); + + datasetSelectBox.selectedIndex = 0; + if ((dbType === "sqlite" && datasets[0]) || (dbType === "mysql" && datasets)) + { + // Put in the datasets. + var length = dbType === "sqlite" ? datasets[0].values.length : datasets.length; + for (i = 0; i < length; i++) + { + newOption = document.createElement("option"); + + var datasetName = dbType === "sqlite" ? datasets[0].values[i][0] : datasets[i].name; + newOption.text = datasetName; + datasetSelectBox.add(newOption); + } + } + + datasetSelectBox.selectedIndex = 0; +} + +apmc.datasetSelect = function() +{ + // The user selected a dataset, so now we can ask them to select the metric + // they want. + var methodSelectBox = document.getElementById("method_select"); + apmc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; + var datasetSelectBox = document.getElementById("dataset_select"); + var datasetName = + datasetSelectBox.options[datasetSelectBox.selectedIndex].text; + + // What metrics do we have available? We can do our actual query for results + // here, then we need to parse it. We want all of the most recent results. + var sqlstr = "SELECT metrics.metric, metrics.build_id, metrics.sweep_id, " + + "metrics.sweep_elem_id, libraries.name, methods.parameters " + + "FROM datasets, methods, metrics, libraries " + + "WHERE metrics.libary_id = libraries.id" + + " AND datasets.id = metrics.dataset_id" + + " AND datasets.name = '" + datasetName + "'" + + " AND methods.name = '" + apmc.methodName + "'" + + " AND metrics.method_id = methods.id;"; + apmc.results = dbExec(sqlstr); + apmc.results = dbType === "sqlite" ? apmc.results[0].values : apmc.results; + + // Also determine the maximum build number for a given library. + apmc.libraryVersions = {} + addLibrary = function(p, c) + { + var lib = (dbType === "sqlite") ? c[4] : c.name; + var params = (dbType === "sqlite") ? c[5] : c.parameters; + var bid = (dbType === "sqlite") ? c[1] : c.build_id; + + if (lib in apmc.libraryVersions) + { + if (params in apmc.libraryVersions[lib]) + { + if (apmc.libraryVersions[lib][params] < bid) + apmc.libraryVersions[lib][params] = bid; + } + else + { + apmc.libraryVersions[lib][params] = bid; + } + } + else + { + apmc.libraryVersions[lib] = {} + apmc.libraryVersions[lib][params] = bid; + } + + return; + } + apmc.results.reduce(addLibrary, []); + + // Filter out rows that weren't the newest. + apmc.results = apmc.results.filter(function(c) + { + return (((dbType === "sqlite") ? c[1] : c.build_id) == + apmc.libraryVersions[((dbType === "sqlite") ? c[4] : c.name)][((dbType === "sqlite") ? c[5] : c.params)]); + }); + + // Lastly, we need to gather all of the sweeps we might have. + apmc.sweeps = {} + addSweep = function(p, c) + { + var params = jQuery.parseJSON((dbType === "sqlite") ? c[5] : c.parameters); + for (e in params) + { + if (params[e].indexOf("sweep(") != -1) + { + // Do we already know about this sweep? + if (params[e] in apmc.sweeps) + continue; + + // Great, we have a sweep---now we need to get the information. + sqlstr = "SELECT type, begin, step, end FROM sweeps WHERE id = " + + ((dbType === "sqlite") ? c[2] : c.sweep_id) + ";"; + var sweepResults = dbExec(sqlstr); + // There should only be one result... + sweepResults = (dbType === "sqlite" ? sweepResults[0].values[0] : sweepResults[0]); + + // Insert these results into the dict. + apmc.sweeps[params[e]] = + { "type": (dbType === "sqlite" ? sweepResults[0] : sweepResults.type), + "begin": (dbType === "sqlite" ? sweepResults[1] : sweepResults.begin), + "step": (dbType === "sqlite" ? sweepResults[2] : sweepResults.step), + "end": (dbType === "sqlite" ? sweepResults[3] : sweepResults.end) }; + } + } + } + apmc.results.reduce(addSweep, []); + + // Now we have to parse through the metrics and see what we find. + addMetric = function(p, c) + { + var json = jQuery.parseJSON(dbType === "sqlite" ? c[0] : c.metric); + for(var k in json) + if(p.indexOf(k) < 0) + p.push(k); + return p; + }; + metrics = apmc.results.reduce(addMetric, []); + + var metric_select_box = document.getElementById("metric_select"); + clearSelectBox(metric_select_box); + for (i = 0; i < metrics.length; i++) + { + var new_option = document.createElement("option"); + new_option.text = metrics[i]; + metric_select_box.add(new_option); + } + metric_select_box.selectedIndex = -1; +} + +apmc.metricSelect = function() +{ + // We've already got the results, and now the user has specified the metric + // they want plotted. + var metricSelectBox = document.getElementById("metric_select"); + apmc.metricName = metricSelectBox.options[metricSelectBox.selectedIndex].text; + + // Obtain unique list of libraries. + apmc.libraries = apmc.results.map( + function(d) { + return dbType === "sqlite" ? (d[4] + ": " + d[5]) : (d.lib + ": " + d.parameters); + }).reduce( + function(p, c) { + if (p.indexOf(c) < 0) p.push(c); return p; + }, []); + + // By default, all libraries are active. + apmc.activeLibraries = {}; + for (i = 0; i < apmc.libraries.length; ++i) + { + apmc.activeLibraries[apmc.libraries[i]] = true; + } + + clearChart(); + buildChart(); +} + +apmc.clearChart = function() +{ + d3.select("svg").remove(); + d3.selectAll(".d3-tip").remove(); + d3.selectAll(".library-select-title").remove(); + d3.selectAll(".library_select_div").remove(); + d3.selectAll(".legendholder").selectAll("*").remove(); +} + +apmc.extractRuntime = function(d) +{ + var json = jQuery.parseJSON(d); + for (var m in json) + { + if (m == "Runtime") + { + if (json[m] == -2) + return "failure"; + else if (json[m] == -1) + return ">9000"; + else + return json[m]; + } + } + + return "failure"; +} + +apmc.extractMetric = function(d, metricName, notFoundValue) +{ + var json = jQuery.parseJSON(d); + for (var m in json) + if (m == metricName) + return json[m]; + + return notFoundValue; +} + +apmc.buildChart = function() +{ + // Get lists of active library/param combinations. + var activeLibraryList = apmc.libraries.map(function(d) { return d; }).reduce( + function(p, c) + { + if (apmc.activeLibraries[c] == true) + p.push(c); + return p; + }, []); + + var maxRuntime = d3.max(apmc.results, + function(d) + { + if (apmc.activeLibraries[dbType === "sqlite" ? (d[4] + ": " + d[5]) : (d.name + ": " + d.parameters)] == false) + { + return 0; + } + else + { + x = apmc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric); + if (x === "failure" || x === ">9000") + return 0; + else + return x; + } + }); + + var maxMetric = d3.max(apmc.results, + function(d) + { + if (apmc.activeLibraries[dbType === "sqlite" ? (d[4] + ": " + d[5]) : (d.name + ": " + d.parameters)] == false) + return 0; + else + return apmc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, + apmc.metricName, 0); + }); + + // Increase so we have 16 spare pixels at the top. + maxMetric *= ((height + 16) / height); + + var runtimeScale = d3.scale.linear() + .domain([0, maxRuntime]) + .range([0, width]); + + // We need to find out how big the sweep is. +// var sweepSql = "SELECT type, begin, step, end FROM sweeps where id = " + apmc.sweepId; +// var sweepResults = dbExec(sweepSql); +// sweepResults = dbType === "sqlite" ? sweepResults[0].values : sweepResults.results; + +// var func = (dbType === "sqlite" ? sweepResults[0][0] : sweepResults[0].type) === "int" ? parseInt : parseFloat; +// var start = func(dbType === "sqlite" ? sweepResults[0][1] : sweepResults[0].start); +// var step = func(dbType === "sqlite" ? sweepResults[0][2] : sweepResults[0].step); +// var end = func(dbType === "sqlite" ? sweepResults[0][3] : sweepResults[0].end); + + var metricScale = d3.scale.linear() + .domain([0, maxMetric]) + .range([height, 0]); + + var xAxis = d3.svg.axis().scale(runtimeScale).orient("bottom"); + var yAxis = d3.svg.axis().scale(metricScale).orient("left"); + + // Create svg object. + var svg = d3.select(".svgholder").append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + // Add x axis. + svg.append("g").attr("id", "xaxis") + .attr("class", "x axis") + .attr("transform", "translate(0, " + height + ")") + .call(xAxis) + .append("text") + .style("text-anchor", "end") + .attr("dx", 500) + .attr("dy", "3em") + .text("Runtime (s)"); + + // Add y axis. + svg.append("g").attr("id", "yaxis") + .attr("class", "y axis") + .call(yAxis) + .append("text") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text(apmc.metricName); + + // Create tooltips. + var tip = d3.tip() + .attr("class", "d3-tip") + .offset([-10, 0]) + .html(function(d) { + var runtime = apmc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric); + if (runtime != ">9000" && runtime != "failure") { + runtime = runtime.toFixed(3); + } + var metricValue = apmc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, apmc.metricName, ""); + metricValue = metricValue.toFixed(3); + + var libName = (dbType === "sqlite" ? d[4] : d.name); + var paramName = (dbType === "sqlite" ? d[5] : d.parameters); + var start = 0; + var step = 0; + // Goal: output, e.g., + // mlpack: + // non_sweep_param_1=x, non_sweep_param_2=x, sweep_param=x + // metric: + // runtime: + // All centered with a border in the desired color. + output = "" + libName + "
"; + // Loop over parameters. + params = jQuery.parseJSON(paramName); + for (p in params) + { + output += p + ": "; + // Is it a sweep option? + if (params[p].indexOf("sweep(") != -1) + { + // Get sweep element. + var sweepElemId = (dbType === "sqlite" ? d[3] : d.sweep_elem_id); + + var func = (apmc.sweeps[params[p]]['type'] === "int" ? parseInt : parseFloat); + var start = func(apmc.sweeps[params[p]]['begin']); + var step = func(apmc.sweeps[params[p]]['step']); + + output += (start + step * sweepElemId); + } + else + { + output += params[p]; + } + output += "; "; + } + output += "
"; + output += apmc.metricName + ": " + metricValue + "
"; + output += "Runtime: " + runtime + "s
"; + + return output; + }); + svg.call(tip); + + // Add all of the data points. + var lineFunc = d3.svg.line() + .x(function(d) { return runtimeScale(mapRuntime(apmc.extractRuntime( + dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .y(function(d) { return metricScale(apmc.extractMetric( + dbType === "sqlite" ? d[0] : d.metric, apmc.metricName, 0)); }) + .interpolate("linear"); + + // Add all of the data points. + for (var l in apmc.libraryVersions) + { + console.log("lib: " + l) + console.log(apmc.libraryVersions[l]); + for (var p in apmc.libraryVersions[l]) + { + if (!apmc.activeLibraries[l + ": " + p]) + continue; + + // Now we have a library/parameters combination. + // Collect all the data and let's see if we have a sweep or just a single + // point. + var res = apmc.results.filter(function(c) + { + var ln = (dbType === "sqlite") ? c[4] : c.name; + var pn = (dbType === "sqlite") ? c[5] : c.parameters; + return ((ln === l) && (pn === p)); + }); + + if (res.length > 0) + { + svg.append('svg:path') + .attr('d', lineFunc(res)) + .attr('stroke', color(l + ': ' + p)) + .attr('stroke-width', 2) + .attr('fill', 'none'); + + // Colored circle enclosed in white circle enclosed in background color + // circle; looks kind of nice. + svg.selectAll("dot").data(res).enter().append("circle") + .attr("r", 6) + .attr("cx", function(d) { return runtimeScale(mapRuntime(apmc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .attr("cy", function(d) { return metricScale(apmc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, apmc.metricName, 0)); }) + .attr('fill', '#222222') + .on('mouseover', function(d, i) + { + d3.select('.d3-tip').style("border-color", color(d[4] + ": " + d[5])); + tip.show(d, i); + }) + .on('mouseout', tip.hide); + svg.selectAll("dot").data(res).enter().append("circle") + .attr("r", 4) + .attr("cx", function(d) { return runtimeScale(mapRuntime(apmc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .attr("cy", function(d) { return metricScale(apmc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, apmc.metricName, 0)); }) + .attr('fill', '#ffffff') + .on('mouseover', function(d, i) + { + d3.select('.d3-tip').style("border-color", color(d[4] + ": " + d[5])); + tip.show(d, i); + }) + .on('mouseout', tip.hide); + svg.selectAll("dot").data(res).enter().append("circle") + .attr("r", 3) + .attr("cx", function(d) { return runtimeScale(mapRuntime(apmc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) + .attr("cy", function(d) { return metricScale(apmc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, apmc.metricName, 0)); }) + .attr('fill', function(d) { return color(d[4] + ": " + d[5]) }) + .on('mouseover', function(d, i) + { + d3.select('.d3-tip').style("border-color", color(d[4] + ": " + d[5])); + tip.show(d, i); + }) + .on('mouseout', tip.hide); + } + } + } + + // Create the library selector. + var librarySelectTitle = d3.select(".legendholder").append("div") + .attr("class", "library-select-title"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-text") + .text("Libraries:"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-open-paren") + .text("("); + librarySelectTitle.append("div") + .attr("class", "library-select-title-enable-all") + .text("enable all") + .on('click', function() { apmc.enableAllLibraries(); }); + librarySelectTitle.append("div") + .attr("class", "library-select-title-bar") + .text("|"); + librarySelectTitle.append("div") + .attr("class", "library-select-title-disable-all") + .text("disable all") + .on('click', function() { apmc.disableAllLibraries(); }); + librarySelectTitle.append("div") + .attr("class", "library-select-title-close-paren") + .text(")"); + + var libraryDivs = d3.select(".legendholder").selectAll("input") + .data(apmc.libraries) + .enter() + .append("div") + .attr("class", "library-select-div") + .attr("id", function(d) { return d + '-library-checkbox-div'; }); + + libraryDivs.append("label") + .attr('for', function(d) { return d + '-library-checkbox'; }) + .style('background', color) + .attr('class', 'library-select-color'); + + libraryDivs.append("input") + .property("checked", function(d) { return apmc.activeLibraries[d]; }) + .attr("type", "checkbox") + .attr("id", function(d) { return d + '-library-checkbox'; }) + .attr('class', 'library-select-box') + .attr("onClick", function(d, i) { return "apmc.toggleLibrary('" + d + "');"; }); + + libraryDivs.append("label") + .attr('for', function(d) { return d + '-library-checkbox'; }) + .attr('class', 'library-select-label') + .text(function(d) { return d; }); +} + +// Toggle a library to on or off. +apmc.toggleLibrary = function(library) +{ + apmc.activeLibraries[library] = !apmc.activeLibraries[library]; + + clearChart(); + buildChart(); +} + +// Set all libraries on. +apmc.enableAllLibraries = function() +{ + for (v in apmc.activeLibraries) { apmc.activeLibraries[v] = true; } + + clearChart(); + buildChart(); +} + +// Set all libraries off. +apmc.disableAllLibraries = function() +{ + for (v in apmc.activeLibraries) { apmc.activeLibraries[v] = false; } + + clearChart(); + buildChart(); +} diff --git a/reports/js/benchmarks/sweep-metric-comparison-view.js b/reports/js/benchmarks/sweep-metric-comparison-view.js index f7f87cf..6b16666 100644 --- a/reports/js/benchmarks/sweep-metric-comparison-view.js +++ b/reports/js/benchmarks/sweep-metric-comparison-view.js @@ -56,7 +56,7 @@ smc.clear = function() smc.listMethods = function() { var methods = dbExec("SELECT DISTINCT methods.name FROM methods, results " - + "WHERE methods.id = results.method_id AND methods.sweep_id != -1 " + + "WHERE methods.id = results.method_id AND results.sweep_id != -1 " + "ORDER BY name;"); var methodSelectBox = document.getElementById("method_select"); @@ -85,10 +85,10 @@ smc.methodSelect = function() var methodSelectBox = document.getElementById("method_select"); smc.methodName = methodSelectBox.options[methodSelectBox.selectedIndex].text; // At higher scope. - var sqlstr = "SELECT DISTINCT methods.parameters, methods.sweep_id, " + + var sqlstr = "SELECT DISTINCT methods.parameters, metrics.sweep_id, " + "metrics.libary_id, COUNT(DISTINCT metrics.libary_id) AS count FROM " + "methods, metrics WHERE methods.name = '" + smc.methodName + - "' AND methods.id = metrics.method_id AND methods.sweep_id != -1 " + + "' AND methods.id = metrics.method_id AND metrics.sweep_id != -1 " + "GROUP BY methods.parameters;"; var params = dbExec(sqlstr); @@ -145,7 +145,7 @@ smc.paramSelect = function() var sqlstr = "SELECT DISTINCT datasets.name FROM datasets, results, methods " + "WHERE datasets.id = results.dataset_id AND methods.name = '" - + smc.methodName + "' AND methods.sweep_id = " + sweepId + " AND " + + smc.methodName + "' AND results.sweep_id = " + sweepId + " AND " + "results.method_id = methods.id AND methods.parameters = '" + smc.paramName + "';"; var datasets = dbExec(sqlstr); @@ -399,7 +399,7 @@ smc.buildChart = function() runtime = runtime.toFixed(2); } var metricValue = smc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, smc.metricName, ""); - return "" + d[2] + "; " + name + ": " + (start + step * d[1]) + ": " + smc.metricName + " " + metricValue + ", " + runtime + "s"; }); + return "" + d[2] + "; " + name + ":
" + (start + step * d[1]) + ": " + smc.metricName + " " + metricValue + ", " + runtime + "s"; }); svg.call(tip); // Add all of the data points. From cf24b62e846ee606cfd4b9fc9ec1580332ff553a Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 21 Sep 2017 11:00:35 -0400 Subject: [PATCH 8/9] Better formatting of library names. --- reports/css/d3-style.css | 1 + .../all-params-metric-comparison-view.js | 39 +++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/reports/css/d3-style.css b/reports/css/d3-style.css index 8344647..124f4e1 100644 --- a/reports/css/d3-style.css +++ b/reports/css/d3-style.css @@ -70,6 +70,7 @@ svg text { } .library-select-div { + clear: both; min-width: 140px; float: left; height: 20px; diff --git a/reports/js/benchmarks/all-params-metric-comparison-view.js b/reports/js/benchmarks/all-params-metric-comparison-view.js index a4751be..308e814 100644 --- a/reports/js/benchmarks/all-params-metric-comparison-view.js +++ b/reports/js/benchmarks/all-params-metric-comparison-view.js @@ -44,6 +44,28 @@ apmc.clear = function() apmc.clearChart(); } +apmc.process_library_name = function(l, p) +{ + // We want to turn the library name and parameters dict into, e.g., + // mlpack: max_iterations=sweep(10, 10, 100), algorithm=lbfgs + var output = l; + var i = 0; + var d = jQuery.parseJSON(p); + if (Object.keys(d).length > 0) + { + output += ": "; + for (pp in d) + { + output += pp + "=" + d[pp]; + i++; + if (i != Object.keys(d).length) + output += ";"; + } + } + + return output; +} + // List the available methods where there is a sweep. apmc.listMethods = function() { @@ -230,11 +252,12 @@ apmc.metricSelect = function() // Obtain unique list of libraries. apmc.libraries = apmc.results.map( function(d) { - return dbType === "sqlite" ? (d[4] + ": " + d[5]) : (d.lib + ": " + d.parameters); + return dbType === "sqlite" ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.lib, d.parameters); }).reduce( function(p, c) { if (p.indexOf(c) < 0) p.push(c); return p; }, []); + console.log(apmc.libraries); // By default, all libraries are active. apmc.activeLibraries = {}; @@ -299,7 +322,7 @@ apmc.buildChart = function() var maxRuntime = d3.max(apmc.results, function(d) { - if (apmc.activeLibraries[dbType === "sqlite" ? (d[4] + ": " + d[5]) : (d.name + ": " + d.parameters)] == false) + if (apmc.activeLibraries[dbType === "sqlite" ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.name, d.parameters)] == false) { return 0; } @@ -316,7 +339,7 @@ apmc.buildChart = function() var maxMetric = d3.max(apmc.results, function(d) { - if (apmc.activeLibraries[dbType === "sqlite" ? (d[4] + ": " + d[5]) : (d.name + ": " + d.parameters)] == false) + if (apmc.activeLibraries[dbType === "sqlite" ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.name, d.parameters)] == false) return 0; else return apmc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, @@ -445,7 +468,7 @@ apmc.buildChart = function() console.log(apmc.libraryVersions[l]); for (var p in apmc.libraryVersions[l]) { - if (!apmc.activeLibraries[l + ": " + p]) + if (!apmc.activeLibraries[apmc.process_library_name(l, p)]) continue; // Now we have a library/parameters combination. @@ -475,7 +498,7 @@ apmc.buildChart = function() .attr('fill', '#222222') .on('mouseover', function(d, i) { - d3.select('.d3-tip').style("border-color", color(d[4] + ": " + d[5])); + d3.select('.d3-tip').style("border-color", color((dbType === "sqlite") ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.name, d.parameters))); tip.show(d, i); }) .on('mouseout', tip.hide); @@ -486,7 +509,7 @@ apmc.buildChart = function() .attr('fill', '#ffffff') .on('mouseover', function(d, i) { - d3.select('.d3-tip').style("border-color", color(d[4] + ": " + d[5])); + d3.select('.d3-tip').style("border-color", color((dbType === "sqlite") ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.name, d.parameters))); tip.show(d, i); }) .on('mouseout', tip.hide); @@ -494,10 +517,10 @@ apmc.buildChart = function() .attr("r", 3) .attr("cx", function(d) { return runtimeScale(mapRuntime(apmc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric), maxRuntime)); }) .attr("cy", function(d) { return metricScale(apmc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, apmc.metricName, 0)); }) - .attr('fill', function(d) { return color(d[4] + ": " + d[5]) }) + .attr('fill', function(d) { return color((dbType === "sqlite") ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.name, d.parameters)) }) .on('mouseover', function(d, i) { - d3.select('.d3-tip').style("border-color", color(d[4] + ": " + d[5])); + d3.select('.d3-tip').style("border-color", color((dbType === "sqlite") ? apmc.process_library_name(d[4], d[5]) : apmc.process_library_name(d.name, d.parameters))); tip.show(d, i); }) .on('mouseout', tip.hide); From 79ca3c26a010f5d2c1a8bb0c1eb5bd2ea7455c2f Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 9 Oct 2017 14:59:27 -0400 Subject: [PATCH 9/9] Updates to views. --- reports/js/benchmarks.js | 3 +- .../all-params-metric-comparison-view.js | 37 +++++++++++++------ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/reports/js/benchmarks.js b/reports/js/benchmarks.js index af9bc8a..47e36ee 100644 --- a/reports/js/benchmarks.js +++ b/reports/js/benchmarks.js @@ -74,7 +74,8 @@ var color = d3.scale.ordinal().range(["#98abc5", "#8a89a6", "#7b6888", "#6b486b" */ function mapRuntime(runtime, max) { - if (runtime == ">9000") { return max; } + if (String(runtime).indexOf(">") != -1) { return max; } + else if (runtime == "timeout") { return max; } else if (runtime == "failure") { return 0; } else { return runtime; } } diff --git a/reports/js/benchmarks/all-params-metric-comparison-view.js b/reports/js/benchmarks/all-params-metric-comparison-view.js index 308e814..b3b8f26 100644 --- a/reports/js/benchmarks/all-params-metric-comparison-view.js +++ b/reports/js/benchmarks/all-params-metric-comparison-view.js @@ -196,10 +196,10 @@ apmc.datasetSelect = function() var params = jQuery.parseJSON((dbType === "sqlite") ? c[5] : c.parameters); for (e in params) { - if (params[e].indexOf("sweep(") != -1) + if (String(params[e]).indexOf("sweep(") != -1) { // Do we already know about this sweep? - if (params[e] in apmc.sweeps) + if (String(params[e]) in apmc.sweeps) continue; // Great, we have a sweep---now we need to get the information. @@ -210,7 +210,7 @@ apmc.datasetSelect = function() sweepResults = (dbType === "sqlite" ? sweepResults[0].values[0] : sweepResults[0]); // Insert these results into the dict. - apmc.sweeps[params[e]] = + apmc.sweeps[String(params[e])] = { "type": (dbType === "sqlite" ? sweepResults[0] : sweepResults.type), "begin": (dbType === "sqlite" ? sweepResults[1] : sweepResults.begin), "step": (dbType === "sqlite" ? sweepResults[2] : sweepResults.step), @@ -289,7 +289,9 @@ apmc.extractRuntime = function(d) if (json[m] == -2) return "failure"; else if (json[m] == -1) - return ">9000"; + return "timeout"; + else if (String(json[m]).indexOf(">") != -1) + return "timeout"; else return json[m]; } @@ -329,7 +331,7 @@ apmc.buildChart = function() else { x = apmc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric); - if (x === "failure" || x === ">9000") + if (x === "failure" || x === "timeout") return 0; else return x; @@ -349,6 +351,7 @@ apmc.buildChart = function() // Increase so we have 16 spare pixels at the top. maxMetric *= ((height + 16) / height); + console.log(maxRuntime); var runtimeScale = d3.scale.linear() .domain([0, maxRuntime]) .range([0, width]); @@ -405,11 +408,19 @@ apmc.buildChart = function() .offset([-10, 0]) .html(function(d) { var runtime = apmc.extractRuntime(dbType === "sqlite" ? d[0] : d.metric); - if (runtime != ">9000" && runtime != "failure") { + if (runtime != "timeout" && runtime != "failure") { runtime = runtime.toFixed(3); } var metricValue = apmc.extractMetric(dbType === "sqlite" ? d[0] : d.metric, apmc.metricName, ""); - metricValue = metricValue.toFixed(3); + if (metricValue != "timeout" && metricValue != "failure" && +String(metricValue).indexOf(">") == -1 && metricValue != "") + { + metricValue = metricValue.toFixed(3); + } + else if (metricValue == "") + { + metricValue = "failure"; + } var libName = (dbType === "sqlite" ? d[4] : d.name); var paramName = (dbType === "sqlite" ? d[5] : d.parameters); @@ -428,7 +439,7 @@ apmc.buildChart = function() { output += p + ": "; // Is it a sweep option? - if (params[p].indexOf("sweep(") != -1) + if (String(params[p]).indexOf("sweep(") != -1) { // Get sweep element. var sweepElemId = (dbType === "sqlite" ? d[3] : d.sweep_elem_id); @@ -441,13 +452,17 @@ apmc.buildChart = function() } else { - output += params[p]; + output += String(params[p]); } output += "; "; } output += "
"; output += apmc.metricName + ": " + metricValue + "
"; - output += "Runtime: " + runtime + "s
"; + output += "Runtime: " + runtime; + if (runtime != "failure" && runtime != "timeout") + { + output += "s
"; + } return output; }); @@ -464,8 +479,6 @@ apmc.buildChart = function() // Add all of the data points. for (var l in apmc.libraryVersions) { - console.log("lib: " + l) - console.log(apmc.libraryVersions[l]); for (var p in apmc.libraryVersions[l]) { if (!apmc.activeLibraries[apmc.process_library_name(l, p)])