diff --git a/Criterion/Analysis.hs b/Criterion/Analysis.hs index cbc0454f..8f45716d 100644 --- a/Criterion/Analysis.hs +++ b/Criterion/Analysis.hs @@ -89,9 +89,9 @@ outlierVariance outlierVariance µ σ a = OutlierVariance effect desc varOutMin where ( effect, desc ) | varOutMin < 0.01 = (Unaffected, "no") - | varOutMin < 0.1 = (Slight, "slight") - | varOutMin < 0.5 = (Moderate, "moderate") - | otherwise = (Severe, "severe") + | varOutMin < 0.1 = (Slight, "a slight") + | varOutMin < 0.5 = (Moderate, "a moderate") + | otherwise = (Severe, "a severe") varOutMin = (minBy varOut 1 (minBy cMax 0 µgMin)) / σb2 varOut c = (ac / a) * (σb2 - ac * σg2) where ac = a - c σb = B.estPoint σ diff --git a/Criterion/EmbeddedData.hs b/Criterion/EmbeddedData.hs index b6e85822..1a4dfa5c 100644 --- a/Criterion/EmbeddedData.hs +++ b/Criterion/EmbeddedData.hs @@ -11,27 +11,19 @@ -- -- When the @embed-data-files@ @Cabal@ flag is enabled, this module exports -- the contents of various files (the @data-files@ from @criterion.cabal@, as --- well as minimized versions of jQuery and Flot) embedded as 'ByteString's. +-- well as a minimized version of Chart.js) embedded as a 'ByteString'. module Criterion.EmbeddedData ( dataFiles - , jQueryContents - , flotContents - , flotErrorbarsContents - , flotNavigateContents + , chartContents ) where import Data.ByteString (ByteString) import Data.FileEmbed (embedDir, embedFile) import Language.Haskell.TH.Syntax (runIO) -import qualified Language.Javascript.Flot as Flot -import qualified Language.Javascript.JQuery as JQuery +import qualified Language.Javascript.Chart as Chart dataFiles :: [(FilePath, ByteString)] dataFiles = $(embedDir "templates") -jQueryContents, flotContents, - flotErrorbarsContents, flotNavigateContents :: ByteString -jQueryContents = $(embedFile =<< runIO JQuery.file) -flotContents = $(embedFile =<< runIO (Flot.file Flot.Flot)) -flotErrorbarsContents = $(embedFile =<< runIO (Flot.file Flot.FlotErrorbars)) -flotNavigateContents = $(embedFile =<< runIO (Flot.file Flot.FlotNavigate)) +chartContents :: ByteString +chartContents = $(embedFile =<< runIO (Chart.file Chart.Chart)) diff --git a/Criterion/Report.hs b/Criterion/Report.hs index d85775d7..833bca78 100644 --- a/Criterion/Report.hs +++ b/Criterion/Report.hs @@ -37,13 +37,12 @@ import Control.Monad.IO.Class (MonadIO(liftIO)) import Control.Monad.Reader (ask) import Criterion.Monad (Criterion) import Criterion.Types -import Data.Aeson (ToJSON (..), Value(..), object, (.=), Value, encode) +import Data.Aeson (ToJSON (..), Value(..), object, (.=), Value) import Data.Data (Data, Typeable) import Data.Foldable (forM_) import GHC.Generics (Generic) import Paths_criterion (getDataFileName) import Statistics.Function (minMax) -import Statistics.Types (confidenceInterval, confidenceLevel, confIntCL, estError) import System.Directory (doesFileExist) import System.FilePath ((), (<.>), isPathSeparator) import System.IO (hPutStrLn, stderr) @@ -53,7 +52,9 @@ import Prelude () import Prelude.Compat import qualified Control.Exception as E import qualified Data.Text as T +#if defined(EMBED) import qualified Data.Text.Lazy.Encoding as TLE +#endif import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.IO as TL @@ -61,13 +62,11 @@ import qualified Data.Vector.Generic as G import qualified Data.Vector.Unboxed as U #if defined(EMBED) -import Criterion.EmbeddedData (dataFiles, jQueryContents, flotContents, - flotErrorbarsContents, flotNavigateContents) +import Criterion.EmbeddedData (dataFiles, chartContents) import qualified Data.ByteString.Lazy as BL import qualified Data.Text.Encoding as TE #else -import qualified Language.Javascript.Flot as Flot -import qualified Language.Javascript.JQuery as JQuery +import qualified Language.Javascript.Chart as Chart #endif -- | Trim long flat tails from a KDE plot. @@ -113,26 +112,18 @@ formatReport reports templateName = do Left err -> fail (show err) -- TODO: throw a template exception? Right x -> return x - jQuery <- jQueryFileContents - flot <- flotFileContents - flotErrorbars <- flotErrorbarsFileContents - flotNavigate <- flotNavigateFileContents - jQueryCriterionJS <- readDataFile ("js" "jquery.criterion.js") - criterionCSS <- readDataFile "criterion.css" + criterionJS <- readDataFile "criterion.js" + criterionCSS <- readDataFile "criterion.css" + chartJS <- chartFileContents -- includes, only top level templates <- getTemplateDir template <- includeTemplate (includeFile [templates]) template0 - reports' <- mapM inner reports let context = object [ "json" .= reports - , "report" .= reports' - , "js-jquery" .= jQuery - , "js-flot" .= flot - , "js-flot-errorbars" .= flotErrorbars - , "js-flot-navigate" .= flotNavigate - , "jquery-criterion-js" .= jQueryCriterionJS + , "js-criterion" .= criterionJS + , "js-chart" .= chartJS , "criterion-css" .= criterionCSS ] @@ -152,17 +143,11 @@ formatReport reports templateName = do criterionWarning $ displayMustacheWarning warning return formatted where - jQueryFileContents, flotFileContents :: IO T.Text + chartFileContents :: IO T.Text #if defined(EMBED) - jQueryFileContents = pure $ TE.decodeUtf8 jQueryContents - flotFileContents = pure $ TE.decodeUtf8 flotContents - flotErrorbarsFileContents = pure $ TE.decodeUtf8 flotErrorbarsContents - flotNavigateFileContents = pure $ TE.decodeUtf8 flotNavigateContents + chartFileContents = pure $ TE.decodeUtf8 chartContents #else - jQueryFileContents = T.readFile =<< JQuery.file - flotFileContents = T.readFile =<< Flot.file Flot.Flot - flotErrorbarsFileContents = T.readFile =<< Flot.file Flot.FlotErrorbars - flotNavigateFileContents = T.readFile =<< Flot.file Flot.FlotNavigate + chartFileContents = T.readFile =<< Chart.file Chart.Chart #endif readDataFile :: FilePath -> IO T.Text @@ -185,58 +170,6 @@ formatReport reports templateName = do fmap TextBlock (f (T.unpack fp)) includeNode _ n = return n - -- Merge Report with it's analysis and outliers - merge :: ToJSON a => a -> Value -> Value - merge x y = case toJSON x of - Object x' -> case y of - Object y' -> Object (x' <> y') - _ -> y - _ -> y - - inner :: Report -> IO Value - inner r@Report {..} = do - reportName' <- sanitizeJSString $ T.pack reportName - return $ merge reportAnalysis $ merge reportOutliers $ object - [ "name" .= reportName' - , "json" .= TLE.decodeUtf8 (encode r) - , "number" .= reportNumber - , "iters" .= vector "x" iters - , "times" .= vector "x" times - , "cycles" .= vector "x" cycles - , "kdetimes" .= vector "x" kdeValues - , "kdepdf" .= vector "x" kdePDF - , "kde" .= vector2 "time" "pdf" kdeValues kdePDF - , "anMeanConfidenceLevel" .= anMeanConfidenceLevel - , "anMeanLowerBound" .= anMeanLowerBound - , "anMeanUpperBound" .= anMeanUpperBound - , "anStdDevLowerBound" .= anStdDevLowerBound - , "anStdDevUpperBound" .= anStdDevUpperBound - ] - where - [KDE{..}] = reportKDEs - SampleAnalysis{..} = reportAnalysis - - iters = measure measIters reportMeasured - times = measure measTime reportMeasured - cycles = measure measCycles reportMeasured - anMeanConfidenceLevel - = confidenceLevel $ confIntCL $ estError anMean - (anMeanLowerBound, anMeanUpperBound) - = confidenceInterval anMean - (anStdDevLowerBound, anStdDevUpperBound) - = confidenceInterval anStdDev - - sanitizeJSString :: T.Text -> IO T.Text - sanitizeJSString str = do - let pieces = T.splitOn "\n" str - case pieces of - (_word1:_word2:_) -> do - criterionWarning $ - "Report name " ++ show str ++ " contains newlines, which " ++ - "will be replaced with spaces in the HTML report." - return $ T.unwords pieces - _ -> return str - criterionWarning :: String -> IO () criterionWarning msg = hPutStrLn stderr $ unlines diff --git a/changelog.md b/changelog.md index 24f7c296..38553797 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,23 @@ +Unreleased + +* The HTML reports have been reworked. + + * The `flot` plotting library (`js-flot` on Hackage) has been replaced by + `Chart.js` (`js-chart`). + * Most practical changes focus on improving the functionality of the overview + chart: + * It now supports logarithmic scale (#213). The scale can be toggled by + clicking the x-axis. + * Manual zooming has been replaced by clicking to focus a single bar. + * It now supports a variety of sort orders. + * The legend can now be toggled on/off and is hidden by default. + * Clicking the name of a group in the legend shows/hides all bars in that + group. + * The regression line on the scatter plot shows confidence interval. + * Better support for mobile and print. + * JSON escaping has been made more robust by no longer directly injecting + reports as JavaScript code. + 1.5.7.0 * Warn if an HTML report name contains newlines, and replace newlines with diff --git a/criterion.cabal b/criterion.cabal index be571ff9..3f993999 100644 --- a/criterion.cabal +++ b/criterion.cabal @@ -33,7 +33,7 @@ tested-with: data-files: templates/*.css templates/*.tpl - templates/js/jquery.criterion.js + templates/*.js description: This library provides a powerful but simple way to measure software @@ -97,8 +97,7 @@ library filepath, Glob >= 0.7.2, microstache >= 1.0.1 && < 1.1, - js-flot, - js-jquery, + js-chart >= 2.9.4 && < 3, mtl >= 2, mwc-random >= 0.8.0.3, optparse-applicative >= 0.13, diff --git a/stack.yaml b/stack.yaml index 7bb40178..ce432e9d 100644 --- a/stack.yaml +++ b/stack.yaml @@ -7,5 +7,6 @@ packages: extra-deps: - microstache-1.0.1.1 - statistics-0.14.0.0 +- js-chart-2.9.4.1 flags: {} diff --git a/templates/criterion.css b/templates/criterion.css index 11e22cab..705af318 100644 --- a/templates/criterion.css +++ b/templates/criterion.css @@ -1,102 +1,143 @@ -html, body { - height: 100%; - margin: 0; +html,body { + padding: 0; margin: 0; + font-family: sans-serif; } - -#wrap { - min-height: 100%; +* { + -webkit-tap-highlight-color: transparent; } - -#main { - overflow: auto; - padding-bottom: 180px; /* must be same height as the footer */ +div.scatter, div.kde { + position: relative; + display: inline-block; + box-sizing: border-box; + width: 50%; + padding: 0 2em; +} +.content, .explanation { + margin: auto; + max-width: 1000px; + padding: 0 20px; } -#footer { - position: relative; - margin-top: -180px; /* negative value of footer height */ - height: 180px; - clear: both; - background: #888; - margin: 40px 0 0; - color: white; - font-size: larger; - font-weight: 300; +#legend-toggle { + cursor: pointer; } -body:before { - /* Opera fix */ - content: ""; - height: 100%; - float: left; - width: 0; - margin-top: -32767px; +.overview-info { + float:right; } -body { - font: 14px Helvetica Neue; - text-rendering: optimizeLegibility; - margin-top: 1em; +.overview-info a { + display: inline-block; + margin-left: 10px; +} +.overview-info .info { + font-size: 120%; + font-weight: 400; + vertical-align: middle; + line-height: 1em; +} +.chevron { + position:relative; + color: black; + display:block; + transition-property: transform; + transition-duration: 400ms; + line-height: 1em; + font-size: 180%; +} +.chevron.right { + transform: scale(-1,1); +} +.chevron::before { + vertical-align: middle; + content:"\2039"; } -a:link { - color: steelblue; - text-decoration: none; +select { + outline: none; + border:none; + background: transparent; } -a:visited { - color: #4a743b; - text-decoration: none; +footer .content { + padding: 0; } -#footer a { - color: white; - text-decoration: underline; +span#explain-interactivity { + display-block: inline; + float: right; + color: #444; + font-size: 0.7em; } -.hover { - color: steelblue; +@media screen and (max-width: 800px) { + div.scatter, div.kde { + width: 100%; + display: block; + } + .report-details .outliers { + margin: auto; + } + .report-details table { + margin: auto; + } +} +table.analysis .low, table.analysis .high { + opacity: 0.5; +} +.report-details { + margin: 2em 0; + page-break-inside: avoid; +} +a, a:hover, a:visited, a:active { text-decoration: none; + color: #309ef2; } - -.body { - width: 960px; - margin: auto; +h1.title { + font-weight: 600; } - -.footfirst { - position: relative; - top: 30px; +h1 { + font-weight: 400; } - -th { - font-weight: 500; - opacity: 0.8; +#overview-chart { + width: 100%; /*height is determined by number of rows in JavaScript */ +} +footer { + background: #777777; + color: #ffffff; + padding: 20px; +} +footer a, footer a:hover, footer a:visited, footer a:active { + text-decoration: underline; + color: #fff; } -th.cibound { - opacity: 0.4; +.explanation { + margin-top: 3em; } -.confinterval { - opacity: 0.5; +.explanation h1 { + font-size: 2.6em; } -h1 { - font-size: 36px; - font-weight: 300; - margin-bottom: .3em; +#grokularation.explanation li { + margin: 1em 0; } -h2 { - font-size: 30px; - font-weight: 300; - margin-bottom: .3em; +#controls-explanation.explanation em { + font-weight: 600; + font-style: normal; } -.meanlegend { - color: #404040; - background-color: #ffffff; - opacity: 0.6; - font-size: smaller; +@media print { + footer { + background: transparent; + color: black; + } + footer a, footer a:hover, footer a:visited, footer a:active { + color: #309ef2; + } + .no-print { + display: none; + } } diff --git a/templates/criterion.js b/templates/criterion.js new file mode 100644 index 00000000..b127edee --- /dev/null +++ b/templates/criterion.js @@ -0,0 +1,870 @@ +(function() { + 'use strict'; + window.addEventListener('beforeprint', function() { + for (var id in Chart.instances) { + Chart.instances[id].resize(); + } + }, false); + + var errorBarPlugin = (function () { + function drawErrorBar(chart, ctx, low, high, y, height, color) { + ctx.save(); + ctx.lineWidth = 3; + ctx.strokeStyle = color; + var area = chart.chartArea; + ctx.rect(area.left, area.top, area.right - area.left, area.bottom - area.top); + ctx.clip(); + ctx.beginPath(); + ctx.moveTo(low, y - height); + ctx.lineTo(low, y + height); + ctx.moveTo(low, y); + ctx.lineTo(high, y); + ctx.moveTo(high, y - height); + ctx.lineTo(high, y + height); + ctx.stroke(); + ctx.restore(); + } + // Avoid sudden jumps in error bars when switching + // between linear and logarithmic scale + function conservativeError(vx, mx, now, final, scale) { + var finalDiff = Math.abs(mx - final); + var diff = Math.abs(vx - now); + return (diff > finalDiff) ? vx + scale * finalDiff : now; + } + return { + afterDatasetDraw: function(chart, easingOptions) { + var ctx = chart.ctx; + var easing = easingOptions.easingValue; + chart.data.datasets.forEach(function(d, i) { + var bars = chart.getDatasetMeta(i).data; + var axis = chart.scales[chart.options.scales.xAxes[0].id]; + bars.forEach(function(b, j) { + var value = axis.getValueForPixel(b._view.x); + var final = axis.getValueForPixel(b._model.x); + var errorBar = d.errorBars[j]; + var low = axis.getPixelForValue(value - errorBar.minus); + var high = axis.getPixelForValue(value + errorBar.plus); + var finalLow = axis.getPixelForValue(final - errorBar.minus); + var finalHigh = axis.getPixelForValue(final + errorBar.plus); + var l = easing === 1 ? finalLow : + conservativeError(b._view.x, b._model.x, low, + finalLow, -1.0); + var h = easing === 1 ? finalHigh : + conservativeError(b._view.x, b._model.x, + high, finalHigh, 1.0); + drawErrorBar(chart, ctx, l, h, b._view.y, 4, errorBar.color); + }); + }); + }, + }; + })(); + + // Formats the ticks on the X-axis on the scatter plot + var iterFormatter = function() { + var denom = 0; + return function(iters, index, values) { + if (iters == 0) { + return ''; + } + if (index == values.length - 1) { + return ''; + } + var power; + if (iters >= 1e9) { + denom = 1e9; + power = '⁹'; + } else if (iters >= 1e6) { + denom = 1e6; + power = '⁶'; + } else if (iters >= 1e3) { + denom = 1e3; + power = '³'; + } else { + denom = 1; + } + if (denom > 1) { + var value = (iters / denom).toFixed(); + return String(value) + '×10' + power; + } else { + return String(iters); + } + }; + }; + + var colors = ["#edc240", "#afd8f8", "#cb4b4b", "#4da74d", "#9440ed"]; + var errorColors = ["#cda220", "#8fb8d8", "#ab2b2b", "#2d872d", "#7420cd"]; + + + // Positions tooltips at cursor. Required for overview since the bars may + // extend past the canvas width. + Chart.Tooltip.positioners.cursor = function(_elems, position) { + return position; + } + + function axisType(logaxis) { + return logaxis ? 'logarithmic' : 'linear'; + } + + function reportSort(a, b) { + return a.reportNumber - b.reportNumber; + } + + // adds groupNumber and group fields to reports; + // returns list of list of reports, grouped by group + function groupReports(reports) { + + function reportGroup(report) { + var parts = report.groups.slice(); + parts.pop(); + return parts.join('/'); + } + + var groups = []; + reports.forEach(function(report) { + report.group = reportGroup(report); + if (groups.length === 0) { + groups.push([report]); + } else { + var prevGroup = groups[groups.length - 1]; + var prevGroupName = prevGroup[0].group; + if (prevGroupName === report.group) { + prevGroup.push(report); + } else { + groups.push([report]); + } + } + report.groupNumber = groups.length - 1; + }); + return groups; + } + + // compares 2 arrays lexicographically + function lex(aParts, bParts) { + for(var i = 0; i < aParts.length && i < bParts.length; i++) { + var x = aParts[i]; + var y = bParts[i]; + if (x < y) { + return -1; + } + if (y < x) { + return 1; + } + } + return aParts.length - bParts.length; + } + function lexicalSort(a, b) { + return lex(a.groups, b.groups); + } + + function reverseLexicalSort(a, b) { + return lex(a.groups.slice().reverse(), b.groups.slice().reverse()); + } + + function durationSort(a, b) { + return a.reportAnalysis.anMean.estPoint - b.reportAnalysis.anMean.estPoint; + } + function reverseDurationSort(a,b) { + return -durationSort(a,b); + } + + function timeUnits(secs) { + if (secs < 0) + return timeUnits(-secs); + else if (secs >= 1e9) + return [1e-9, "Gs"]; + else if (secs >= 1e6) + return [1e-6, "Ms"]; + else if (secs >= 1) + return [1, "s"]; + else if (secs >= 1e-3) + return [1e3, "ms"]; + else if (secs >= 1e-6) + return [1e6, "\u03bcs"]; + else if (secs >= 1e-9) + return [1e9, "ns"]; + else if (secs >= 1e-12) + return [1e12, "ps"]; + return [1, "s"]; + } + + function formatUnit(raw, unit, precision) { + var v = precision ? raw.toPrecision(precision) : Math.round(raw); + var label = String(v) + ' ' + unit; + return label; + } + + function formatTime(value, precision) { + var units = timeUnits(value); + var scale = units[0]; + return formatUnit(value * scale, units[1], precision); + } + + // pure function that produces the 'data' object of the overview chart + function overviewData(state, reports) { + var order = state.order; + var sorter = order === 'report-index' ? reportSort + : order === 'lex' ? lexicalSort + : order === 'colex' ? reverseLexicalSort + : order === 'duration' ? durationSort + : order === 'rev-duration' ? reverseDurationSort + : reportSort; + var sortedReports = reports.filter(function(report) { + return !state.hidden[report.groupNumber]; + }).slice().sort(sorter); + var data = sortedReports.map(function(report) { + return report.reportAnalysis.anMean.estPoint; + }); + var labels = sortedReports.map(function(report) { + return report.groups.join(' / '); + }); + var upperBound = function(report) { + var est = report.reportAnalysis.anMean; + return est.estPoint + est.estError.confIntUDX; + }; + var errorBars = sortedReports.map(function(report) { + var est = report.reportAnalysis.anMean; + return { + minus: est.estError.confIntLDX, + plus: est.estError.confIntUDX, + color: errorColors[report.groupNumber % errorColors.length] + }; + }); + var top = sortedReports.map(upperBound).reduce(function(a, b) { + return Math.max(a, b); + }, 0); + var scale = top; + if(state.activeReport !== null) { + reports.forEach(function(report) { + if(report.reportNumber === state.activeReport) { + scale = upperBound(report); + } + }); + } + + return { + labels: labels, + top: top, + max: scale * 1.1, + reports: sortedReports, + datasets: [{ + borderWidth: 1, + backgroundColor: sortedReports.map(function(report) { + var active = report.reportNumber === state.activeReport; + var alpha = active ? 'ff' : 'a0'; + var color = colors[report.groupNumber % colors.length] + alpha; + if (active) { + return Chart.helpers.getHoverColor(color); + } else { + return color; + } + }), + barThickness: 16, + barPercentage: 0.8, + data: data, + errorBars: errorBars, + minBarLength: 2, + }] + }; + } + + function inside(box, point) { + return (point.x >= box.left && point.x <= box.right && point.y >= box.top && + point.y <= box.bottom); + } + + function overviewHover(event, elems) { + var chart = this; + var xAxis = chart.scales[chart.options.scales.xAxes[0].id]; + var yAxis = chart.scales[chart.options.scales.yAxes[0].id]; + var point = Chart.helpers.getRelativePosition(event, chart); + var over = + (inside(xAxis, point) || inside(yAxis, point) || elems.length > 0); + if (over) { + chart.canvas.style.cursor = "pointer"; + } else { + chart.canvas.style.cursor = "default"; + } + } + + // Re-renders the overview after clicking/sorting + function renderOverview(state, reports, chart) { + var data = overviewData(state, reports); + var xaxis = chart.options.scales.xAxes[0]; + xaxis.ticks.max = data.max; + chart.config.data.datasets[0].backgroundColor = data.datasets[0].backgroundColor; + chart.config.data.datasets[0].errorBars = data.datasets[0].errorBars; + chart.config.data.datasets[0].data = data.datasets[0].data; + chart.options.scales.xAxes[0].type = axisType(state.logaxis); + chart.options.legend.display = state.legend; + chart.data.labels = data.labels; + chart.update(); + } + + function overviewClick(state, reports) { + return function(event, elems) { + var chart = this; + var xAxis = chart.scales[chart.options.scales.xAxes[0].id]; + var yAxis = chart.scales[chart.options.scales.yAxes[0].id]; + var point = Chart.helpers.getRelativePosition(event, chart); + var sorted = overviewData(state, reports).reports; + + function activateBar(index) { + // Trying to activate active bar disables instead + if (sorted[index].reportNumber === state.activeReport) { + state.activeReport = null; + } else { + state.activeReport = sorted[index].reportNumber; + } + } + + if (inside(xAxis, point)) { + state.activeReport = null; + state.logaxis = !state.logaxis; + renderOverview(state, reports, chart); + } else if (inside(yAxis, point)) { + var index = yAxis.getValueForPixel(point.y); + activateBar(index); + renderOverview(state, reports, chart); + } else if (elems.length > 0) { + var elem = elems[0]; + var index = elem._index; + activateBar(index); + state.logaxis = false; + renderOverview(state, reports, chart); + } else if(inside(chart.chartArea, point)) { + state.activeReport = null; + renderOverview(state, reports, chart); + } + }; + } + + // listener for sort drop-down + function overviewSort(state, reports, chart) { + return function(event) { + state.order = event.currentTarget.value; + renderOverview(state, reports, chart); + }; + } + + // Returns a formatter for the ticks on the X-axis of the overview + function overviewTick(state) { + return function(value, index, values) { + var label = formatTime(value); + if (state.logaxis) { + const remain = Math.round(value / + (Math.pow(10, Math.floor(Chart.helpers.log10(value))))); + if (index === values.length - 1) { + // Draw endpoint if we don't span a full order of magnitude + if (values[index] / values[1] < 10) { + return label; + } else { + return ''; + } + } + if (remain === 1) { + return label; + } + return ''; + } else { + // Don't show the right endpoint + if (index === values.length - 1) { + return ''; + } + return label; + } + } + } + + function mkOverview(reports) { + var canvas = document.createElement('canvas'); + + var state = { + logaxis: false, + activeReport: null, + order: 'index', + hidden: {}, + legend: false, + }; + + + var data = overviewData(state, reports); + var chart = new Chart(canvas.getContext('2d'), { + type: 'horizontalBar', + data: data, + plugins: [errorBarPlugin], + options: { + onHover: overviewHover, + onClick: overviewClick(state, reports), + onResize: function(chart, size) { + if (size.width < 800) { + chart.options.scales.yAxes[0].ticks.mirror = true; + chart.options.scales.yAxes[0].ticks.padding = -10; + chart.options.scales.yAxes[0].ticks.fontColor = '#000'; + } else { + chart.options.scales.yAxes[0].ticks.fontColor = '#666'; + chart.options.scales.yAxes[0].ticks.mirror = false; + chart.options.scales.yAxes[0].ticks.padding = 0; + } + }, + elements: { + rectangle: { + borderWidth: 2, + }, + }, + scales: { + yAxes: [{ + ticks: { + // make sure we draw the ticks above the error bars + z: 2, + } + }], + xAxes: [{ + display: true, + type: axisType(state.logaxis), + ticks: { + autoSkip: false, + min: 0, + max: data.top * 1.1, + minRotation: 0, + maxRotation: 0, + callback: overviewTick(state), + } + }] + }, + responsive: true, + maintainAspectRatio: false, + legend: { + display: state.legend, + position: 'right', + onLeave: function() { + chart.canvas.style.cursor = 'default'; + }, + onHover: function() { + chart.canvas.style.cursor = 'pointer'; + }, + onClick: function(_event, item) { + // toggle hidden + state.hidden[item.groupNumber] = !state.hidden[item.groupNumber]; + renderOverview(state, reports, chart); + }, + labels: { + boxWidth: 12, + generateLabels: function() { + var groups = []; + var groupNames = []; + reports.forEach(function(report) { + var index = groups.indexOf(report.groupNumber); + if (index === -1) { + groups.push(report.groupNumber); + var groupName = report.groups.slice(0,report.groups.length-1).join(' / '); + groupNames.push(groupName); + } + }); + return groups.map(function(groupNumber, index) { + var color = colors[groupNumber % colors.length]; + return { + text: groupNames[index], + fillStyle: color, + hidden: state.hidden[groupNumber], + groupNumber: groupNumber, + }; + }); + }, + }, + }, + tooltips: { + position: 'cursor', + callbacks: { + label: function(item) { + return formatTime(item.xLabel, 3); + }, + }, + }, + title: { + display: false, + text: 'Chart.js Horizontal Bar Chart' + } + } + }); + document.getElementById('sort-overview') + .addEventListener('change', overviewSort(state, reports, chart)); + var toggle = document.getElementById('legend-toggle'); + toggle.addEventListener('mouseup', function () { + state.legend = !state.legend; + if(state.legend) { + toggle.classList.add('right'); + } else { + toggle.classList.remove('right'); + } + renderOverview(state, reports, chart); + }) + return canvas; + } + + function mkKDE(report) { + var canvas = document.createElement('canvas'); + var mean = report.reportAnalysis.anMean.estPoint; + var units = timeUnits(mean); + var scale = units[0]; + var reportKDE = report.reportKDEs[0]; + var data = reportKDE.kdeValues.map(function(time, index) { + var pdf = reportKDE.kdePDF[index]; + return { + x: time * scale, + y: pdf + }; + }); + var chart = new Chart(canvas.getContext('2d'), { + type: 'line', + data: { + datasets: [{ + label: 'KDE', + borderColor: colors[0], + borderWidth: 2, + backgroundColor: '#00000000', + data: data, + hoverBorderWidth: 1, + pointHitRadius: 8, + }, + { + label: 'mean' + } + ], + }, + plugins: [{ + afterDraw: function(chart) { + var ctx = chart.ctx; + var area = chart.chartArea; + var axis = chart.scales[chart.options.scales.xAxes[0].id]; + var value = axis.getPixelForValue(mean * scale); + ctx.save(); + ctx.strokeStyle = colors[1]; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.moveTo(value, area.top); + ctx.lineTo(value, area.bottom); + ctx.stroke(); + ctx.restore(); + }, + }], + options: { + title: { + display: true, + text: report.groups.join(' / ') + ' — time densities', + }, + elements: { + point: { + radius: 0, + hitRadius: 0 + } + }, + scales: { + xAxes: [{ + display: true, + type: 'linear', + scaleLabel: { + display: false, + labelString: 'Time' + }, + ticks: { + min: reportKDE.kdeValues[0] * scale, + max: reportKDE.kdeValues[reportKDE.kdeValues.length - 1] * scale, + callback: function(value, index, values) { + // Don't show endpoints + if (index === 0 || index === values.length - 1) { + return ''; + } + var str = String(value) + ' ' + units[1]; + return str; + }, + } + }], + yAxes: [{ + display: true, + type: 'linear', + ticks: { + min: 0, + callback: function() { + return ''; + }, + }, + }] + }, + responsive: true, + legend: { + display: false, + position: 'right', + }, + tooltips: { + mode: 'nearest', + callbacks: { + title: function() { + return ''; + }, + label: function( + item) { + return formatUnit(item.xLabel, units[1], 3); + }, + }, + }, + hover: { + intersect: false + }, + } + }); + return canvas; + } + + function mkScatter(report) { + + // collect the measured value for a given regression + function getMeasured(key) { + var ix = report.reportKeys.indexOf(key); + return report.reportMeasured.map(function(x) { + return x[ix]; + }); + } + + var canvas = document.createElement('canvas'); + var times = getMeasured("time"); + var iters = getMeasured("iters"); + var lastIter = iters[iters.length - 1]; + var olsTime = report.reportAnalysis.anRegress[0].regCoeffs.iters; + var dataPoints = times.map(function(time, i) { + return { + x: iters[i], + y: time + } + }); + var formatter = iterFormatter(); + var chart = new Chart(canvas.getContext('2d'), { + type: 'scatter', + data: { + datasets: [{ + data: dataPoints, + label: 'scatter', + borderWidth: 2, + pointHitRadius: 8, + borderColor: colors[1], + backgroundColor: '#fff', + }, + { + data: [ + {x: 0, y: 0 }, + { x: lastIter, y: olsTime.estPoint * lastIter } + ], + label: 'regression', + type: 'line', + backgroundColor: "#00000000", + borderColor: colors[0], + pointRadius: 0, + }, + { + data: [{ + x: 0, + y: 0 + }, { + x: lastIter, + y: (olsTime.estPoint - olsTime.estError.confIntLDX) * lastIter, + }], + label: 'lower', + type: 'line', + fill: 1, + borderWidth: 0, + pointRadius: 0, + borderColor: '#00000000', + backgroundColor: colors[0] + '33', + }, + { + data: [{ + x: 0, + y: 0 + }, { + x: lastIter, + y: (olsTime.estPoint + olsTime.estError.confIntUDX) * lastIter, + }], + label: 'upper', + type: 'line', + fill: 1, + borderWidth: 0, + borderColor: '#00000000', + pointRadius: 0, + backgroundColor: colors[0] + '33', + }, + ], + }, + options: { + title: { + display: true, + text: report.groups.join(' / ') + ' — time per iteration', + }, + scales: { + yAxes: [{ + display: true, + type: 'linear', + scaleLabel: { + display: false, + labelString: 'Time' + }, + ticks: { + callback: function(value, index, values) { + return formatTime(value); + }, + } + }], + xAxes: [{ + display: true, + type: 'linear', + scaleLabel: { + display: false, + labelString: 'Iterations' + }, + ticks: { + callback: formatter, + max: lastIter, + } + }], + }, + legend: { + display: false, + }, + tooltips: { + callbacks: { + label: function(ttitem, ttdata) { + var iters = ttitem.xLabel; + var duration = ttitem.yLabel; + return formatTime(duration, 3) + ' / ' + + iters.toLocaleString() + ' iters'; + }, + }, + }, + } + }); + return canvas; + } + + // Create an HTML Element with attributes and child nodes + function elem(tag, props, children) { + var node = document.createElement(tag); + if (children) { + children.forEach(function(child) { + if (typeof child === 'string') { + var txt = document.createTextNode(child); + node.appendChild(txt); + } else { + node.appendChild(child); + } + }); + } + Object.assign(node, props); + return node; + } + + function bounds(analysis) { + var mean = analysis.estPoint; + return { + low: mean - analysis.estError.confIntLDX, + mean: mean, + high: mean + analysis.estError.confIntUDX + }; + } + + function confidence(level) { + return String(1 - level) + ' confidence level'; + } + + function mkOutliers(report) { + var outliers = report.reportAnalysis.anOutlierVar; + return elem('div', {className: 'outliers'}, [ + elem('p', {}, [ + 'Outlying measurements have ', + outliers.ovDesc, + ' (', String((outliers.ovFraction * 100).toPrecision(3)), '%)', + ' effect on estimated standard deviation.' + ]) + ]); + } + + function mkTable(report) { + var analysis = report.reportAnalysis; + var timep4 = function(t) { + return formatTime(t, 3) + }; + var idformatter = function(t) { + return t.toPrecision(3) + }; + var rows = [ + Object.assign({ + label: 'OLS regression', + formatter: timep4 + }, + bounds(analysis.anRegress[0].regCoeffs.iters)), + Object.assign({ + label: 'R² goodness-of-fit', + formatter: idformatter + }, + bounds(analysis.anRegress[0].regRSquare)), + Object.assign({ + label: 'Mean execution time', + formatter: timep4 + }, + bounds(analysis.anMean)), + Object.assign({ + label: 'Standard deviation', + formatter: timep4 + }, + bounds(analysis.anStdDev)), + ]; + return elem('table', { + className: 'analysis' + }, [ + elem('thead', {}, [ + elem('tr', {}, [ + elem('th'), + elem('th', { + className: 'low', + title: confidence(analysis.anRegress[0].regCoeffs.iters.estError.confIntCL) + }, ['lower bound']), + elem('th', {}, ['estimate']), + elem('th', { + className: 'high', + title: confidence(analysis.anRegress[0].regCoeffs.iters.estError.confIntCL) + }, ['upper bound']), + ]) + ]), + elem('tbody', {}, rows.map(function(row) { + return elem('tr', {}, [ + elem('td', {}, [row.label]), + elem('td', {className: 'low'}, [row.formatter(row.low, 4)]), + elem('td', {}, [row.formatter(row.mean)]), + elem('td', {className: 'high'}, [row.formatter(row.high, 4)]), + ]); + })) + ]); + } + document.addEventListener('DOMContentLoaded', function() { + var reportData = JSON.parse(document.getElementById('report-data') + .getAttribute('data-report-json')) + .map(function(report) { + report.groups = report.reportName.split('/'); + return report; + }); + groupReports(reportData); + var overview = document.getElementById('overview-chart'); + var overviewLineHeight = 16 * 1.25; + overview.style.height = + String(overviewLineHeight * reportData.length + 36) + 'px'; + overview.appendChild(mkOverview(reportData.slice())); + var reports = document.getElementById('reports'); + reportData.forEach(function(report, i) { + var id = 'report_' + String(i); + reports.appendChild( + elem('div', {id: id, className: 'report-details'}, [ + elem('h1', {}, [elem('a', {href: '#' + id}, [report.groups.join(' / ')])]), + elem('div', {className: 'kde'}, [mkKDE(report)]), + elem('div', {className: 'scatter'}, [mkScatter(report)]), + mkTable(report), mkOutliers(report) + ])); + }); + }, false); +})(); diff --git a/templates/default.tpl b/templates/default.tpl index 9d6dcb3f..8cd6ec62 100644 --- a/templates/default.tpl +++ b/templates/default.tpl @@ -1,362 +1,138 @@ - + - - - criterion report - - - - - - - - - -
-
-

criterion performance measurements

- -

overview

- -

want to understand this report?

- -
- -{{#report}} -

{{name}}

- - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
lower boundestimateupper bound
OLS regressionxxxxxxxxx
R² goodness-of-fitxxxxxxxxx
Mean execution time{{anMeanLowerBound}}{{anMean.estPoint}}{{anMeanUpperBound}}
Standard deviation{{anStdDevLowerBound}}{{anStdDev.estPoint}}{{anStdDevUpperBound}}
- - -

Outlying measurements have {{anOutlierVar.ovDesc}} - ({{anOutlierVar.ovFraction}}%) - effect on estimated standard deviation.

- -{{/report}} - -

understanding this report

- -

In this report, each function benchmarked by criterion is assigned - a section of its own. The charts in each section are active; if - you hover your mouse over data points and annotations, you will see - more details.

- - - -

Under the charts is a small table. - The first two rows are the results of a linear regression run - on the measurements displayed in the right-hand chart.

- - - -

We use a statistical technique called - the bootstrap - to provide confidence intervals on our estimates. The - bootstrap-derived upper and lower bounds on estimates let you see - how accurate we believe those estimates to be. (Hover the mouse - over the table headers to see the confidence levels.)

- -

A noisy benchmarking environment can cause some or many - measurements to fall far from the mean. These outlying - measurements can have a significant inflationary effect on the - estimate of the standard deviation. We calculate and display an - estimate of the extent to which the standard deviation has been - inflated by outliers.

- - + + + criterion report + + + + + +
+

criterion performance measurements

+

want to understand this report?

+

overview

+
+ + + ⓘ + + +
+ +
-
+
-