From f2e3f44a8815e7202ca2eecea0f7af1c07e17b8d Mon Sep 17 00:00:00 2001 From: Jordi Vilalta Prat Date: Mon, 11 Jan 2016 19:11:49 +0100 Subject: [PATCH 1/6] Initial JUnit report support. --- telescope.lua | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++ tsc | 5 +++ 2 files changed, 97 insertions(+) diff --git a/telescope.lua b/telescope.lua index 7250653..28ef294 100644 --- a/telescope.lua +++ b/telescope.lua @@ -578,12 +578,104 @@ local function summary_report(contexts, results) return table.concat(buffer, " "), r end +--- Writes a detailed report with the results of each test, in JUnit XML format. +-- Each context is converted into a test suite. +-- @param contexts The contexts returned by load_contexts. +-- @param results The results returned by run. +-- @function junit_report +local function junit_report(contexts, results) + local ancestor_separator = " / " + + local suites = {} + for i, item in ipairs(contexts) do + if item.context then + local name = item.name + local ancestors = ancestors(i, contexts) + for _, ancestor in pairs(ancestors) do + name = contexts[ancestor].name .. ancestor_separator .. name + end + + suites[i] = { name = name, cases = {} } + else + table.insert(suites[item.parent].cases, i) + end + end + + local xml_escapes = { + [">"] = ">", + ["<"] = "<", + ['"'] = """ + } + local function escape_xml(str) + return str:gsub("[<>]", function(a) return xml_escapes[a] end) + end + + local function write_test(report, test) + -- TODO: classname="" status="" + -- time: Time taken (in seconds) to execute the test + report:write(' ', "\n") + + if test.status_code == status_codes.err then + -- my crash report + -- TODO: message="" type="" + -- Contains as a text node relevant data for the error, e.g., a stack trace + report:write(' ' .. escape_xml(table.concat(test.message, "\n")) .. '', "\n") + elseif test.status_code == status_codes.fail then + -- my stack trace + -- TODO: message="" type="" + report:write(' ' .. escape_xml(table.concat(test.message, "\n")) .. '', "\n") + elseif test.status_code == status_codes.pass then + elseif test.status_code == status_codes.pending then + report:write(' ', "\n") + elseif test.status_code == status_codes.unassertive then + -- Treat it like passed + end + + -- TODO: + -- TODO: + report:write(' ', "\n") + end + + local function write_suite(report, suite) + -- TODO: disabled="" id="" package="" skipped="" + -- timestamp: when the test was executed. Timezone may not be specified. + -- hostname: Host on which the tests were executed. 'localhost' should be used if the hostname cannot be determined. + -- failures: The total number of tests in the suite that failed. A failure is a test which the code has explicitly failed by using the mechanisms for that purpose. e.g., via an assertEquals + -- errors: The total number of tests in the suite that errorrd. An errored test is one that had an unanticipated problem. e.g., an unchecked throwable; or a problem with the implementation of the test. + -- time: Time taken (in seconds) to execute the tests in the suite + -- it can contain a ... block + report:write(' ', "\n") + + for _, case in ipairs(suite.cases) do + write_test(report, results[case]) + end + + report:write(' ', "\n") + end + + local report = io.open("TEST.xml", "w") + report:write('', "\n") + -- TODO: name="" disabled="" errors="" failures="" tests="" time="" + report:write('', "\n") + + for i in ipairs(contexts) do + suite = suites[i] + if suite then + write_suite(report, suite) + end + end + + report:write('', "\n") + report:close() +end + _M.after_aliases = after_aliases _M.make_assertion = make_assertion _M.assertion_message_prefix = assertion_message_prefix _M.before_aliases = before_aliases _M.context_aliases = context_aliases _M.error_report = error_report +_M.junit_report = junit_report _M.load_contexts = load_contexts _M.run = run _M.test_report = test_report diff --git a/tsc b/tsc index a9ec753..1f51cae 100755 --- a/tsc +++ b/tsc @@ -162,6 +162,7 @@ Options: -h,-? --help Show this text -v --version Show version -c --luacov Output a coverage file using Luacov (http://luacov.luaforge.net/) + -j --junit Generate a JUnit compatible .xml report files --load= Load a Lua file before executing command --name= Only run tests whose name matches a Lua string pattern --shake Use shake as the front-end for tests @@ -296,6 +297,10 @@ if opts.c or opts.coverage then os.remove("luacov.stats.out") end +if opts.j or opts.junit then + telescope.junit_report(contexts, results, summary, data) +end + for _, v in pairs(results) do if v.status_code == telescope.status_codes.err or v.status_code == telescope.status_codes.fail then From 3911709918e30dc475f91561e25019298a9e856e Mon Sep 17 00:00:00 2001 From: Jordi Vilalta Prat Date: Tue, 12 Jan 2016 12:03:06 +0100 Subject: [PATCH 2/6] Added test time reporting (relying on Lua's clock precision) --- telescope.lua | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/telescope.lua b/telescope.lua index 28ef294..4450191 100644 --- a/telescope.lua +++ b/telescope.lua @@ -426,6 +426,12 @@ local function run(contexts, callbacks, test_filter) end end + -- Returns the number of elapsed seconds with decimals. + -- TODO: On POSIX systems it returns CPU time (not what we want), on Windows it returns elapsed time + local function get_seconds() + return os.clock() + end + for i, v in filter(contexts, function(i, v) return v.test and test_filter(v) end) do env = newEnv() -- Setup a new environment for this test @@ -454,7 +460,9 @@ local function run(contexts, callbacks, test_filter) -- check if it's a function because pending tests will just have "true" if type(v.test) == "function" then + result.start_time = get_seconds() result.status_code, result.assertions_invoked, result.message = invoke_test(v.test) + result.end_time = get_seconds() invoke_callback(status_names[result.status_code], result) else result.status_code = status_codes.pending @@ -612,8 +620,11 @@ local function junit_report(contexts, results) local function write_test(report, test) -- TODO: classname="" status="" - -- time: Time taken (in seconds) to execute the test - report:write(' ', "\n") + local test_time = "" + if test.start_time and test.end_time then + test_time = ' time="' .. (test.end_time - test.start_time) .. '"' + end + report:write(' ', "\n") if test.status_code == status_codes.err then -- my crash report From 73bde742c879ba8d18a5bd1ccf9f77d52a1c8998 Mon Sep 17 00:00:00 2001 From: Jordi Vilalta Prat Date: Tue, 12 Jan 2016 15:14:27 +0100 Subject: [PATCH 3/6] Split XML reports by input file. --- telescope.lua | 92 +++++++++++++++++++++++++++++---------------------- tsc | 11 +++--- 2 files changed, 59 insertions(+), 44 deletions(-) diff --git a/telescope.lua b/telescope.lua index 4450191..42ffc2b 100644 --- a/telescope.lua +++ b/telescope.lua @@ -590,25 +590,11 @@ end -- Each context is converted into a test suite. -- @param contexts The contexts returned by load_contexts. -- @param results The results returned by run. +-- @param file_contexts A table that associates the test filenames with the last context corresponding to it. -- @function junit_report -local function junit_report(contexts, results) +local function junit_report(contexts, results, file_contexts) local ancestor_separator = " / " - local suites = {} - for i, item in ipairs(contexts) do - if item.context then - local name = item.name - local ancestors = ancestors(i, contexts) - for _, ancestor in pairs(ancestors) do - name = contexts[ancestor].name .. ancestor_separator .. name - end - - suites[i] = { name = name, cases = {} } - else - table.insert(suites[item.parent].cases, i) - end - end - local xml_escapes = { [">"] = ">", ["<"] = "<", @@ -648,36 +634,64 @@ local function junit_report(contexts, results) end local function write_suite(report, suite) - -- TODO: disabled="" id="" package="" skipped="" - -- timestamp: when the test was executed. Timezone may not be specified. - -- hostname: Host on which the tests were executed. 'localhost' should be used if the hostname cannot be determined. - -- failures: The total number of tests in the suite that failed. A failure is a test which the code has explicitly failed by using the mechanisms for that purpose. e.g., via an assertEquals - -- errors: The total number of tests in the suite that errorrd. An errored test is one that had an unanticipated problem. e.g., an unchecked throwable; or a problem with the implementation of the test. - -- time: Time taken (in seconds) to execute the tests in the suite - -- it can contain a ... block - report:write(' ', "\n") - - for _, case in ipairs(suite.cases) do - write_test(report, results[case]) - end + -- TODO: disabled="" id="" package="" skipped="" + -- timestamp: when the test was executed. Timezone may not be specified. + -- hostname: Host on which the tests were executed. 'localhost' should be used if the hostname cannot be determined. + -- failures: The total number of tests in the suite that failed. A failure is a test which the code has explicitly failed by using the mechanisms for that purpose. e.g., via an assertEquals + -- errors: The total number of tests in the suite that errorrd. An errored test is one that had an unanticipated problem. e.g., an unchecked throwable; or a problem with the implementation of the test. + -- time: Time taken (in seconds) to execute the tests in the suite + -- it can contain a ... block + report:write(' ', "\n") + + for _, case in ipairs(suite.cases) do + write_test(report, results[case]) + end - report:write(' ', "\n") + report:write(' ', "\n") end - local report = io.open("TEST.xml", "w") - report:write('', "\n") - -- TODO: name="" disabled="" errors="" failures="" tests="" time="" - report:write('', "\n") + local function write_file(filename, suites) + local report = io.open("TEST-" .. filename .. ".xml", "w") + report:write('', "\n") + -- TODO: disabled="" errors="" failures="" tests="" time="" + report:write('', "\n") - for i in ipairs(contexts) do - suite = suites[i] - if suite then - write_suite(report, suite) + for i in ipairs(contexts) do + suite = suites[i] + if suite then + write_suite(report, suite) + end end + + report:write('', "\n") + report:close() end - report:write('', "\n") - report:close() + + local i = 1 + for _, file_info in ipairs(file_contexts) do + local suites = {} + + while i <= file_info.last_context do + local item = contexts[i] + + if item.context then + local name = item.name + local ancestors = ancestors(i, contexts) + for _, ancestor in pairs(ancestors) do + name = contexts[ancestor].name .. ancestor_separator .. name + end + + suites[i] = { name = name, cases = {} } + else + table.insert(suites[item.parent].cases, i) + end + + i = i + 1 + end + + write_file(file_info.name, suites) + end end _M.after_aliases = after_aliases diff --git a/tsc b/tsc index 1f51cae..32b01c3 100755 --- a/tsc +++ b/tsc @@ -265,10 +265,11 @@ for _, callback in ipairs(callback_args) do end local contexts = {} -if opts["shake"] then - for _, file in ipairs(files) do shake.load_contexts(file, contexts) end -else - for _, file in ipairs(files) do telescope.load_contexts(file, contexts) end +local file_contexts = {} +local load_contexts = opts["shake"] and shake.load_contexts or telescope.load_contexts +for _, file in ipairs(files) do + load_contexts(file, contexts) + table.insert(file_contexts, { name = string.match(file, "(.*)%."), last_context = #contexts }) end local buffer = {} @@ -298,7 +299,7 @@ if opts.c or opts.coverage then end if opts.j or opts.junit then - telescope.junit_report(contexts, results, summary, data) + telescope.junit_report(contexts, results, file_contexts) end for _, v in pairs(results) do From b2f24784ac9ea9e9e829f7800bab563b0f6253b1 Mon Sep 17 00:00:00 2001 From: Jordi Vilalta Prat Date: Wed, 3 Feb 2016 12:02:50 +0100 Subject: [PATCH 4/6] Export JUnit reports to a folder --- telescope.lua | 12 ++++++++---- tsc | 6 +++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/telescope.lua b/telescope.lua index 42ffc2b..af1c545 100644 --- a/telescope.lua +++ b/telescope.lua @@ -591,8 +591,10 @@ end -- @param contexts The contexts returned by load_contexts. -- @param results The results returned by run. -- @param file_contexts A table that associates the test filenames with the last context corresponding to it. +-- @param out_path The path where the output files will be written -- @function junit_report -local function junit_report(contexts, results, file_contexts) +local function junit_report(contexts, results, file_contexts, out_path) + assert(type(out_path) == "string") local ancestor_separator = " / " local xml_escapes = { @@ -650,8 +652,8 @@ local function junit_report(contexts, results, file_contexts) report:write(' ', "\n") end - local function write_file(filename, suites) - local report = io.open("TEST-" .. filename .. ".xml", "w") + local function write_file(out_path, filename, suites) + local report = io.open(out_path .. "/TEST-" .. filename .. ".xml", "w") report:write('', "\n") -- TODO: disabled="" errors="" failures="" tests="" time="" report:write('', "\n") @@ -668,6 +670,8 @@ local function junit_report(contexts, results, file_contexts) end + -- TODO: Find a portable way to create a directory + os.execute("mkdir " .. out_path) local i = 1 for _, file_info in ipairs(file_contexts) do local suites = {} @@ -690,7 +694,7 @@ local function junit_report(contexts, results, file_contexts) i = i + 1 end - write_file(file_info.name, suites) + write_file(out_path, file_info.name, suites) end end diff --git a/tsc b/tsc index 32b01c3..497af7f 100755 --- a/tsc +++ b/tsc @@ -162,7 +162,7 @@ Options: -h,-? --help Show this text -v --version Show version -c --luacov Output a coverage file using Luacov (http://luacov.luaforge.net/) - -j --junit Generate a JUnit compatible .xml report files + --junit= Generate JUnit compatible .xml report files in the specified path --load= Load a Lua file before executing command --name= Only run tests whose name matches a Lua string pattern --shake Use shake as the front-end for tests @@ -298,8 +298,8 @@ if opts.c or opts.coverage then os.remove("luacov.stats.out") end -if opts.j or opts.junit then - telescope.junit_report(contexts, results, file_contexts) +if opts.junit then + telescope.junit_report(contexts, results, file_contexts, opts.junit) end for _, v in pairs(results) do From 667cc43fb588f0915d50310a4d5927e7c4394f06 Mon Sep 17 00:00:00 2001 From: Jordi Vilalta Prat Date: Wed, 3 Feb 2016 12:17:24 +0100 Subject: [PATCH 5/6] Also escape ampersand when generating XML --- telescope.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/telescope.lua b/telescope.lua index af1c545..7debc93 100644 --- a/telescope.lua +++ b/telescope.lua @@ -600,10 +600,11 @@ local function junit_report(contexts, results, file_contexts, out_path) local xml_escapes = { [">"] = ">", ["<"] = "<", + ["&"] = "&", ['"'] = """ } local function escape_xml(str) - return str:gsub("[<>]", function(a) return xml_escapes[a] end) + return str:gsub("[<>&\"]", function(a) return xml_escapes[a] end) end local function write_test(report, test) From 01536272de7835573231bf0dcdd69ab4582865b1 Mon Sep 17 00:00:00 2001 From: Jordi Vilalta Prat Date: Thu, 4 Feb 2016 20:10:14 +0100 Subject: [PATCH 6/6] Use different exit codes for fails and errors --- tsc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tsc b/tsc index 497af7f..0c3b308 100755 --- a/tsc +++ b/tsc @@ -302,9 +302,15 @@ if opts.junit then telescope.junit_report(contexts, results, file_contexts, opts.junit) end +local fail = false for _, v in pairs(results) do - if v.status_code == telescope.status_codes.err or - v.status_code == telescope.status_codes.fail then + if v.status_code == telescope.status_codes.err then os.exit(1) end + if v.status_code == telescope.status_codes.fail then + fail = true + end +end +if fail then + os.exit(2) end