diff --git a/README.md b/README.md index 49ab1db..f0e742c 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ The buildkite annotation context to use. Useful to differentiate multiple runs o Default: `-(.*).xml` -The regular expression (with capture group) that matches the job UUID in the junit file names. This is used to create the job links in the annotation. +The regular expression (with capture group) that matches the job UUID in the junit file names. This is used to create the job links in the annotation. To use this, configure your test reporter to embed the `$BUILDKITE_JOB_ID` environment variable into your junit file names. For example `"junit-buildkite-job-$BUILDKITE_JOB_ID.xml"`. @@ -54,7 +54,7 @@ There are two options for this: * `file` * displays: `MyClass::UnderTest text of the failed expectation in path/to/my_class/under_test.file_ext` -### `fail-build-on-error` (optional) +### `fail-build-on-error` (optional) Default: `false` @@ -70,6 +70,12 @@ Exit code of the plugin if the call to `buildkite-agent artifact download` fails Minimum amount of run tests that need to be analyzed or a failure will be reported. It is useful to ensure that tests are actually run and report files to analyze do contain information. +### `report-skipped` (optional, boolean) + +Default: `false` + +Will add a list of skipped tests at the end of the annotation. Note that even if there are skipped tests, the annotation may not be added unless other options or results of the processing forces it to. + ### `report-slowest` (optional) Default: `0` diff --git a/hooks/command b/hooks/command index fbdc9b2..a6c7cc7 100755 --- a/hooks/command +++ b/hooks/command @@ -51,6 +51,7 @@ docker \ --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN:-}" \ --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT:-}" \ --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST:-}" \ + --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED:-}" \ "${RUBY_IMAGE}" ruby /src/bin/annotate /junits \ > "$annotation_path" @@ -78,9 +79,9 @@ if [ $has_errors -eq 0 ]; then echo "Will create annotation anyways" create_annotation=1 fi - + if [[ -e "${annotation_path}" ]]; then - TOTAL_TESTS=$(head -4 "${annotation_path}" | grep 'Total tests' | cut -d\ -f3) + TOTAL_TESTS=$(head -5 "${annotation_path}" | grep 'Total tests' | cut -d\ -f3) else TOTAL_TESTS=0 fi @@ -95,7 +96,7 @@ elif ! check_size; then # creating a simplified version of the annotation mv "${annotation_path}" "${annotation_path}2" - head -4 "${annotation_path}2" >"${annotation_path}" + head -5 "${annotation_path}2" >"${annotation_path}" # || true is to avoid issues if no summary is found grep '' "${annotation_path}2" >>"${annotation_path}" || true diff --git a/plugin.yml b/plugin.yml index 7ce94bf..fe87f6d 100644 --- a/plugin.yml +++ b/plugin.yml @@ -24,6 +24,8 @@ configuration: type: string min-tests: type: integer + report-skipped: + type: boolean report-slowest: type: integer ruby-image: diff --git a/ruby/bin/annotate b/ruby/bin/annotate index 8d6ef65..1321142 100755 --- a/ruby/bin/annotate +++ b/ruby/bin/annotate @@ -18,8 +18,9 @@ failure_format = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT'] failure_format = 'classname' if !failure_format || failure_format.empty? report_slowest = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST'].to_i +report_skipped = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED'] == 'true' -class Failure < Struct.new(:name, :unit_name, :body, :job, :type, :message) +class Failure < Struct.new(:name, :unit_name, :body, :job, :message) end class Timing < Struct.new(:name, :unit_name, :time) @@ -27,7 +28,11 @@ end junit_report_files = Dir.glob(File.join(junits_dir, "**", "*"), File::FNM_DOTMATCH) testcases = 0 -failures = [] +tests = { + failure: [], + error: [], + skipped: [] +} timings = [] def text_content(element) @@ -64,30 +69,27 @@ junit_report_files.sort.each do |file| unit_name = testcase.attributes[failure_format].to_s time = testcase.attributes['time'].to_f timings << Timing.new(name, unit_name, time) - testcase.elements.each("failure") do |failure| - failures << Failure.new(name, unit_name, text_content(failure), job, :failure, message_content(failure)) - end - testcase.elements.each("error") do |error| - failures << Failure.new(name, unit_name, text_content(error), job, :error, message_content(error)) + testcase.elements.each("failure | error | skipped") do |elem| + tests[elem.name.to_sym] << Failure.new(name, unit_name, text_content(elem), job, message_content(elem)) end end end STDERR.puts "--- ✍️ Preparing annotation" -failures_count = failures.select {|f| f.type == :failure }.length -errors_count = failures.select {|f| f.type == :error }.length - -puts "Failures: #{failures_count}" -puts "Errors: #{errors_count}" +puts "Failures: #{tests[:failure].length}" +puts "Errors: #{tests[:error].length}" +puts "Skipped: #{tests[:skipped].length}" puts "Total tests: #{testcases}" -failures.each do |failure| +skipped = tests.delete(:skipped) # save value for later + +tests.values.flatten.each do |failure| puts "" puts "
" - puts "#{failure.name} in #{failure.unit_name}\n\n" + puts "#{CGI.escapeHTML failure.name} in #{CGI.escapeHTML failure.unit_name}\n\n" if failure.message - puts "

#{failure.message.chomp.strip}

\n\n" + puts "

#{CGI.escapeHTML failure.message.chomp.strip}

\n\n" end if failure.body puts "
#{CGI.escapeHTML(failure.body.chomp.strip)}
\n\n" @@ -114,4 +116,17 @@ if report_slowest > 0 puts "
" end -exit 64 if failures.any? # special exit code to signal test failures +if report_skipped + STDERR.puts "Reporting skipped tests" + puts "" + puts "
" + puts "#{skipped.length} tests skipped\n\n" + puts "
    " + skipped.each do |sk| + puts "
  1. #{CGI.escapeHTML sk.name} in #{CGI.escapeHTML sk.unit_name} (#{CGI.escapeHTML sk.message || "no reason"})
    2. \n" + end + puts "
" + puts "
" +end + +exit 64 if tests.values.flatten.any? # special exit code to signal test failures diff --git a/ruby/tests/annotate_test.rb b/ruby/tests/annotate_test.rb index 823c242..8db280d 100644 --- a/ruby/tests/annotate_test.rb +++ b/ruby/tests/annotate_test.rb @@ -15,6 +15,7 @@ assert_equal stdout, <<~OUTPUT Failures: 0 Errors: 0 + Skipped: 0 Total tests: 8 OUTPUT @@ -34,81 +35,82 @@ assert_equal stdout, <<~OUTPUT Failures: 4 Errors: 0 + Skipped: 0 Total tests: 6 - +
Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec - +

expected: 250 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
-      
+
         expected: 250
              got: 500
-      
+
         (compared using eql?)
       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- + in Job #1
- +
Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec - +

expected: 700 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
-      
+
         expected: 700
              got: 500
-      
+
         (compared using eql?)
       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- + in Job #2
Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec - +

expected: 700 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
-      
+
         expected: 700
              got: 500
-      
+
         (compared using eql?)
       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- + in Job #3
Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec - +

expected: 250 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
-      
+
         expected: 250
              got: 500
-      
+
         (compared using eql?)
       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- + in Job #3
OUTPUT @@ -129,81 +131,82 @@ assert_equal stdout, <<~OUTPUT Failures: 2 Errors: 2 + Skipped: 0 Total tests: 6
- Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec - -

expected: 250 got: 500 (compared using eql?)

+ Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec + +

expected: 700 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
-      
-        expected: 250
+
+        expected: 700
              got: 500
-      
+
         (compared using eql?)
       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- - in Job #1 + + in Job #2
- +
Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec - +

expected: 700 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
-      
+
         expected: 700
              got: 500
-      
+
         (compared using eql?)
       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- - in Job #2 + + in Job #3
- Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec - -

expected: 700 got: 500 (compared using eql?)

+ Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec + +

expected: 250 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
-      
-        expected: 700
+
+        expected: 250
              got: 500
-      
+
         (compared using eql?)
       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- - in Job #3 + + in Job #1
Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec - +

expected: 250 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
-      
+
         expected: 250
              got: 500
-      
+
         (compared using eql?)
       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- + in Job #3
OUTPUT @@ -222,24 +225,25 @@ assert_equal stdout, <<~OUTPUT Failures: 1 Errors: 0 + Skipped: 0 Total tests: 2 - +
Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec - +

expected: 250 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
-      
+
         expected: 250
              got: 500
-      
+
         (compared using eql?)
       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- + in Job #123-456
OUTPUT @@ -260,16 +264,17 @@ assert_equal stdout, <<~OUTPUT Failures: 2 Errors: 2 + Skipped: 0 Total tests: 6
- Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in ./spec/models/account_spec.rb + Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in ./spec/models/account_spec.rb -

expected: 250 got: 500 (compared using eql?)

+

expected: 700 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
 
-        expected: 250
+        expected: 700
              got: 500
 
         (compared using eql?)
@@ -278,7 +283,7 @@
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- in Job #1 + in Job #2
@@ -297,17 +302,17 @@ ./spec/support/log.rb:17:in `run' ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>' - in Job #2 + in Job #3
- Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in ./spec/models/account_spec.rb + Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in ./spec/models/account_spec.rb -

expected: 700 got: 500 (compared using eql?)

+

expected: 250 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
 
-        expected: 700
+        expected: 250
              got: 500
 
         (compared using eql?)
@@ -316,7 +321,7 @@
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- in Job #3 + in Job #1
@@ -355,81 +360,82 @@ assert_equal stdout, <<~OUTPUT Failures: 4 Errors: 0 + Skipped: 0 Total tests: 6 - +
Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec - +

expected: 250 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
-      
+
         expected: 250
              got: 500
-      
+
         (compared using eql?)
       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- + in Job #1
- +
Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec - +

expected: 700 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
-      
+
         expected: 700
              got: 500
-      
+
         (compared using eql?)
       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- + in Job #2
Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec - +

expected: 700 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
-      
+
         expected: 700
              got: 500
-      
+
         (compared using eql?)
       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- + in Job #3
Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec - +

expected: 250 got: 500 (compared using eql?)

Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
-      
+
         expected: 250
              got: 500
-      
+
         (compared using eql?)
       ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
       ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
       ./spec/support/log.rb:17:in `run'
       ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'
- + in Job #3
OUTPUT @@ -448,6 +454,7 @@ assert_equal stdout, <<~OUTPUT Failures: 1 Errors: 0 + Skipped: 0 Total tests: 2
@@ -472,18 +479,19 @@ assert_equal stdout, <<~OUTPUT Failures: 1 Errors: 2 + Skipped: 0 Total tests: 4
Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec
- +
Account#maximum_jobs_added_by_pipeline_changer returns 100 by default in spec.models.account_spec - +
- +
Account#maximum_jobs_added_by_pipeline_changer returns 50 by default in spec.models.account_spec @@ -504,6 +512,7 @@ assert_equal stdout, <<~OUTPUT Failures: 0 Errors: 1 + Skipped: 0 Total tests: 2
@@ -534,8 +543,9 @@ assert_equal stdout, <<~OUTPUT Failures: 0 Errors: 0 + Skipped: 0 Total tests: 8 - +
5 slowest tests @@ -566,9 +576,55 @@ assert_equal stdout, <<~OUTPUT Failures: 0 Errors: 0 + Skipped: 0 + Total tests: 2 + OUTPUT + + assert_equal 0, status.exitstatus + end + + it "correctly parses skipped tests" do + stdout, stderr, status = Open3.capture3("#{__dir__}/../bin/annotate", "#{__dir__}/skipped-test/") + + assert_equal stderr, <<~OUTPUT + Parsing junit.xml + --- ✍️ Preparing annotation + OUTPUT + + assert_equal stdout, <<~OUTPUT + Failures: 0 + Errors: 0 + Skipped: 1 Total tests: 2 OUTPUT assert_equal 0, status.exitstatus end + + it "can report skipped tests" do + stdout, stderr, status = Open3.capture3("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED=true", "#{__dir__}/../bin/annotate", "#{__dir__}/skipped-test/") + + assert_equal stderr, <<~OUTPUT + Parsing junit.xml + --- ✍️ Preparing annotation + Reporting skipped tests + OUTPUT + + assert_equal stdout, <<~OUTPUT + Failures: 0 + Errors: 0 + Skipped: 1 + Total tests: 2 + +
+ 1 tests skipped + +
    +
  1. Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec (because of reasons)
    2. +
+
+ OUTPUT + + assert_equal 0, status.exitstatus + end end diff --git a/ruby/tests/skipped-test/junit.xml b/ruby/tests/skipped-test/junit.xml new file mode 100644 index 0000000..dead56c --- /dev/null +++ b/ruby/tests/skipped-test/junit.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/tests/2-tests-1-failure.output b/tests/2-tests-1-failure.output index 24f96af..0df43ac 100644 --- a/tests/2-tests-1-failure.output +++ b/tests/2-tests-1-failure.output @@ -1,5 +1,6 @@ Failures: 0 Errors: 1 +Skipped: 0 Total tests: 2
diff --git a/tests/command.bats b/tests/command.bats index 9cddfc1..044e7a4 100644 --- a/tests/command.bats +++ b/tests/command.bats @@ -12,6 +12,8 @@ export artifacts_tmp="tests/tmp/junit-artifacts" export annotation_tmp="tests/tmp/junit-annotation" export annotation_input="tests/tmp/annotation.input" +DOCKER_STUB_DEFAULT_OPTIONS='--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* --env \* \*' + @test "runs the annotator and creates the annotation" { export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR=false @@ -25,7 +27,7 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : echo '
Failure
' && exit 64" + "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : echo '
Failure
' && exit 64" run "$PWD/hooks/command" @@ -53,14 +55,14 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" + "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" assert_success assert_output --partial "Annotation added with context junit_custom_context" - + unstub mktemp unstub buildkite-agent unstub docker @@ -80,7 +82,7 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN='custom_(*)_pattern.xml' --env \* --env \* \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" + "--log-level error run --rm --volume \* --volume \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN='custom_(*)_pattern.xml' --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" @@ -107,7 +109,7 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT='file' --env \* \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" + "--log-level error run --rm --volume \* --volume \* --env \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT='file' --env \* --env \* \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" @@ -132,7 +134,7 @@ export annotation_input="tests/tmp/annotation.input" "artifact download \* \* : echo Downloaded artifact \$3 to \$4" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : echo 'Total tests: 0'" + "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : echo 'Total tests: 0'" run "$PWD/hooks/command" @@ -156,7 +158,7 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : echo 'Total tests: 0'" + "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : echo 'Total tests: 0'" run "$PWD/hooks/command" @@ -195,7 +197,7 @@ export annotation_input="tests/tmp/annotation.input" "artifact download \* \* : echo Downloaded artifact \$3 to \$4" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" + "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" @@ -227,7 +229,7 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" + "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" @@ -235,7 +237,7 @@ export annotation_input="tests/tmp/annotation.input" assert_output --partial "Failures too large to annotate" assert_output --partial "using a simplified annotation" - assert_equal "5 ${annotation_input}" "$(wc -l "${annotation_input}" | cut -f 1)" + assert_equal "6 ${annotation_input}" "$(wc -l "${annotation_input}" | cut -f 1)" unstub docker unstub du @@ -257,7 +259,7 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" + "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" @@ -281,12 +283,12 @@ export annotation_input="tests/tmp/annotation.input" stub du \ "-k \* : echo 1025$'\t'\$2" \ "-k \* : echo 1025$'\t'\$2" - + stub buildkite-agent \ "artifact download \* \* : echo Downloaded artifact \$3 to \$4" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" + "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" @@ -312,7 +314,7 @@ export annotation_input="tests/tmp/annotation.input" "artifact download \* \* : echo Downloaded artifact \$3 to \$4" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 147" + "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 147" run "$PWD/hooks/command" @@ -380,7 +382,7 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : echo 'Total tests: 0'" + "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : echo 'Total tests: 0'" run "$PWD/hooks/command" @@ -406,7 +408,7 @@ export annotation_input="tests/tmp/annotation.input" "artifact download \* \* : echo Downloaded artifact \$3 to \$4" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : echo 'Total tests: 100'" + "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : echo 'Total tests: 100'" run "$PWD/hooks/command" @@ -432,7 +434,7 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" + "${DOCKER_STUB_DEFAULT_OPTIONS} ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" @@ -457,7 +459,7 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:special ruby /src/bin/annotate /junits : echo '
Failure
' && exit 64" + "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* --env \* ruby:special ruby /src/bin/annotate /junits : echo '
Failure
' && exit 64" run "$PWD/hooks/command" @@ -470,4 +472,31 @@ export annotation_input="tests/tmp/annotation.input" unstub buildkite-agent unstub docker rm "${annotation_input}" -} \ No newline at end of file +} + +@test "can customize skipped tests variable" { + export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" + export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED="true" + + stub mktemp \ + "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ + "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" + + stub buildkite-agent \ + "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ + "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" + + stub docker \ + "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED='true' \* ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" + + run "$PWD/hooks/command" + + assert_success + + assert_output --partial "Annotation added" + + unstub mktemp + unstub buildkite-agent + unstub docker + rm "${annotation_input}" +}