diff --git a/btest-diff b/btest-diff index 72557d6b..05bed144 100755 --- a/btest-diff +++ b/btest-diff @@ -12,10 +12,67 @@ # TEST_DIFF_CANONIFIER # TEST_DIFF_BRIEF # TEST_DIFF_FILE_MAX_LINES +# +# This script has the following exit codes: +# +# When TEST_MODE is TEST: +# 0 - Comparison succeeded, files are the same +# 1 - Problems with input file/args or running TEST_DIFF_CANONIFIER, or file contents differ +# 2 - Other diffing trouble (inherited from diff) +# 100 - No baseline to compare to available +# +# When TEST_MODE is UPDATE: +# 0 - Baseline updated +# 1 - Problems with input file/args or running TEST_DIFF_CANONIFIER +# +# When TEST_MODE is UPDATE_INTERACTIVE: +# 0 - Baseline updated, or nothing to update +# 1 - Problems with input file/args or running TEST_DIFF_CANONIFIER, or user skips a deviating baseline +# 200 - User asks to abort after a deviating baseline +# +# Otherwise: exits with 1 + +# It's okay to check $? explicitly: +# shellcheck disable=SC2181 # Maximum number of lines to show from mismatching input file by default. MAX_LINES=100 +# Header line we tuck onto new baselines generated by +# btest-diff. Serves both as a warning and as an indicator that the +# baseline has been run through the TEST_DIFF_CANONIFIER (if any). +HEADER="### btest baseline data generated by btest-diff. Do not edit. Use btest -U/-u to update." + +# Predicate, succeeds if the given baseline is canonicalized. +is_canon_baseline() { + local input="$1" + + if head -n 1 "$input" | grep -q -F "$HEADER" 2>/dev/null; then + return 0 + fi + + return 1 +} + +# Prints the requested baseline to standard out if it is +# canonicalized. Otherwise, fails. +get_canon_baseline() { + local input="$1" + + ! is_canon_baseline "$input" && return 1 + tail -n +2 "$input" +} + +# Updates the given baseline to the given destination, prepending our +# header. +update_canon_baseline() { + local input="$1" + local output="$2" + + echo "$HEADER" >"$output" + cat "$input" >>"$output" +} + if [ -n "$TEST_DIFF_FILE_MAX_LINES" ]; then MAX_LINES=$TEST_DIFF_FILE_MAX_LINES fi @@ -25,22 +82,23 @@ if [ "$TEST_DIAGNOSTICS" = "" ]; then fi if [ "$#" -lt 1 ]; then - echo "btest-diff: wrong number of arguments" >$TEST_DIAGNOSTICS - exit 1 + echo "btest-diff: wrong number of arguments" >$TEST_DIAGNOSTICS + exit 1 fi -input=$1 -canon=`echo $input | sed 's#/#.#g'` +input="$1" +# shellcheck disable=SC2001 +canon=$(echo "$input" | sed 's#/#.#g') shift -if [ ! -f $input ]; then - echo "btest-diff: input $input does not exist." >$TEST_DIAGNOSTICS - exit 1 +if [ ! -f "$input" ]; then + echo "btest-diff: input $input does not exist." >$TEST_DIAGNOSTICS + exit 1 fi tmpfiles="" -function delete_tmps { - rm -f $tmpfiles 2>/dev/null +delete_tmps() { + rm -f $tmpfiles 2>/dev/null } trap delete_tmps 0 @@ -51,84 +109,106 @@ rm -f $TEST_DIAGNOSTICS 2>/dev/null echo "== File ===============================" >>$TEST_DIAGNOSTICS -if [ ! -e $TEST_BASELINE/$canon ]; then - cat $input >>$TEST_DIAGNOSTICS +if [ ! -e "$TEST_BASELINE/$canon" ]; then + cat "$input" >>$TEST_DIAGNOSTICS elif [ -n "$TEST_DIFF_BRIEF" ]; then echo "" >>$TEST_DIAGNOSTICS else - if [ `wc -l $input | awk '{print $1}'` -le $MAX_LINES ]; then - cat $input >>$TEST_DIAGNOSTICS + if [ "$(wc -l "$input" | awk '{print $1}')" -le "$MAX_LINES" ]; then + cat "$input" >>$TEST_DIAGNOSTICS else - head -$MAX_LINES $input >>$TEST_DIAGNOSTICS + head -n "$MAX_LINES" "$input" >>$TEST_DIAGNOSTICS echo "[... File too long, truncated ...]" >>$TEST_DIAGNOSTICS fi fi -if [ -e $TEST_BASELINE/$canon ]; then - error=0 - - if [ "$TEST_DIFF_CANONIFIER" != "" ]; then - diff1=/tmp/test-diff.$$.$canon.baseline.tmp - diff2=/tmp/test-diff.$$.$canon.tmp - tmpfiles="$tmpfiles $diff1 $diff2" - - eval $TEST_DIFF_CANONIFIER $TEST_BASELINE/$canon <$TEST_BASELINE/$canon >$diff1 - if [ $? -ne 0 ]; then - error=1 - echo "== Error ==============================" >>$TEST_DIAGNOSTICS - echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$TEST_BASELINE/$canon'" >>$TEST_DIAGNOSTICS - result=1 - fi - - eval $TEST_DIFF_CANONIFIER $input <$input >$diff2 - if [ $? -ne 0 ]; then - error=1 - echo "== Error ==============================" >>$TEST_DIAGNOSTICS - echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$input'" >>$TEST_DIAGNOSTICS - result=1 - fi - else - diff1=$TEST_BASELINE/$canon - diff2=$input - fi - - if [ $error -eq 0 ]; then - echo "== Diff ===============================" >>$TEST_DIAGNOSTICS - diff -au $@ $diff1 $diff2 >>$TEST_DIAGNOSTICS - result=$? - fi -else - echo "== Error ==============================" >>$TEST_DIAGNOSTICS - echo "test-diff: no baseline found." >>$TEST_DIAGNOSTICS - result=100 +# If no canonifier is defined, just copy. Simplifies code layout. +if [ -z "$TEST_DIFF_CANONIFIER" ]; then + TEST_DIFF_CANONIFIER="cat" +fi + +canon_output=/tmp/test-diff.$$.$canon.tmp +tmpfiles="$tmpfiles $canon_output" +error=0 + +# Canonicalize the new test output. +# shellcheck disable=SC2094 +eval "$TEST_DIFF_CANONIFIER" "$input" <"$input" >"$canon_output" +if [ $? -ne 0 ]; then + echo "== Error ==============================" >>$TEST_DIAGNOSTICS + echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$input'" >>$TEST_DIAGNOSTICS + error=1 + result=1 +fi + +if [ -e "$TEST_BASELINE/$canon" ]; then + canon_baseline=/tmp/test-diff.$$.$canon.baseline.tmp + tmpfiles="$tmpfiles $canon_baseline" + + # Prepare the baseline. When created by a recent btest-diff, we + # don't need to re-canonicalize, otherwise we do. + if ! get_canon_baseline "$TEST_BASELINE/$canon" >"$canon_baseline"; then + # It's an older uncanonicalized baseline, so canonicalize + # it now prior to comparison. Future updates via btest + # -U/-u will then store it canonicalized. + eval "$TEST_DIFF_CANONIFIER" <"$TEST_BASELINE/$canon" >"$canon_baseline" + if [ $? -ne 0 ]; then + echo "== Error ==============================" >>$TEST_DIAGNOSTICS + echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$TEST_BASELINE/$canon'" >>$TEST_DIAGNOSTICS + error=1 + result=1 + fi + fi + + if [ $error -eq 0 ]; then + echo "== Diff ===============================" >>$TEST_DIAGNOSTICS + diff -au "$@" "$canon_baseline" "$canon_output" >>$TEST_DIAGNOSTICS + result=$? + fi +elif [ "$TEST_MODE" = "TEST" ]; then + echo "== Error ==============================" >>$TEST_DIAGNOSTICS + echo "test-diff: no baseline found." >>$TEST_DIAGNOSTICS + result=100 fi echo "=======================================" >>$TEST_DIAGNOSTICS -if [ "$TEST_MODE" == "TEST" ]; then - exit $result +if [ "$TEST_MODE" = "TEST" ]; then + exit $result + +elif [ "$TEST_MODE" = "UPDATE_INTERACTIVE" ]; then + + # We had a problem running the canonifier + if [ "$error" != 0 ]; then + exit 1 + fi + + # There's no change to the baseline, so skip user interaction + if [ "$result" = 0 ]; then + exit 0 + fi -elif [ "$TEST_MODE" == "UPDATE_INTERACTIVE" ]; then + btest-ask-update + rc=$? - if [ "$result" == 0 ]; then - exit 0 - fi + echo -n "$TEST_NAME ..." >/dev/tty - btest-ask-update - rc=$? + if [ $rc = 0 ]; then + update_canon_baseline "$canon_output" "$TEST_BASELINE/$canon" + exit 0 + fi - echo -n "$TEST_NAME ..." >/dev/tty + exit $rc - if [ $rc == 0 ]; then - cp $input $TEST_BASELINE/$canon - exit 0 - fi +elif [ "$TEST_MODE" = "UPDATE" ]; then - exit $rc + # We had a problem running the canonifier + if [ "$error" != 0 ]; then + exit 1 + fi -elif [ "$TEST_MODE" == "UPDATE" ]; then - cp $input $TEST_BASELINE/$canon - exit 0 + update_canon_baseline "$canon_output" "$TEST_BASELINE/$canon" + exit 0 fi echo "test-diff: unknown test mode $TEST_MODE" >$TEST_DIAGNOSTICS diff --git a/testing/tests/canonifier-conversion.test b/testing/tests/canonifier-conversion.test new file mode 100644 index 00000000..c1c0e486 --- /dev/null +++ b/testing/tests/canonifier-conversion.test @@ -0,0 +1,52 @@ +# This test verifies that we only canonicalize baselines once, namely +# when canonicalizing new test output, and that baselines get +# converted over to our header-tagged format. +# +# Test prep: make our canonifier executable +# %TEST-EXEC: chmod 755 ./diff-double-x +# +# Verify that without an existing baseline, we canonify a new one. +# %TEST-EXEC: btest -U %INPUT +# %TEST-EXEC: head -1 Baseline/canonifier-conversion/output | grep -q 'Do not edit' +# %TEST-EXEC: tail -1 Baseline/canonifier-conversion/output | grep xx +# %TEST-EXEC: cp Baseline/canonifier-conversion/output base.1 +# +# For testing conversion, first create a "dated" baseline. +# %TEST-EXEC: echo x >Baseline/canonifier-conversion/output +# +# Verify that it succeeds: +# %TEST-EXEC: btest %INPUT +# +# Update the baseline. This should prefix btest-diff's header and canonify: +# %TEST-EXEC: btest -U %INPUT +# +# Verify that these have happened and preserve baseline: +# %TEST-EXEC: head -1 Baseline/canonifier-conversion/output | grep -q 'Do not edit' +# %TEST-EXEC: tail -1 Baseline/canonifier-conversion/output | grep xx +# %TEST-EXEC: cp Baseline/canonifier-conversion/output base.2 +# +# Verify that it still succeeds: +# %TEST-EXEC: btest %INPUT +# +# Update the baseline again. +# %TEST-EXEC: btest -U %INPUT +# %TEST-EXEC: cp Baseline/canonifier-conversion/output base.3 +# +# Verify the updated baselines remained identical. +# %TEST-EXEC: test "$(cat base.1)" = "$(cat base.2)" && test "$(cat base.2)" = "$(cat base.3)" + +@TEST-EXEC: echo x >output +@TEST-EXEC: btest-diff output + +%TEST-START-FILE btest.cfg +[btest] +TmpDir = .tmp + +[environment] +TEST_DIFF_CANONIFIER=%(testbase)s/diff-double-x +%TEST-END-FILE + +%TEST-START-FILE diff-double-x +#! /usr/bin/env bash +sed 's/x/xx/g' +%TEST-END-FILE