From 01cffddffa5bfda9cc11892dd288101291b38291 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 7 Oct 2020 15:56:29 -0700 Subject: [PATCH 1/4] Consistent indentation for btest-diff, no other changes --- btest-diff | 108 ++++++++++++++++++++++++++--------------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/btest-diff b/btest-diff index 72557d6b..601b3e9a 100755 --- a/btest-diff +++ b/btest-diff @@ -25,8 +25,8 @@ if [ "$TEST_DIAGNOSTICS" = "" ]; then fi if [ "$#" -lt 1 ]; then - echo "btest-diff: wrong number of arguments" >$TEST_DIAGNOSTICS - exit 1 + echo "btest-diff: wrong number of arguments" >$TEST_DIAGNOSTICS + exit 1 fi input=$1 @@ -34,13 +34,13 @@ canon=`echo $input | sed 's#/#.#g'` shift if [ ! -f $input ]; then - echo "btest-diff: input $input does not exist." >$TEST_DIAGNOSTICS - exit 1 + echo "btest-diff: input $input does not exist." >$TEST_DIAGNOSTICS + exit 1 fi tmpfiles="" function delete_tmps { - rm -f $tmpfiles 2>/dev/null + rm -f $tmpfiles 2>/dev/null } trap delete_tmps 0 @@ -65,70 +65,70 @@ else fi if [ -e $TEST_BASELINE/$canon ]; then - error=0 - - if [ "$TEST_DIFF_CANONIFIER" != "" ]; then - diff1=/tmp/test-diff.$$.$canon.baseline.tmp - diff2=/tmp/test-diff.$$.$canon.tmp - tmpfiles="$tmpfiles $diff1 $diff2" - - eval $TEST_DIFF_CANONIFIER $TEST_BASELINE/$canon <$TEST_BASELINE/$canon >$diff1 - if [ $? -ne 0 ]; then - error=1 - echo "== Error ==============================" >>$TEST_DIAGNOSTICS - echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$TEST_BASELINE/$canon'" >>$TEST_DIAGNOSTICS - result=1 - fi - - eval $TEST_DIFF_CANONIFIER $input <$input >$diff2 - if [ $? -ne 0 ]; then - error=1 - echo "== Error ==============================" >>$TEST_DIAGNOSTICS - echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$input'" >>$TEST_DIAGNOSTICS - result=1 - fi - else - diff1=$TEST_BASELINE/$canon - diff2=$input - fi - - if [ $error -eq 0 ]; then - echo "== Diff ===============================" >>$TEST_DIAGNOSTICS - diff -au $@ $diff1 $diff2 >>$TEST_DIAGNOSTICS - result=$? - fi + error=0 + + if [ "$TEST_DIFF_CANONIFIER" != "" ]; then + diff1=/tmp/test-diff.$$.$canon.baseline.tmp + diff2=/tmp/test-diff.$$.$canon.tmp + tmpfiles="$tmpfiles $diff1 $diff2" + + eval $TEST_DIFF_CANONIFIER $TEST_BASELINE/$canon <$TEST_BASELINE/$canon >$diff1 + if [ $? -ne 0 ]; then + error=1 + echo "== Error ==============================" >>$TEST_DIAGNOSTICS + echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$TEST_BASELINE/$canon'" >>$TEST_DIAGNOSTICS + result=1 + fi + + eval $TEST_DIFF_CANONIFIER $input <$input >$diff2 + if [ $? -ne 0 ]; then + error=1 + echo "== Error ==============================" >>$TEST_DIAGNOSTICS + echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$input'" >>$TEST_DIAGNOSTICS + result=1 + fi + else + diff1=$TEST_BASELINE/$canon + diff2=$input + fi + + if [ $error -eq 0 ]; then + echo "== Diff ===============================" >>$TEST_DIAGNOSTICS + diff -au $@ $diff1 $diff2 >>$TEST_DIAGNOSTICS + result=$? + fi else - echo "== Error ==============================" >>$TEST_DIAGNOSTICS - echo "test-diff: no baseline found." >>$TEST_DIAGNOSTICS - result=100 + echo "== Error ==============================" >>$TEST_DIAGNOSTICS + echo "test-diff: no baseline found." >>$TEST_DIAGNOSTICS + result=100 fi echo "=======================================" >>$TEST_DIAGNOSTICS if [ "$TEST_MODE" == "TEST" ]; then - exit $result + exit $result elif [ "$TEST_MODE" == "UPDATE_INTERACTIVE" ]; then - if [ "$result" == 0 ]; then - exit 0 - fi + if [ "$result" == 0 ]; then + exit 0 + fi - btest-ask-update - rc=$? + btest-ask-update + rc=$? - echo -n "$TEST_NAME ..." >/dev/tty + echo -n "$TEST_NAME ..." >/dev/tty - if [ $rc == 0 ]; then - cp $input $TEST_BASELINE/$canon - exit 0 - fi + if [ $rc == 0 ]; then + cp $input $TEST_BASELINE/$canon + exit 0 + fi - exit $rc + exit $rc elif [ "$TEST_MODE" == "UPDATE" ]; then - cp $input $TEST_BASELINE/$canon - exit 0 + cp $input $TEST_BASELINE/$canon + exit 0 fi echo "test-diff: unknown test mode $TEST_MODE" >$TEST_DIAGNOSTICS From 7a6b5e0bddd01c7029017624c76208ecf4b71bb7 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Sat, 3 Oct 2020 23:34:59 -0700 Subject: [PATCH 2/4] Canonify outputs when updating baselines via btest-diff This leverages the same canonicalization btest-diff already applies during test output comparison against baselines also when updating those baselines (usually via btest -U/-u). This removes a bunch of noise from the baselines, including personal home directories, timestamps, etc. Since btest-diff doesn't know whether the baseline has undergone this canonicalization, it continues to canonicalize the basline prior to diffing. To track whether it has canonicalized a baseline when updating, btest-diff now also prepends a header to the generated baseline that warns users about the fact that it is auto-generated. The presence of this header doubles as a marker for canonicalization. Includes a test-case. --- btest-diff | 100 ++++++++++++++++++----- testing/tests/canonifier-conversion.test | 52 ++++++++++++ 2 files changed, 130 insertions(+), 22 deletions(-) create mode 100644 testing/tests/canonifier-conversion.test diff --git a/btest-diff b/btest-diff index 601b3e9a..9648b3cc 100755 --- a/btest-diff +++ b/btest-diff @@ -16,6 +16,41 @@ # Maximum number of lines to show from mismatching input file by default. MAX_LINES=100 +# Header line we tuck onto new baselines generated by +# btest-diff. Serves both as a warning and as an indicator that the +# baseline has been run through the TEST_DIFF_CANONIFIER (if any). +HEADER="### btest baseline data generated by btest-diff. Do not edit. Use btest -U/-u to update." + +# Predicate, succeeds if the given baseline is canonicalized. +function is_canon_baseline { + local input="$1" + + if head -1 "$input" | grep -q -F "$HEADER" 2>/dev/null; then + return 0 + fi + + return 1 +} + +# Prints the requested baseline to standard out if it is +# canonicalized. Otherwise, fails. +function get_canon_baseline { + local input="$1" + + ! is_canon_baseline "$input" && return 1 + tail -n +2 "$input" +} + +# Updates the given baseline to the given destination, prepending our +# header. +function update_canon_baseline { + local input="$1" + local output="$2" + + echo "$HEADER" >"$output" + cat "$input" >>"$output" +} + if [ -n "$TEST_DIFF_FILE_MAX_LINES" ]; then MAX_LINES=$TEST_DIFF_FILE_MAX_LINES fi @@ -64,40 +99,49 @@ else fi fi -if [ -e $TEST_BASELINE/$canon ]; then - error=0 +# If no canonifier is defined, just copy. Simplifies code layout. +if [ -z "$TEST_DIFF_CANONIFIER" ]; then + TEST_DIFF_CANONIFIER="cat" +fi - if [ "$TEST_DIFF_CANONIFIER" != "" ]; then - diff1=/tmp/test-diff.$$.$canon.baseline.tmp - diff2=/tmp/test-diff.$$.$canon.tmp - tmpfiles="$tmpfiles $diff1 $diff2" +canon_output=/tmp/test-diff.$$.$canon.tmp +tmpfiles="$tmpfiles $canon_output" +error=0 - eval $TEST_DIFF_CANONIFIER $TEST_BASELINE/$canon <$TEST_BASELINE/$canon >$diff1 +# Canonicalize the new test output. +eval "$TEST_DIFF_CANONIFIER" "$input" <"$input" >"$canon_output" +if [ $? -ne 0 ]; then + echo "== Error ==============================" >>$TEST_DIAGNOSTICS + echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$input'" >>$TEST_DIAGNOSTICS + error=1 + result=1 +fi + +if [ -e $TEST_BASELINE/$canon ]; then + canon_baseline=/tmp/test-diff.$$.$canon.baseline.tmp + tmpfiles="$tmpfiles $canon_baseline" + + # Prepare the baseline. When created by a recent btest-diff, we + # don't need to re-canonicalize, otherwise we do. + if ! get_canon_baseline $TEST_BASELINE/$canon >$canon_baseline; then + # It's an older uncanonicalized baseline, so canonicalize + # it now prior to comparison. Future updates via btest + # -U/-u will then store it canonicalized. + eval $TEST_DIFF_CANONIFIER <$TEST_BASELINE/$canon >$canon_baseline if [ $? -ne 0 ]; then - error=1 echo "== Error ==============================" >>$TEST_DIAGNOSTICS echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$TEST_BASELINE/$canon'" >>$TEST_DIAGNOSTICS - result=1 - fi - - eval $TEST_DIFF_CANONIFIER $input <$input >$diff2 - if [ $? -ne 0 ]; then error=1 - echo "== Error ==============================" >>$TEST_DIAGNOSTICS - echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$input'" >>$TEST_DIAGNOSTICS result=1 fi - else - diff1=$TEST_BASELINE/$canon - diff2=$input fi if [ $error -eq 0 ]; then echo "== Diff ===============================" >>$TEST_DIAGNOSTICS - diff -au $@ $diff1 $diff2 >>$TEST_DIAGNOSTICS + diff -au $@ $canon_baseline $canon_output >>$TEST_DIAGNOSTICS result=$? fi -else +elif [ "$TEST_MODE" = "TEST" ]; then echo "== Error ==============================" >>$TEST_DIAGNOSTICS echo "test-diff: no baseline found." >>$TEST_DIAGNOSTICS result=100 @@ -110,6 +154,12 @@ if [ "$TEST_MODE" == "TEST" ]; then elif [ "$TEST_MODE" == "UPDATE_INTERACTIVE" ]; then + # We had a problem running the canonifier + if [ "$error" != 0 ]; then + exit 1 + fi + + # There's no change to the baseline, so skip user interaction if [ "$result" == 0 ]; then exit 0 fi @@ -120,14 +170,20 @@ elif [ "$TEST_MODE" == "UPDATE_INTERACTIVE" ]; then echo -n "$TEST_NAME ..." >/dev/tty if [ $rc == 0 ]; then - cp $input $TEST_BASELINE/$canon + update_canon_baseline $canon_output $TEST_BASELINE/$canon exit 0 fi exit $rc elif [ "$TEST_MODE" == "UPDATE" ]; then - cp $input $TEST_BASELINE/$canon + + # We had a problem running the canonifier + if [ "$error" != 0 ]; then + exit 1 + fi + + update_canon_baseline $canon_output $TEST_BASELINE/$canon exit 0 fi diff --git a/testing/tests/canonifier-conversion.test b/testing/tests/canonifier-conversion.test new file mode 100644 index 00000000..c1c0e486 --- /dev/null +++ b/testing/tests/canonifier-conversion.test @@ -0,0 +1,52 @@ +# This test verifies that we only canonicalize baselines once, namely +# when canonicalizing new test output, and that baselines get +# converted over to our header-tagged format. +# +# Test prep: make our canonifier executable +# %TEST-EXEC: chmod 755 ./diff-double-x +# +# Verify that without an existing baseline, we canonify a new one. +# %TEST-EXEC: btest -U %INPUT +# %TEST-EXEC: head -1 Baseline/canonifier-conversion/output | grep -q 'Do not edit' +# %TEST-EXEC: tail -1 Baseline/canonifier-conversion/output | grep xx +# %TEST-EXEC: cp Baseline/canonifier-conversion/output base.1 +# +# For testing conversion, first create a "dated" baseline. +# %TEST-EXEC: echo x >Baseline/canonifier-conversion/output +# +# Verify that it succeeds: +# %TEST-EXEC: btest %INPUT +# +# Update the baseline. This should prefix btest-diff's header and canonify: +# %TEST-EXEC: btest -U %INPUT +# +# Verify that these have happened and preserve baseline: +# %TEST-EXEC: head -1 Baseline/canonifier-conversion/output | grep -q 'Do not edit' +# %TEST-EXEC: tail -1 Baseline/canonifier-conversion/output | grep xx +# %TEST-EXEC: cp Baseline/canonifier-conversion/output base.2 +# +# Verify that it still succeeds: +# %TEST-EXEC: btest %INPUT +# +# Update the baseline again. +# %TEST-EXEC: btest -U %INPUT +# %TEST-EXEC: cp Baseline/canonifier-conversion/output base.3 +# +# Verify the updated baselines remained identical. +# %TEST-EXEC: test "$(cat base.1)" = "$(cat base.2)" && test "$(cat base.2)" = "$(cat base.3)" + +@TEST-EXEC: echo x >output +@TEST-EXEC: btest-diff output + +%TEST-START-FILE btest.cfg +[btest] +TmpDir = .tmp + +[environment] +TEST_DIFF_CANONIFIER=%(testbase)s/diff-double-x +%TEST-END-FILE + +%TEST-START-FILE diff-double-x +#! /usr/bin/env bash +sed 's/x/xx/g' +%TEST-END-FILE From 0ab5004e9668f2632eb6f0f55e120dc7c04234a1 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 7 Oct 2020 16:25:23 -0700 Subject: [PATCH 3/4] A shellcheck pass on btest-diff --- btest-diff | 53 +++++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/btest-diff b/btest-diff index 9648b3cc..79d44c04 100755 --- a/btest-diff +++ b/btest-diff @@ -12,6 +12,9 @@ # TEST_DIFF_CANONIFIER # TEST_DIFF_BRIEF # TEST_DIFF_FILE_MAX_LINES +# +# It's okay to check $? explicitly: +# shellcheck disable=SC2181 # Maximum number of lines to show from mismatching input file by default. MAX_LINES=100 @@ -22,10 +25,10 @@ MAX_LINES=100 HEADER="### btest baseline data generated by btest-diff. Do not edit. Use btest -U/-u to update." # Predicate, succeeds if the given baseline is canonicalized. -function is_canon_baseline { +is_canon_baseline() { local input="$1" - if head -1 "$input" | grep -q -F "$HEADER" 2>/dev/null; then + if head -n 1 "$input" | grep -q -F "$HEADER" 2>/dev/null; then return 0 fi @@ -34,7 +37,7 @@ function is_canon_baseline { # Prints the requested baseline to standard out if it is # canonicalized. Otherwise, fails. -function get_canon_baseline { +get_canon_baseline() { local input="$1" ! is_canon_baseline "$input" && return 1 @@ -43,7 +46,7 @@ function get_canon_baseline { # Updates the given baseline to the given destination, prepending our # header. -function update_canon_baseline { +update_canon_baseline() { local input="$1" local output="$2" @@ -64,17 +67,18 @@ if [ "$#" -lt 1 ]; then exit 1 fi -input=$1 -canon=`echo $input | sed 's#/#.#g'` +input="$1" +# shellcheck disable=SC2001 +canon=$(echo "$input" | sed 's#/#.#g') shift -if [ ! -f $input ]; then +if [ ! -f "$input" ]; then echo "btest-diff: input $input does not exist." >$TEST_DIAGNOSTICS exit 1 fi tmpfiles="" -function delete_tmps { +delete_tmps() { rm -f $tmpfiles 2>/dev/null } @@ -86,15 +90,15 @@ rm -f $TEST_DIAGNOSTICS 2>/dev/null echo "== File ===============================" >>$TEST_DIAGNOSTICS -if [ ! -e $TEST_BASELINE/$canon ]; then - cat $input >>$TEST_DIAGNOSTICS +if [ ! -e "$TEST_BASELINE/$canon" ]; then + cat "$input" >>$TEST_DIAGNOSTICS elif [ -n "$TEST_DIFF_BRIEF" ]; then echo "" >>$TEST_DIAGNOSTICS else - if [ `wc -l $input | awk '{print $1}'` -le $MAX_LINES ]; then - cat $input >>$TEST_DIAGNOSTICS + if [ "$(wc -l "$input" | awk '{print $1}')" -le "$MAX_LINES" ]; then + cat "$input" >>$TEST_DIAGNOSTICS else - head -$MAX_LINES $input >>$TEST_DIAGNOSTICS + head -n "$MAX_LINES" "$input" >>$TEST_DIAGNOSTICS echo "[... File too long, truncated ...]" >>$TEST_DIAGNOSTICS fi fi @@ -109,6 +113,7 @@ tmpfiles="$tmpfiles $canon_output" error=0 # Canonicalize the new test output. +# shellcheck disable=SC2094 eval "$TEST_DIFF_CANONIFIER" "$input" <"$input" >"$canon_output" if [ $? -ne 0 ]; then echo "== Error ==============================" >>$TEST_DIAGNOSTICS @@ -117,17 +122,17 @@ if [ $? -ne 0 ]; then result=1 fi -if [ -e $TEST_BASELINE/$canon ]; then +if [ -e "$TEST_BASELINE/$canon" ]; then canon_baseline=/tmp/test-diff.$$.$canon.baseline.tmp tmpfiles="$tmpfiles $canon_baseline" # Prepare the baseline. When created by a recent btest-diff, we # don't need to re-canonicalize, otherwise we do. - if ! get_canon_baseline $TEST_BASELINE/$canon >$canon_baseline; then + if ! get_canon_baseline "$TEST_BASELINE/$canon" >"$canon_baseline"; then # It's an older uncanonicalized baseline, so canonicalize # it now prior to comparison. Future updates via btest # -U/-u will then store it canonicalized. - eval $TEST_DIFF_CANONIFIER <$TEST_BASELINE/$canon >$canon_baseline + eval "$TEST_DIFF_CANONIFIER" <"$TEST_BASELINE/$canon" >"$canon_baseline" if [ $? -ne 0 ]; then echo "== Error ==============================" >>$TEST_DIAGNOSTICS echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$TEST_BASELINE/$canon'" >>$TEST_DIAGNOSTICS @@ -138,7 +143,7 @@ if [ -e $TEST_BASELINE/$canon ]; then if [ $error -eq 0 ]; then echo "== Diff ===============================" >>$TEST_DIAGNOSTICS - diff -au $@ $canon_baseline $canon_output >>$TEST_DIAGNOSTICS + diff -au "$@" "$canon_baseline" "$canon_output" >>$TEST_DIAGNOSTICS result=$? fi elif [ "$TEST_MODE" = "TEST" ]; then @@ -149,10 +154,10 @@ fi echo "=======================================" >>$TEST_DIAGNOSTICS -if [ "$TEST_MODE" == "TEST" ]; then +if [ "$TEST_MODE" = "TEST" ]; then exit $result -elif [ "$TEST_MODE" == "UPDATE_INTERACTIVE" ]; then +elif [ "$TEST_MODE" = "UPDATE_INTERACTIVE" ]; then # We had a problem running the canonifier if [ "$error" != 0 ]; then @@ -160,7 +165,7 @@ elif [ "$TEST_MODE" == "UPDATE_INTERACTIVE" ]; then fi # There's no change to the baseline, so skip user interaction - if [ "$result" == 0 ]; then + if [ "$result" = 0 ]; then exit 0 fi @@ -169,21 +174,21 @@ elif [ "$TEST_MODE" == "UPDATE_INTERACTIVE" ]; then echo -n "$TEST_NAME ..." >/dev/tty - if [ $rc == 0 ]; then - update_canon_baseline $canon_output $TEST_BASELINE/$canon + if [ $rc = 0 ]; then + update_canon_baseline "$canon_output" "$TEST_BASELINE/$canon" exit 0 fi exit $rc -elif [ "$TEST_MODE" == "UPDATE" ]; then +elif [ "$TEST_MODE" = "UPDATE" ]; then # We had a problem running the canonifier if [ "$error" != 0 ]; then exit 1 fi - update_canon_baseline $canon_output $TEST_BASELINE/$canon + update_canon_baseline "$canon_output" "$TEST_BASELINE/$canon" exit 0 fi From a7b772ca24b275263209c854e1399a3729a14f4e Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Thu, 8 Oct 2020 12:07:50 -0700 Subject: [PATCH 4/4] Document btest-diff's exit codes --- btest-diff | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/btest-diff b/btest-diff index 79d44c04..05bed144 100755 --- a/btest-diff +++ b/btest-diff @@ -13,6 +13,25 @@ # TEST_DIFF_BRIEF # TEST_DIFF_FILE_MAX_LINES # +# This script has the following exit codes: +# +# When TEST_MODE is TEST: +# 0 - Comparison succeeded, files are the same +# 1 - Problems with input file/args or running TEST_DIFF_CANONIFIER, or file contents differ +# 2 - Other diffing trouble (inherited from diff) +# 100 - No baseline to compare to available +# +# When TEST_MODE is UPDATE: +# 0 - Baseline updated +# 1 - Problems with input file/args or running TEST_DIFF_CANONIFIER +# +# When TEST_MODE is UPDATE_INTERACTIVE: +# 0 - Baseline updated, or nothing to update +# 1 - Problems with input file/args or running TEST_DIFF_CANONIFIER, or user skips a deviating baseline +# 200 - User asks to abort after a deviating baseline +# +# Otherwise: exits with 1 + # It's okay to check $? explicitly: # shellcheck disable=SC2181