When updating baselines, canonify them

btest-diff

@@ -12,10 +12,67 @@
 #   TEST_DIFF_CANONIFIER
 #   TEST_DIFF_BRIEF
 #   TEST_DIFF_FILE_MAX_LINES
+#
+# This script has the following exit codes:
+#
+# When TEST_MODE is TEST:
+# 0 - Comparison succeeded, files are the same
+# 1 - Problems with input file/args or running TEST_DIFF_CANONIFIER, or file contents differ
+# 2 - Other diffing trouble (inherited from diff)
+# 100 - No baseline to compare to available
+#
+# When TEST_MODE is UPDATE:
+# 0 - Baseline updated
+# 1 - Problems with input file/args or running TEST_DIFF_CANONIFIER
+#
+# When TEST_MODE is UPDATE_INTERACTIVE:
+# 0 - Baseline updated, or nothing to update
+# 1 - Problems with input file/args or running TEST_DIFF_CANONIFIER, or user skips a deviating baseline
+# 200 - User asks to abort after a deviating baseline
+#
+# Otherwise: exits with 1
+
+# It's okay to check $? explicitly:
+# shellcheck disable=SC2181
 
 # Maximum number of lines to show from mismatching input file by default.
 MAX_LINES=100
 
+# Header line we tuck onto new baselines generated by
+# btest-diff. Serves both as a warning and as an indicator that the
+# baseline has been run through the TEST_DIFF_CANONIFIER (if any).
+HEADER="### btest baseline data generated by btest-diff. Do not edit. Use btest -U/-u to update."
+
+# Predicate, succeeds if the given baseline is canonicalized.
+is_canon_baseline() {
+    local input="$1"
+
+    if head -n 1 "$input" | grep -q -F "$HEADER" 2>/dev/null; then
+        return 0
+    fi
+
+    return 1
+}
+
+# Prints the requested baseline to standard out if it is
+# canonicalized. Otherwise, fails.
+get_canon_baseline() {
+    local input="$1"
+
+    ! is_canon_baseline "$input" && return 1
+    tail -n +2 "$input"
+}
+
+# Updates the given baseline to the given destination, prepending our
+# header.
+update_canon_baseline() {
+    local input="$1"
+    local output="$2"
+
+    echo "$HEADER" >"$output"
+    cat "$input" >>"$output"
+}
+
 if [ -n "$TEST_DIFF_FILE_MAX_LINES" ]; then
     MAX_LINES=$TEST_DIFF_FILE_MAX_LINES
 fi
@@ -25,22 +82,23 @@ if [ "$TEST_DIAGNOSTICS" = "" ]; then
 fi
 
 if [ "$#" -lt 1 ]; then
-   echo "btest-diff: wrong number of arguments" >$TEST_DIAGNOSTICS
-   exit 1
+    echo "btest-diff: wrong number of arguments" >$TEST_DIAGNOSTICS
+    exit 1
 fi
 
-input=$1
-canon=`echo $input | sed 's#/#.#g'`
+input="$1"
+# shellcheck disable=SC2001
+canon=$(echo "$input" | sed 's#/#.#g')
 shift
 
-if [ ! -f $input ]; then
-   echo "btest-diff: input $input does not exist." >$TEST_DIAGNOSTICS
-   exit 1
+if [ ! -f "$input" ]; then
+    echo "btest-diff: input $input does not exist." >$TEST_DIAGNOSTICS
+    exit 1
 fi
 
 tmpfiles=""
-function delete_tmps {
-     rm -f $tmpfiles 2>/dev/null
+delete_tmps() {
+    rm -f $tmpfiles 2>/dev/null
 }
 
 trap delete_tmps 0
@@ -51,84 +109,106 @@ rm -f $TEST_DIAGNOSTICS  2>/dev/null
 
 echo "== File ===============================" >>$TEST_DIAGNOSTICS
 
-if [ ! -e $TEST_BASELINE/$canon ]; then
-    cat $input >>$TEST_DIAGNOSTICS
+if [ ! -e "$TEST_BASELINE/$canon" ]; then
+    cat "$input" >>$TEST_DIAGNOSTICS
 elif [ -n "$TEST_DIFF_BRIEF" ]; then
     echo "<Content not shown>" >>$TEST_DIAGNOSTICS
 else
-    if [ `wc -l $input | awk '{print $1}'` -le $MAX_LINES ]; then
-        cat $input >>$TEST_DIAGNOSTICS
+    if [ "$(wc -l "$input" | awk '{print $1}')" -le "$MAX_LINES" ]; then
+        cat "$input" >>$TEST_DIAGNOSTICS
     else
-        head -$MAX_LINES $input >>$TEST_DIAGNOSTICS
+        head -n "$MAX_LINES" "$input" >>$TEST_DIAGNOSTICS
         echo "[... File too long, truncated ...]" >>$TEST_DIAGNOSTICS
     fi
 fi
 
-if [ -e $TEST_BASELINE/$canon ]; then
-   error=0
-
-   if [ "$TEST_DIFF_CANONIFIER" != "" ]; then
-       diff1=/tmp/test-diff.$$.$canon.baseline.tmp
-       diff2=/tmp/test-diff.$$.$canon.tmp
-       tmpfiles="$tmpfiles $diff1 $diff2"
-
-       eval $TEST_DIFF_CANONIFIER $TEST_BASELINE/$canon <$TEST_BASELINE/$canon >$diff1
-       if [ $? -ne 0 ]; then
-           error=1
-           echo "== Error ==============================" >>$TEST_DIAGNOSTICS
-           echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$TEST_BASELINE/$canon'" >>$TEST_DIAGNOSTICS
-           result=1
-       fi
-
-       eval $TEST_DIFF_CANONIFIER $input <$input >$diff2
-       if [ $? -ne 0 ]; then
-           error=1
-           echo "== Error ==============================" >>$TEST_DIAGNOSTICS
-           echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$input'" >>$TEST_DIAGNOSTICS
-           result=1
-       fi
-   else
-       diff1=$TEST_BASELINE/$canon
-       diff2=$input
-   fi
-
-   if [ $error -eq 0 ]; then
-       echo "== Diff ===============================" >>$TEST_DIAGNOSTICS
-       diff -au $@ $diff1 $diff2 >>$TEST_DIAGNOSTICS
-       result=$?
-   fi
-else
-   echo "== Error ==============================" >>$TEST_DIAGNOSTICS
-   echo "test-diff: no baseline found." >>$TEST_DIAGNOSTICS
-   result=100
+# If no canonifier is defined, just copy. Simplifies code layout.
+if [ -z "$TEST_DIFF_CANONIFIER" ]; then
+    TEST_DIFF_CANONIFIER="cat"
+fi
+
+canon_output=/tmp/test-diff.$$.$canon.tmp
+tmpfiles="$tmpfiles $canon_output"
+error=0
+
+# Canonicalize the new test output.
+# shellcheck disable=SC2094
+eval "$TEST_DIFF_CANONIFIER" "$input" <"$input" >"$canon_output"
+if [ $? -ne 0 ]; then
+    echo "== Error ==============================" >>$TEST_DIAGNOSTICS
+    echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$input'" >>$TEST_DIAGNOSTICS
+    error=1
+    result=1
+fi
+
+if [ -e "$TEST_BASELINE/$canon" ]; then
+    canon_baseline=/tmp/test-diff.$$.$canon.baseline.tmp
+    tmpfiles="$tmpfiles $canon_baseline"
+
+    # Prepare the baseline. When created by a recent btest-diff, we
+    # don't need to re-canonicalize, otherwise we do.
+    if ! get_canon_baseline "$TEST_BASELINE/$canon" >"$canon_baseline"; then
+        # It's an older uncanonicalized baseline, so canonicalize
+        # it now prior to comparison. Future updates via btest
+        # -U/-u will then store it canonicalized.
+        eval "$TEST_DIFF_CANONIFIER" <"$TEST_BASELINE/$canon" >"$canon_baseline"
+        if [ $? -ne 0 ]; then
+            echo "== Error ==============================" >>$TEST_DIAGNOSTICS
+            echo "btest-diff: TEST_DIFF_CANONIFIER failed on file '$TEST_BASELINE/$canon'" >>$TEST_DIAGNOSTICS
+            error=1
+            result=1
+        fi
+    fi
+
+    if [ $error -eq 0 ]; then
+        echo "== Diff ===============================" >>$TEST_DIAGNOSTICS
+        diff -au "$@" "$canon_baseline" "$canon_output" >>$TEST_DIAGNOSTICS
+        result=$?
+    fi
+elif [ "$TEST_MODE" = "TEST" ]; then
+    echo "== Error ==============================" >>$TEST_DIAGNOSTICS
+    echo "test-diff: no baseline found." >>$TEST_DIAGNOSTICS
+    result=100
 fi
 
 echo "=======================================" >>$TEST_DIAGNOSTICS
 
-if [ "$TEST_MODE" == "TEST" ]; then
-   exit $result
+if [ "$TEST_MODE" = "TEST" ]; then
+    exit $result
+
+elif [ "$TEST_MODE" = "UPDATE_INTERACTIVE" ]; then
+
+    # We had a problem running the canonifier
+    if [ "$error" != 0 ]; then
+        exit 1
+    fi
+
+    # There's no change to the baseline, so skip user interaction
+    if [ "$result" = 0 ]; then
+        exit 0
+    fi
 
-elif [ "$TEST_MODE" == "UPDATE_INTERACTIVE" ]; then
+    btest-ask-update
+    rc=$?
 
-   if [ "$result" == 0 ]; then
-       exit 0
-   fi
+    echo -n "$TEST_NAME ..." >/dev/tty
 
-   btest-ask-update
-   rc=$?
+    if [ $rc = 0 ]; then
+        update_canon_baseline "$canon_output" "$TEST_BASELINE/$canon"
+        exit 0
+    fi
 
-   echo -n "$TEST_NAME ..." >/dev/tty
+    exit $rc
 
-   if [ $rc == 0 ]; then
-       cp $input $TEST_BASELINE/$canon
-       exit 0
-   fi
+elif [ "$TEST_MODE" = "UPDATE" ]; then
 
-   exit $rc
+    # We had a problem running the canonifier
+    if [ "$error" != 0 ]; then
+        exit 1
+    fi
 
-elif [ "$TEST_MODE" == "UPDATE" ]; then
-   cp $input $TEST_BASELINE/$canon
-   exit 0
+    update_canon_baseline "$canon_output" "$TEST_BASELINE/$canon"
+    exit 0
 fi
 
 echo "test-diff: unknown test mode $TEST_MODE" >$TEST_DIAGNOSTICS

testing/tests/canonifier-conversion.test

@@ -0,0 +1,52 @@
+# This test verifies that we only canonicalize baselines once, namely
+# when canonicalizing new test output, and that baselines get
+# converted over to our header-tagged format.
+#
+# Test prep: make our canonifier executable
+# %TEST-EXEC: chmod 755 ./diff-double-x
+#
+# Verify that without an existing baseline, we canonify a new one.
+# %TEST-EXEC: btest -U %INPUT
+# %TEST-EXEC: head -1 Baseline/canonifier-conversion/output | grep -q 'Do not edit'
+# %TEST-EXEC: tail -1 Baseline/canonifier-conversion/output | grep xx
+# %TEST-EXEC: cp Baseline/canonifier-conversion/output base.1
+#
+# For testing conversion, first create a "dated" baseline.
+# %TEST-EXEC: echo x >Baseline/canonifier-conversion/output
+#
+# Verify that it succeeds:
+# %TEST-EXEC: btest %INPUT
+#
+# Update the baseline. This should prefix btest-diff's header and canonify:
+# %TEST-EXEC: btest -U %INPUT
+#
+# Verify that these have happened and preserve baseline:
+# %TEST-EXEC: head -1 Baseline/canonifier-conversion/output | grep -q 'Do not edit'
+# %TEST-EXEC: tail -1 Baseline/canonifier-conversion/output | grep xx
+# %TEST-EXEC: cp Baseline/canonifier-conversion/output base.2
+#
+# Verify that it still succeeds:
+# %TEST-EXEC: btest %INPUT
+#
+# Update the baseline again.
+# %TEST-EXEC: btest -U %INPUT
+# %TEST-EXEC: cp Baseline/canonifier-conversion/output base.3
+#
+# Verify the updated baselines remained identical.
+# %TEST-EXEC: test "$(cat base.1)" = "$(cat base.2)" && test "$(cat base.2)" = "$(cat base.3)"
+
+@TEST-EXEC: echo x >output
+@TEST-EXEC: btest-diff output
+
+%TEST-START-FILE btest.cfg
+[btest]
+TmpDir      = .tmp
+
+[environment]
+TEST_DIFF_CANONIFIER=%(testbase)s/diff-double-x
+%TEST-END-FILE
+
+%TEST-START-FILE diff-double-x
+#! /usr/bin/env bash
+sed 's/x/xx/g'
+%TEST-END-FILE