Use assertions inside test/ui.

Scripts for the UI tests now have assert_* functions, so they can verify the
expected results from the commands in the script itself, instead of comparing
their output with the *.output files.

src/tests.rs

@@ -1,7 +1,4 @@
-//! Run scripts in `tests/ui/*.sh`, and compare their outputs with the
-//! `.output` files.
-//!
-//! This code was adapted from the `bash_builtins` crate.
+//! Run scripts in `tests/ui/*.sh`.
 
 use std::env;
 use std::ffi::OsStr;
@@ -15,16 +12,6 @@ use serde_json as json;
 /// Variable to send the path of the test file.
 const TEST_FILE_VAR: &str = "TEST_FILE_PATH";
 
-macro_rules! check_output {
-    ($output:expr) => {
-        if !$output.status.success() {
-            std::io::stdout().write_all(&$output.stdout).unwrap();
-            std::io::stderr().write_all(&$output.stderr).unwrap();
-            panic!("command failed");
-        }
-    };
-}
-
 /// Invoke `cargo-build` and read its output in JSON format.
 ///
 /// Cargo should send the path of the `.so` file in a `"compiler-artifact"`
@@ -38,7 +25,9 @@ fn build() -> PathBuf {
         .output()
         .unwrap();
 
-    check_output!(build);
+    if !build.status.success() {
+        panic!("cargo-build failed");
+    }
 
     for line in build.stdout.split(|c| *c == b'\n') {
         if let Ok(msg) = json::from_slice::<json::Value>(line) {
@@ -67,10 +56,16 @@ fn create_runner_file(target: &Path) -> PathBuf {
         &mut output,
         "
             exec 2>&1
+            set -euo pipefail
+
             load_builtin() {{
                 enable -f '{}' timehistory
             }}
 
+            for lib in tests/ui/_*.sh; do
+                source $lib
+            done
+
             source ${}
         ",
         build().display(),
@@ -112,11 +107,9 @@ fn check_ui() {
             continue;
         }
 
-        let expected_output = {
-            let mut exp_path = path.clone();
-            exp_path.set_extension("output");
-            fs::read_to_string(exp_path).unwrap_or_default()
-        };
+        if path.file_name().unwrap().to_string_lossy().starts_with("_") {
+            continue;
+        }
 
         let bash = Command::new("bash")
             .env("LC_ALL", "C")
@@ -129,28 +122,20 @@ fn check_ui() {
 
         // Wait until the script is done.
         let output = bash.wait_with_output().unwrap();
-        check_output!(output);
-
-        // Capture output and compare with the expected one.
-        //
-        // The path of the runner script is replaced with "$RUNNER".
-        let test_output = String::from_utf8(output.stdout)
-            .unwrap()
-            .replace(test_runner.to_str().unwrap_or_default(), "$RUNNER");
-
-        if test_output != expected_output {
-            let test_name = path.file_name().unwrap();
-            let mut output_copy = target.join(test_name);
-            output_copy.set_extension("current-output");
-
+        if !output.status.success() {
             failed += 1;
-            eprintln!("### {}: failed", path.display());
-            eprintln!("=== OUTPUT ({})\n{}\n", output_copy.display(), test_output);
-            eprintln!("=== EXPECTED\n{}\n", expected_output);
 
-            fs::write(output_copy, test_output).unwrap();
+            let header = format!("==== [FAIL] {} ====", path.display());
+            let footer = (0..header.len()).map(|_| '=').collect::<String>();
+
+            eprintln!(
+                "{}\n{}\n{}",
+                header,
+                String::from_utf8_lossy(&output.stdout),
+                footer
+            );
         }
-    }
 
-    assert_eq!(failed, 0);
+        assert_eq!(failed, 0);
+    }
 }

tests/ui/_assert.sh

@@ -0,0 +1,45 @@
+#! bash
+#
+# assert functions to be used in *.sh tests.
+
+ASSERT_OUTPUT() {
+  local cmd="$1"
+  local expected="${2:-}";
+
+  # Read expected output from std if there is only one argument.
+  if [ $# -eq 1 ]
+  then
+    while read -sr
+    do
+      if [ -n "$expected" ]
+      then
+        expected+=$'\n'
+      fi
+
+      expected+="$REPLY"
+    done
+  fi
+
+  local output=$(eval "$cmd" 2>&1)
+
+  if [ "$output" != "$expected" ]
+  then
+    printf 'assertion failed: invalid output for "%s"\n' "$cmd"
+    printf ' expected: %q\n' "$expected"
+    printf '   output: %q\n' "$output"
+
+    exit 1
+  fi
+
+  return 0
+}
+
+ASSERT_FAILS() {
+  if eval "$*"
+  then
+    printf 'assertion failed: "%s" should fail\n' "$*"
+    exit 1
+  fi
+
+  return 0
+}

tests/ui/change-config.sh

@@ -1,9 +1,17 @@
+# bash
+# Test to change builtin configuration.
+
 load_builtin
 
 timehistory -L 5000 -F '%n\t%P\t%C'
-timehistory -C
+
+ASSERT_OUTPUT \
+  "timehistory -C" \
+  $'-L 5000 -F \'%n\\\\t%P\\\\t%C\''
 
 timehistory -F '> %C'
 
 command expr 1 + 2
-timehistory
+ASSERT_OUTPUT \
+  "timehistory" \
+  "> expr 1 '+' 2"

tests/ui/clear-history.sh

@@ -1,9 +1,9 @@
+# Test for the clear history option.
+
 load_builtin
 
 /bin/true
-timehistory -f '%C'
+test -n "$(timehistory)"
 
-echo .
 timehistory -R
-timehistory
-echo .
+test -z "$(timehistory)"

tests/ui/remove-after-panic.sh

@@ -1,16 +1,22 @@
+# Test to use the shell session after a panic in the builtin.
+
 load_builtin
 
 type timehistory
 
 /bin/true
-timehistory -f '- %n'
+ASSERT_OUTPUT \
+  "timehistory -f '- %n'" \
+  "- 1"
 
 # Force a panic.
-timehistory -P &> /dev/null
-timehistory
+ASSERT_FAILS timehistory -P
+ASSERT_FAILS timehistory
 
 /bin/echo 2
 enable -d timehistory
-type timehistory
+ASSERT_FAILS type timehistory
 
-/bin/echo 3
+ASSERT_OUTPUT \
+  "/bin/echo 3" \
+  "3"

tests/ui/remove.sh

@@ -1,11 +1,20 @@
+# Test to use the shell session after removing the builtin.
+
 load_builtin
 
 /bin/echo 1
-timehistory -f '- %n'
+ASSERT_OUTPUT \
+  "timehistory -f '- %n'" \
+  "- 1"
 
 enable -d timehistory
 /bin/echo 2
 
+ASSERT_FAILS timehistory
+
 load_builtin
 /bin/echo 3
-timehistory -f '- %n'
+
+ASSERT_OUTPUT \
+  "timehistory -f '> %n'" \
+  "> 1"

tests/ui/rotate-history.sh

@@ -1,3 +1,5 @@
+# Test for the history limit.
+
 load_builtin
 
 timehistory -L 5
@@ -6,4 +8,12 @@ do
   /bin/true $N
 done
 
-timehistory -f '%n\t%C'
+ASSERT_OUTPUT \
+  "timehistory -f '%n,%C'" \
+  <<-ITEMS
+	6,/bin/true 6
+	7,/bin/true 7
+	8,/bin/true 8
+	9,/bin/true 9
+	10,/bin/true 10
+ITEMS

tests/ui/single-entry.sh

@@ -1,10 +1,18 @@
+# Test to print a single entry from the history.
+
 load_builtin
 
 /bin/true 1
 /bin/true 2
 /bin/true 3
 
-timehistory -f '%n %C' 2
-timehistory -f '%n %C' +1
+ASSERT_OUTPUT \
+  "timehistory -f '%n %C' 2" \
+  "2 /bin/true 2"
+
+ASSERT_OUTPUT \
+  "timehistory -f '%n %C' +1" \
+  "3 /bin/true 3"
 
-timehistory x || :
+( timehistory x 2>&1 || : ) \
+  | grep 'timehistory: invalid digit found in string'

tests/ui/track-subshell.sh

@@ -1,3 +1,5 @@
+# Test to track commands in a subshell.
+
 load_builtin
 
 timehistory -F '%C'
@@ -8,4 +10,10 @@ timehistory -F '%C'
 ) &
 
 wait
-timehistory -f '%Tx,%C'
+ASSERT_OUTPUT \
+  "timehistory -f '%Tx,%C'" \
+  <<-ITEMS
+	0,/bin/echo 1
+	0,/bin/echo 2
+	1,/bin/false
+ITEMS