Update tests

tests/decoder/intgemm/.gitignore

@@ -2,3 +2,5 @@
 *.out.bleu
 *.src
 *.ref
+*.bleu_score
+*.bleu_score.expected

tests/decoder/intgemm/test_intgemm_8bit.sh

@@ -47,10 +47,16 @@ $MRT_MARIAN/marian-decoder \
 
 # Print current and expected BLEU for debugging
 python3 $MRT_TOOLS/sacrebleu/sacrebleu.py newstest2018.ref < $prefix.out | tee $prefix.out.bleu
-cat $prefix.$suffix.expected.bleu
+# BLEU scores calculated on AVX, AVX2, AVX512 should be very similar, but does not have to be identical
+tail -n1 $prefix.*.expected.bleu || true
 
-# Compare with the expected output
-$MRT_TOOLS/diff.sh $prefix.out $prefix.$suffix.expected > $prefix.diff
+# Compare with the expected output, allow up to 20 different lines
+$MRT_TOOLS/diff.sh $prefix.out $prefix.$suffix.expected 20 > $prefix.diff
+
+# Compare BLEU scores
+cut -f3 -d' ' $prefix.out.bleu > $prefix.out.bleu_score
+cut -f3 -d' ' $prefix.$suffix.expected.bleu > $prefix.out.bleu_score.expected
+$MRT_TOOLS/diff-nums.py -a -p 0.9 $prefix.out.bleu_score{,.expected}
 
 
 # Exit with success code

tests/decoder/intgemm/test_intgemm_8bit_avx2.sh

@@ -48,8 +48,13 @@ python3 $MRT_TOOLS/sacrebleu/sacrebleu.py newstest2018.ref < $prefix.out | tee $
 # BLEU scores calculated on AVX, AVX2, AVX512 should be very similar, but does not have to be identical
 tail -n1 $prefix.*.expected.bleu || true
 
-# Compare with the expected output
-$MRT_TOOLS/diff.sh $prefix.out $prefix.$suffix.expected > $prefix.diff
+# Compare with the expected output, allow up to 20 different lines
+$MRT_TOOLS/diff.sh $prefix.out $prefix.$suffix.expected 20 > $prefix.diff
+
+# Compare BLEU scores
+cut -f3 -d' ' $prefix.out.bleu > $prefix.out.bleu_score
+cut -f3 -d' ' $prefix.$suffix.expected.bleu > $prefix.out.bleu_score.expected
+$MRT_TOOLS/diff-nums.py -a -p 0.9 $prefix.out.bleu_score{,.expected}
 
 
 # Exit with success code

tools/diff.sh

@@ -1,3 +1,19 @@
 #!/bin/bash
-[[ "$#" -eq 2 ]] && >&2 echo "Command: $(realpath $0) $(realpath -m $1) $(realpath -m $2)"
+#
+# Usage:
+#   ./diff.sh file1 file2 [number-of-allowed-diff-lines]
+
+[[ "$#" -ge 2 ]] && >&2 echo "Command: $(realpath $0) $(realpath -m $1) $(realpath -m $2)"
 diff $1 $2
+exitcode=$?
+if [ -z "$3" ]; then
+  exit $exitcode
+else
+  numlines=$(diff -y --suppress-common-lines $1 $2 | wc -l)
+  >&2 echo "Different lines: $numlines, allowed: $3"
+  if [[ "$numlines" -gt "$3" ]]; then
+    exit $exitcode
+  else
+    exit 0
+  fi
+fi