validation/test-suite

#!/bin/sh

#set -x

cd $(dirname "$0")

default_path=".."
default_cmd="sparse \$file"
default_args="$SPARSE_TEST_ARGS"
tests_list=""
prog_name=`basename $0`

if [ ! -x "$default_path/sparse-llvm" ]; then
	disabled_cmds="sparsec sparsei sparse-llvm sparse-llvm-dis"
fi
if [ ! -x "$default_path/scheck" ]; then
	disabled_cmds="$disabled_cmds scheck"
fi

# flags:
#	- some tests gave an unexpected result
failed=0

# counts:
#	- tests that have not been converted to test-suite format
#	- tests that are disabled
#	- tests that passed
#	- tests that failed
#	- tests that failed but are known to fail
unhandled_tests=0
disabled_tests=0
ok_tests=0
ko_tests=0
known_ko_tests=0

# defaults to not verbose
[ -z "$V" ] && V=0
vquiet=""
quiet=0
abort=0


##
# verbose(string) - prints string if we are in verbose mode
verbose()
{
	[ "$V" -eq "1" ] && echo "        $1"
	return 0
}

##
# warning(string) - prints a warning
warning()
{
	[ "$quiet" -ne 1 ] && echo "warning: $1"
	return 0
}

##
# error(string[, die]) - prints an error and exits with value die if given
error()
{
	[ "$quiet" -ne 1 ] && echo "error: $1"
	[ -n "$2" ] && exit $2
	return 0
}


##
# get_tag_value(file) - get the 'check-<...>' tags & values
get_tag_value()
{
	check_name=""
	check_command="$default_cmd"
	check_exit_value=0
	check_timeout=0
	check_known_to_fail=0
	check_error_ignore=0
	check_output_ignore=0
	check_output_contains=0
	check_output_excludes=0
	check_output_pattern=0
	check_output_match=0
	check_output_returns=0
	check_arch_ignore=""
	check_arch_only=""
	check_assert=""
	check_cpp_if=""

	lines=$(grep '^ \* check-[a-z-]*' $1 | \
		sed -e 's/^ \* \(check-[a-z-]*:*\) *\(.*\)$/\1 \2/')

	while read tag val; do
		#echo "-> tag: '$tag'"
		#echo "-> val: '$val'"
		case $tag in
		check-name:)		check_name="$val" ;;
		check-command:)		check_command="$val" ;;
		check-exit-value:)	check_exit_value="$val" ;;
		check-timeout:)		[ -z "$val" ] && val=1
					check_timeout="$val" ;;
		check-known-to-fail)	check_known_to_fail=1 ;;
		check-error-ignore)	check_error_ignore=1 ;;
		check-output-ignore)	check_output_ignore=1 ;;
		check-output-contains:)	check_output_contains=1 ;;
		check-output-excludes:)	check_output_excludes=1 ;;
		check-output-pattern)	check_output_pattern=1 ;;
		check-output-match)	check_output_match=1 ;;
		check-output-returns:)	check_output_returns=1 ;;
		check-arch-ignore:)	arch=$(uname -m)
					check_arch_ignore="$val" ;;
		check-arch-only:)	arch=$(uname -m)
					check_arch_only="$val" ;;
		check-assert:)		check_assert="$val" ;;
		check-cpp-if:)		check_cpp_if="$val" ;;

		check-description:)	;;	# ignore
		check-note:)		;;	# ignore
		check-warning:)		;;	# ignore
		check-error-start)	;;	# ignore
		check-error-end)	;;	# ignore
		check-output-start)	;;	# ignore
		check-output-end)	;;	# ignore
		check-should-pass)	;;	# ignore, unused annotation
		check-should-fail)	;;	# ignore, unused annotation
		check-should-warn)	;;	# ignore, unused annotation
		check-*)		error "$1: unknown tag '$tag'" 1 ;;
		esac
	done << EOT
	$lines
EOT
}

##
# helper for has_(each|none)_patterns()
has_patterns()
{
	ifile="$1"
	patt="$2"
	ofile="$3"
	cmp="$4"
	msg="$5"
	grep "$patt:" "$ifile" | \
	sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
	while read val; do
		grep -s -q "$val" "$ofile"
		if [ "$?" $cmp 0 ]; then
			error "	Pattern '$val' unexpectedly $msg"
			return 1
		fi
	done

	return $?
}

##
# has_each_patterns(ifile tag ofile) - does ofile contains some
#                        of the patterns given by ifile's tags?
#
# returns 0 if all present, 1 otherwise
has_each_patterns()
{
	has_patterns "$1" "$2" "$4" -ne "$3"
}

##
# has_none_patterns(ifile tag ofile) - does ofile contains some
#                        of the patterns given by ifile's tags?
#
# returns 1 if any present, 0 otherwise
has_none_patterns()
{
	has_patterns "$1" "$2" "$4" -eq "$3"
}

##
# minmax_patterns(ifile tag ofile) - does ofile contains the
#                        the patterns given by ifile's tags
#                        the right number of time?
minmax_patterns()
{
	ifile="$1"
	patt="$2"
	ofile="$3"
	grep "$patt([0-9-]*\(, *\)*[0-9-]*):" "$ifile" | \
	sed -e "s/^.*$patt(\([0-9]*\)): *\(.*\)/\1 eq \2/" \
	    -e "s/^.*$patt(\([0-9-]*\), *\([0-9-]*\)): *\(.*\)/\1 \2 \3/" | \
	while read min max pat; do
		n=$(grep -s "$pat" "$ofile" | wc -l)
		if [ "$max" = "eq" ]; then
		    if [ "$n" -ne "$min" ]; then
			error "	Pattern '$pat' expected $min times but got $n times"
			return 1
		    fi
		    continue
		fi
		if [ "$min" != '-' ]; then
		    if [ "$n" -lt "$min" ]; then
			error "	Pattern '$pat' expected min $min times but got $n times"
			return 1
		    fi
		fi
		if [ "$max" != '-' ]; then
		    if [ "$n" -gt "$max" ]; then
			error "	Pattern '$pat' expected max $max times but got $n times"
			return 1
		    fi
		fi
	done

	return $?
}

##
match_patterns()
{
	ifile="$1"
	patt="$2"
	ofile="$3"
	grep "$patt" "$ifile" | sed -e "s/^.*$patt(\(.*\)): *\(.*\)$/\1 \2/" | \
	while read ins pat; do
		grep -s "^	$ins\\.*[0-9]* " "$ofile" | grep -v -s -q "$pat"
		if [ "$?" -ne 1 ]; then
			error "	IR doesn't match '$pat'"
			return 1
		fi
	done

	return $?
}

##
return_patterns()
{
	ifile="$1"
	patt="$2"
	ofile="$3"
	grep "$patt:" "$ifile" | sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
	while read ret; do
		grep -s "^	ret\\.[0-9]" "$ofile" | grep -v -s -q "[ \$]${ret}\$"
		if [ "$?" -ne 1 ]; then
			error "	Return doesn't match '$ret'"
			return 1
		fi
	done

	return $?
}

##
# arg_file(filename) - checks if filename exists
arg_file()
{
	[ -z "$1" ] && {
		do_usage
		exit 1
	}
	[ -e "$1" ] || {
		error "Can't open file $1"
		exit 1
	}
	return 0
}


##
do_usage()
{
echo "$prog_name - a tiny automatic testing script"
echo "Usage: $prog_name [option(s)] [command] [arguments]"
echo
echo "options:"
echo "    -a|--abort                 Abort the tests as soon as one fails."
echo "    -q|--quiet                 Be extra quiet while running the tests."
echo "    --args='...'               Add these options to the test command."
echo
echo "commands:"
echo "    [file ...]                 Runs the test suite on the given file(s)."
echo "                               If a directory is given, run only those files."
echo "                               If no file is given, run the whole testsuite."
echo "    single file                Run the test in 'file'."
echo "    format file [name [cmd]]   Help writing a new test case using cmd."
echo
echo "    [command] help             Print usage."
}

disable()
{
	disabled_tests=$(($disabled_tests + 1))
	if [ -z "$vquiet" ]; then
		echo "  SKIP    $1 ($2)"
	fi
}

##
# do_test(file) - tries to validate a test case
#
# it "parses" file, looking for check-* tags and tries to validate
# the test against an expected result
# returns:
#	- 0 if the test passed,
#	- 1 if it failed,
#	- 2 if it is not a "test-suite" test.
#	- 3 if the test is disabled.
do_test()
{
	test_failed=0
	file="$1"
	quiet=0

	get_tag_value $file

	# can this test be handled by test-suite ?
	# (it has to have a check-name key in it)
	if [ "$check_name" = "" ]; then
		warning "$file: test unhandled"
		unhandled_tests=$(($unhandled_tests + 1))
		return 2
	fi
	test_name="$check_name"

	# does the test provide a specific command ?
	if [ "$check_command" = "" ]; then
		check_command="$defaut_command"
	fi

	# check for disabled commands
	set -- $check_command
	base_cmd=$1
	for i in $disabled_cmds; do
		if [ "$i" = "$base_cmd" ] ; then
			disable "$test_name" "$file"
			return 3
		fi
	done
	if [ "$check_arch_ignore" != "" ]; then
		if echo $arch | egrep -q -w "$check_arch_ignore"; then
			disable "$test_name" "$file"
			return 3
		fi
	fi
	if [ "$check_arch_only" != "" ]; then
		if ! (echo $arch | egrep -q -w "$check_arch_only"); then
			disable "$test_name" "$file"
			return 3
		fi
	fi
	if [ "$check_assert" != "" ]; then
		res=$(../sparse - 2>&1 >/dev/null <<- EOF
			_Static_assert($check_assert, "$check_assert");
			EOF
		)
		if [ "$res" != "" ]; then
			disable "$test_name" "$file"
			return 3
		fi
	fi
	if [ "$check_cpp_if" != "" ]; then
		res=$(../sparse -E - 2>/dev/null <<- EOF
			#if !($check_cpp_if)
			fail
			#endif
			EOF
		)
		if [ "$res" != "" ]; then
			disable "$test_name" "$file"
			return 3
		fi
	fi

	if [ -z "$vquiet" ]; then
		echo "  TEST    $test_name ($file)"
	fi

	verbose "Using command       : $(echo "$@")"

	# grab the expected exit value
	expected_exit_value=$check_exit_value
	verbose "Expecting exit value: $expected_exit_value"

	# do we want a timeout?
	pre_cmd=""
	if [ $check_timeout -ne 0 ]; then
		pre_cmd="timeout $check_timeout"
	fi

	shift
	# launch the test command and
	# grab the actual output & exit value
	eval $pre_cmd $default_path/$base_cmd $default_args "$@" \
		1> $file.output.got 2> $file.error.got
	actual_exit_value=$?

	must_fail=$check_known_to_fail
	[ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
	known_ko_tests=$(($known_ko_tests + $must_fail))

	for stream in error output; do
		eval ignore=\$check_${stream}_ignore
		[ $ignore -eq 1 ] && continue

		# grab the expected output
		sed -n "/check-$stream-start/,/check-$stream-end/p" $file \
		| grep -v check-$stream > "$file".$stream.expected

		diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
		if [ "$?" -ne "0" ]; then
			error "actual $stream text does not match expected $stream text."
			error  "see $file.$stream.* for further investigation."
			[ $quiet -ne 1 ] && cat "$file".$stream.diff
			test_failed=1
		fi
	done

	if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
		error "Actual exit value does not match the expected one."
		error "expected $expected_exit_value, got $actual_exit_value."
		test_failed=1
	fi

	# verify the 'check-output-contains/excludes' tags
	if [ $check_output_contains -eq 1 ]; then
		has_each_patterns "$file" 'check-output-contains' absent $file.output.got
		if [ "$?" -ne "0" ]; then
			test_failed=1
		fi
	fi
	if [ $check_output_excludes -eq 1 ]; then
		has_none_patterns "$file" 'check-output-excludes' present $file.output.got
		if [ "$?" -ne "0" ]; then
			test_failed=1
		fi
	fi
	if [ $check_output_pattern -eq 1 ]; then
		# verify the 'check-output-pattern(...)' tags
		minmax_patterns "$file" 'check-output-pattern' $file.output.got
		if [ "$?" -ne "0" ]; then
			test_failed=1
		fi
	fi
	if [ $check_output_match -eq 1 ]; then
		# verify the 'check-output-match($insn): $patt' tags
		match_patterns "$file" 'check-output-match' $file.output.got
		if [ "$?" -ne "0" ]; then
			test_failed=1
		fi
	fi
	if [ $check_output_returns -eq 1 ]; then
		# verify the 'check-output-return: $value' tags
		return_patterns "$file" 'check-output-returns' $file.output.got
		if [ "$?" -ne "0" ]; then
			test_failed=1
		fi
	fi

	if [ "$must_fail" -eq "1" ]; then
		if [ "$test_failed" -eq "1" ]; then
			[ -z "$vquiet" ] && \
			echo "info: XFAIL: test '$file' is known to fail"
		else
			echo "error: XPASS: test '$file' is known to fail but succeed!"
		fi
	else
		if [ "$test_failed" -eq "1" ]; then
			echo "error: FAIL: test '$file' failed"
		else
			[ "$V" -ne "0" ] && \
			echo "info: PASS: test '$file' passed"
		fi
	fi

	if [ "$test_failed" -ne "$must_fail" ]; then
		[ $abort -eq 1 ] && exit 1
		test_failed=1
		failed=1
	fi

	if [ "$test_failed" -eq "1" ]; then
		ko_tests=$(($ko_tests + 1))
	else
		ok_tests=$(($ok_tests + 1))
		rm -f $file.{error,output}.{expected,got,diff}
	fi
	return $test_failed
}

do_test_suite()
{
	for i in $tests_list; do
		do_test "$i"
	done

	OK=OK
	[ $failed -eq 0 ] || OK=KO

	# prints some numbers
	tests_nr=$(($ok_tests + $ko_tests))
	echo "$OK: out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
	if [ "$known_ko_tests" -ne 0 ]; then
		echo "	$known_ko_tests of them are known to fail"
	fi
	if [ "$unhandled_tests" -ne "0" ]; then
		echo "	$unhandled_tests tests could not be handled by $prog_name"
	fi
	if [ "$disabled_tests" -ne "0" ]; then
		echo "	$disabled_tests tests were disabled"
	fi
}

##
do_format_help() {
echo "Usage: $prog_name [option(s)] [--]format file [name [cmd]]"
echo
echo "options:"
echo "    -a                         append the created test to the input file"
echo "    -f                         write a test known to fail"
echo "    -l                         write a test for linearized code"
echo "    -r                         write a test for linearized code returning 1"
echo "    -p                         write a test for pre-processing"
echo "    -s                         write a test for symbolic checking"
echo
echo "argument(s):"
echo "    file                       file containing the test case(s)"
echo "    name                       name for the test case (defaults to file)"
echo "    cmd                        command to be used (defaults to 'sparse \$file')"
}

##
# do_format([options,] file[, name[, cmd]]) - helps a test writer to format test-suite tags
do_format()
{
	def_cmd="$default_cmd"
	append=0
	linear=0
	fail=0
	ret=''

	while [ $# -gt 0 ] ; do
		case "$1" in
		-a)
			append=1 ;;
		-f)
			fail=1 ;;
		-l)
			def_cmd='test-linearize -Wno-decl $file'
			linear=1 ;;
		-r)
			def_cmd='test-linearize -Wno-decl $file'
			ret=1 ;;
		-p)
			def_cmd='sparse -E $file' ;;
		-s)
			def_cmd='scheck $file' ;;

		help|-*)
			do_format_help
			return 0
			;;
		*)	break ;;
		esac
		shift
		continue
	done

	if [ $# -lt 1 -o $# -gt 3 ]; then
		do_format_help
		return 0
	fi

	arg_file "$1" || return 1

	file="$1"
	fname="$2"
	[ -z "$fname" ] && fname="$(basename "$1" .c)"
	fcmd="$3"
	[ -z "$fcmd" ] && fcmd="$def_cmd"

	cmd=`eval echo $default_path/$fcmd`
	$cmd 1> $file.output.got 2> $file.error.got
	fexit_value=$?
	[ $append != 0 ] && exec >> $file
	cat <<_EOF

/*
 * check-name: $fname
_EOF
	if [ "$fcmd" != "$default_cmd" ]; then
		echo " * check-command: $fcmd"
	fi
	if [ "$fexit_value" -ne "0" ]; then
		echo " * check-exit-value: $fexit_value"
	fi
	if [ $fail != 0 ]; then
		echo " * check-known-to-fail"
	fi
	if [ "$ret" != '' ]; then
		echo ' *'
		echo ' * check-output-ignore'
		echo " * check-output-returns: $ret"
		rm -f "$file.output.got"
	fi
	if [ $linear != 0 ]; then
		echo ' *'
		echo ' * check-output-ignore'
		echo ' * check-output-contains: xyz\\\\.'
		echo ' * check-output-excludes: \\\\.'
	fi
	for stream in output error; do
		if [ -s "$file.$stream.got" ]; then
			echo " *"
			echo " * check-$stream-start"
			cat "$file.$stream.got"
			echo " * check-$stream-end"
		fi
	done
	echo " */"
	return 0
}

## allow flags from environment
set -- $SPARSE_TEST_FLAGS "$@"

## process the flags
while [ "$#" -gt "0" ]; do
	case "$1" in
	-a|--abort)
		abort=1
		;;
	-q|--quiet)
		vquiet=1
		;;
	--args=*)
		default_args="${1#--args=}";
		;;

	single|--single)
		arg_file "$2"
		do_test "$2"
		case "$?" in
			0) echo "$2 passed !";;
			1) echo "$2 failed !";;
			2) echo "$2 can't be handled by $prog_name";;
		esac
		exit $failed
		;;
	format|--format)
		shift
		do_format "$@"
		exit 0
		;;
	help)
		do_usage
		exit 1
		;;

	*.c|*.cdoc)
		tests_list="$tests_list $1"
		;;
	*)
		if [ ! -d "$1" ]; then
			do_usage
			exit 1
		fi
		tests_list="$tests_list $(find "$1" -name '*.c' | sort)"
		;;
	esac
	shift
done

if [ -z "$tests_list" ]; then
	tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
fi

do_test_suite
exit $failed