[2/4] tests: Add simple shell test drivers

Message ID	20180813030506.429060-3-amitay@ozlabs.org
State	Accepted
Headers	show Return-Path: <pdbg-bounces+incoming=patchwork.ozlabs.org@lists.ozlabs.org> From: Amitay Isaacs <amitay@ozlabs.org> To: pdbg@lists.ozlabs.org Date: Mon, 13 Aug 2018 13:05:04 +1000 Message-Id: <20180813030506.429060-3-amitay@ozlabs.org> In-Reply-To: <20180813030506.429060-1-amitay@ozlabs.org> References: <20180813030506.429060-1-amitay@ozlabs.org> Subject: [Pdbg] [PATCH 2/4] tests: Add simple shell test drivers Precedence: list Cc: Amitay Isaacs <amitay@ozlabs.org> MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pdbg-bounces+incoming=patchwork.ozlabs.org@lists.ozlabs.org Sender: "Pdbg" <pdbg-bounces+incoming=patchwork.ozlabs.org@lists.ozlabs.org>
Series	Enable testing and add some tests \| expand [0/4] Enable testing and add some tests [1/4] configure: Enable tests [2/4] tests: Add simple shell test drivers [3/4] gitignore: Ignore tests related files [4/4] tests: Add target selection tests

Context	Check	Description
snowpatch_ozlabs/apply_patch	success	master/apply_patch Successfully applied
snowpatch_ozlabs/build-multiarch	success	Test build-multiarch on branch master

diff --git a/Makefile.am b/Makefile.am index f6b8195..d7254ff 100644 --- a/Makefile.am +++ b/Makefile.am @@ -8,6 +8,9 @@ check_PROGRAMS = optcmd_test TESTS = optcmd_test +TEST_EXTENSIONS = .sh +SH_LOG_DRIVER = $(SHELL) $(srcdir)/tests/run_test.sh + ACLOCAL_AMFLAGS = -Im4 AM_CFLAGS = -I$(top_srcdir)/ccan/array_size -Wall -Werror -O2 @@ -25,7 +28,10 @@ EXTRA_DIST = \ p9w-fsi.dts.m4 \ p9z-fsi.dts.m4 \ template.S \ - generate_dt_header.sh + generate_dt_header.sh \ + tests/driver.sh \ + tests/run_test.sh \ + tests/test_driver.sh if TARGET_ARM DT_ARM = p8-fsi.dts p8-i2c.dts \ diff --git a/tests/driver.sh b/tests/driver.sh new file mode 100644 index 0000000..2b2ee49 --- /dev/null +++ b/tests/driver.sh @@ -0,0 +1,247 @@ +#!/bin/sh + +# +# Simple test driver for shell based testsuite +# +# Each testsuite is a shell script, which sources this driver file. +# Following functions can be used to define testsuite and tests. +# +# test_group <testsuite-name> +# +# Define a test suite. This function should be called before calling any +# other functions +# +# test_result <rc> <--|output> +# +# Define the exit code and the output for the test. If there is no output +# from the commands, then '--' can be specified to denote empty output. +# Multi-line output can be added using a here document. +# +# test_run <command> [<arguments>] +# +# Define the command to execute for the test. +# +# test_skip +# +# This can be called before test_run, to skip the test. This is useful to +# write tests which are dependent on the environment (e.g. architecture). +# +# +# Example: +# +# test_group "my group of tests" +# +# test_result 0 output1 +# test_run command1 arguments1 +# +# test_result 1 -- +# test_run command2 arguments2 +# +# test_result 0 output3 +# if [ $condition ] ; then +# test_skip +# fi +# test_run command3 arguments3 +# + +set -u + +TESTDIR=$(dirname "$0") +if [ "$TESTDIR" = "." ] ; then + TESTDIR=$(cd "$TESTDIR"; pwd) +fi +SRCDIR=$(dirname "$TESTDIR") +PATH="$SRCDIR":$PATH + +test_name=${TEST_NAME:-$0} +test_logfile=${TEST_LOG:-} +test_trsfile=${TEST_TRS:-} +test_color=${TEST_COLOR:-yes} + +red= grn= lgn= blu= mgn= std= +if [ $test_color = yes ] ; then + red='[0;31m' # Red. + grn='[0;32m' # Green. + lgn='[1;32m' # Light green. + blu='[1;34m' # Blue. + mgn='[0;35m' # Magenta. + std='[m' # No color. +fi + +test_started=0 +test_skipped=0 +test_defined=0 + +count_total=0 +count_skipped=0 +count_failed=0 + +trap 'test_end' 0 + +test_group () +{ + test_name=${1:-$test_name} + test_started=1 + + echo "-- $test_name" +} + +test_end () +{ + trap 0 + if [ $count_total -eq 0 ] ; then + status=99 + elif [ $count_failed -gt 0 ] ; then + status=1 + elif [ $count_skipped -eq $count_total ] ; then + status=77 + else + status=0 + fi + + exit $status +} + +test_error () +{ + echo "$@" + exit 99 +} + +test_log () +{ + if [ -z "$test_logfile" ] ; then + echo "$@" + else + echo "$@" >> "$test_logfile" + fi +} + +test_trs () +{ + if [ -n "$test_trsfile" ] ; then + echo "$@" >> "$test_trsfile" + fi +} + +test_output () +{ + rc=${1:-1} + required_rc=${2:-0} + output_mismatch=${3:-0} + + if [ $required_rc -eq 0 ] ; then + expect_failure=no + else + expect_failure=yes + fi + + case $rc:$expect_failure:$output_mismatch in + 0:*:1) col=$red res=FAIL ;; + 0:yes:*) col=$red res=XPASS ;; + 0:*:*) col=$grn res=PASS ;; + 77:*:*) col=$blu res=SKIP ;; + *:*:1) col=$red res=FAIL ;; + *:yes:*) col=$lgn res=XFAIL ;; + *:*:*) col=$red res=FAIL ;; + esac + + if [ -n "$test_logfile" ] ; then + test_log "${res} ${test_cmd} (exit status: ${rc})" + fi + test_trs ":test-result: ${res}" + + if [ -n "$test_trsfile" ] ; then + indent=" " + else + indent="" + fi + + echo "${indent}${col}${res}${std}: ${test_cmd}" + + count_total=$(( count_total + 1 )) + if [ $res = "SKIP" ] ; then + count_skipped=$(( count_skipped + 1 )) + elif [ $res = "XPASS" -o $res = "FAIL" ] ; then + count_failed=$(( count_failed + 1 )) + fi +} + +#--------------------------------------------------------------------- +# Public functions +#--------------------------------------------------------------------- + +test_result () +{ + if [ $test_started -eq 0 ] ; then + test_error "ERROR: missing call to test_group" + fi + + required_rc="${1:-0}" + if [ $# -eq 2 ] ; then + if [ "$2" = "--" ] ; then + required_output="" + else + required_output="$2" + fi + else + if ! tty -s ; then + required_output=$(cat) + else + required_output="" + fi + fi + + test_defined=1 +} + +test_skip () +{ + if [ $test_started -eq 0 ] ; then + test_error "ERROR: missing call to test_group" + fi + + test_skipped=1 +} + +test_run () +{ + output_mismatch=0 + + if [ $test_started -eq 0 ] ; then + test_error "ERROR: missing call to test_group" + fi + + if [ $test_defined -eq 0 ] ; then + test_error "ERROR: missing call to test_result" + fi + + test_cmd="$@" + + if [ $test_skipped -eq 1 ] ; then + test_output 77 + test_skipped=0 + test_defined=0 + return + fi + + output=$("$@" 2>&1) + rc=$? + + if [ $rc -ne $required_rc ] ; then + test_log "expected rc: $required_rc" + test_log "output rc: $rc" + fi + + if [ "$output" != "$required_output" ] ; then + test_log "expected:" + test_log "$required_output" + test_log "output:" + test_log "$output" + output_mismatch=1 + fi + + test_output $rc $required_rc $output_mismatch + test_skipped=0 + test_defined=0 +} diff --git a/tests/run_test.sh b/tests/run_test.sh new file mode 100755 index 0000000..d5661b3 --- /dev/null +++ b/tests/run_test.sh @@ -0,0 +1,91 @@ +#!/bin/sh + +# Derived from automake basic testsuite driver + +set -u + +usage_error () +{ + echo "$0: $*" >&2 + print_usage >&2 + exit 2 +} + +print_usage () +{ + cat <<EOF +Usage: $0 --test-name NAME --log-file PATH --trs-file PATH + [--expect-failure={yes|no}] [--color-tests={yes|no}] + [--enable-hard-errors={yes|no}] [--] + TEST-SCRIPT [TEST-SCRIPT-ARGUMENTS] +The '--test-name', '--log-file' and '--trs-file' options are mandatory. +EOF +} + +TEST_NAME= +TEST_LOG= +TEST_TRS= +TEST_COLOR=no +expect_failure=no +enable_hard_errors=yes + +while [ $# -gt 0 ] ; do + case "$1" in + --help) print_usage; exit $?;; + --test-name) TEST_NAME=$2; shift;; + --log-file) TEST_LOG="$2"; shift;; + --trs-file) TEST_TRS="$2"; shift;; + --color-tests) TEST_COLOR=$2; shift;; + --expect-failure) expect_failure=$2; shift;; + --enable-hard-errors) enable_hard_errors=$2; shift;; + --) shift; break;; + -*) usage_error "invalid option: '$1'";; + *) break;; + esac + shift +done + +missing_opts="" +[ -z "$TEST_NAME" ] && missing_opts="$missing_opts --test-name" +[ -z "$TEST_LOG" ] && missing_opts="$missing_opts --log-file" +[ -z "$TEST_TRS" ] && missing_opts="$missing_opts --trs-file" +if [ -n "$missing_opts" ] ; then + usage_error "these mandatory options are missing:$missing_opts" +fi + +if [ $# -eq 0 ] ; then + usage_error "missing argument" +fi + +do_exit='rm -f $log_file $trs_file; (exit $st) ; exit $st' +trap "st=129; $do_exit" 1 +trap "st=130; $do_exit" 2 +trap "st=141; $do_exit" 13 +trap "st=143; $do_exit" 15 + +export TEST_NAME TEST_LOG TEST_TRS TEST_COLOR + +# Test script is run here. +"$@" +estatus=$? + +if [ $enable_hard_errors = no -a $estatus -eq 99 ] ; then + tweaked_estatus=1 +else + tweaked_estatus=$estatus +fi + +case $tweaked_estatus:$expect_failure in +0:yes) res=XPASS recheck=yes gcopy=yes ;; +0:*) res=PASS recheck=no gcopy=no ;; +77:*) res=SKIP recheck=no gcopy=yes ;; +99:*) res=ERROR recheck=yes gcopy=yes ;; +*:yes) res=XFAIL recheck=no gcopy=yes ;; +*:*) res=FAIL recheck=yes gcopy=yes ;; +esac + +echo "$res $TEST_NAME (exit_status: $estatus)" >> "$TEST_LOG" + +echo ":global-test-result: $res" >> "$TEST_TRS" +echo ":recheck: $recheck" >> "$TEST_TRS" +echo ":copy-in-global-log: $gcopy" >> "$TEST_TRS" diff --git a/tests/test_driver.sh b/tests/test_driver.sh new file mode 100755 index 0000000..6b00bf6 --- /dev/null +++ b/tests/test_driver.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +TEST_COLOR=no +TEST_LOG=/dev/null +TEST_TRS="" + +. $(dirname "$0")/driver.sh + +test_group "test driver tests" + +echo "test should PASS" +test_result 0 foo +test_run echo foo + +echo "test should FAIL" +test_result 0 foo +test_run echo bar + +echo "test should XPASS" +test_result 1 foo +test_run echo foo + +echo "test should FAIL" +test_result 1 foo +test_run echo bar + +echo "test should XFAIL" +test_result 1 -- +test_run false + +echo "test should FAIL" +test_result 1 -- +test_run echo foo && false + +echo "test should SKIP" +test_result 0 -- +test_skip +test_run echo foo

[2/4] tests: Add simple shell test drivers

Checks

Commit Message

Patch