PCRE2Project/pcre2
1
0
Fork 0
mirror of https://github.com/PCRE2Project/pcre2.git synced 2025-10-16 05:17:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Improve RunTest to continue after a test failure (#696)

Browse Source 
This makes it easier to se all the failures at once, rather than
having to fix one at a time. The output is now grouped into
directories.
This commit is contained in:
Nicholas Wilson
2025-02-15 11:50:25 +00:00
committed by GitHub
parent 0d0ac3aa0f
commit db3b532aa0
6 changed files with 110 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.gitignore vendored
View File

@@ -77,9 +77,19 @@ testtry
testtry2
testtrygrep
testSinput
testSoutput
testbtables
testsaved1
testsaved2
testoutput8
testoutput8-jit
testoutput8-dfa
testoutput16
testoutput16-jit
testoutput16-dfa
testoutput32
testoutput32-jit
testoutput32-dfa
m4/libtool.m4
m4/ltoptions.m4

8
Makefile.am
View File

@@ -841,6 +841,7 @@ EXTRA_DIST += \
CLEANFILES += \
testSinput \
testSoutput \
test3input \
test3output \
test3outputA \
@@ -854,6 +855,13 @@ CLEANFILES += \
testtrygrep \
testNinputgrep
clean-local: testoutput-clean
.PHONY: testoutput-clean
testoutput-clean:
-rm -rf testoutput8 testoutput8-jit testoutput8-dfa
-rm -rf testoutput16 testoutput16-jit testoutput16-dfa
-rm -rf testoutput32 testoutput32-jit testoutput32-dfa
## ------------ End of testing -------------

6
README
View File

@@ -637,9 +637,9 @@ NON-AUTOTOOLS-BUILD.
The RunTest script runs the pcre2test test program (which is documented in its
own man page) on each of the relevant testinput files in the testdata
directory, and compares the output with the contents of the corresponding
testoutput files. RunTest uses a file called testtry to hold the main output
from pcre2test. Other files whose names begin with "test" are used as working
files in some tests.
testoutput files. RunTest places its output in directories
testoutput{8,16,32}{,-jit,-dfa}. Other files whose names begin with "test" are
used as working files in some tests.
Some tests are relevant only when certain build-time options were selected. For
example, the tests for UTF-8/16/32 features are run only when Unicode support

135
RunTest
View File

@@ -147,6 +147,8 @@ else
exit 1
fi
yield=0
# ------ Function to check results of a test -------
@@ -161,8 +163,9 @@ fi
checkresult()
{
if [ $1 -ne 0 ] ; then
echo "** pcre2test failed - check testtry"
exit 1
echo "** pcre2test failed - check testoutput$bits$3/testoutput$2"
yield=1
return
fi
case "$3" in
-jit) with=" with JIT";;
@@ -174,14 +177,15 @@ checkresult()
# We currently only use the #if ... #endif support in pcre2test for EBCDIC
# testing. Run in "preprocess-only" mode (-E) on the testoutput file to trim
# the output lines matching the input lines which are discarded.
$sim $pcre2test -q -E "$cf_out" >testtry2
cf_out=testtry2
$sim $pcre2test -q -E "$cf_out" >testoutput$bits$3/testoutput$2-expected
cf_out=testoutput$bits$3/testoutput$2-expected
fi
$cf "$cf_out" testtry
$cf "$cf_out" testoutput$bits$3/testoutput$2
if [ $? != 0 ] ; then
echo ""
echo "** Test $2 failed$with"
exit 1
yield=1
return
fi
echo " OK$with"
}
@@ -191,11 +195,13 @@ checkresult()
checkspecial()
{
$sim $valgrind $vjs $pcre2test $1 >>testtry
$sim $valgrind $vjs $pcre2test $1 >>testSoutput
if [ $? -ne 0 ] ; then
echo "** pcre2test $1 failed - check testtry"
exit 1
echo "** pcre2test $1 failed - check testSoutput"
yield=1
return 1
fi
return 0
}
@@ -521,6 +527,11 @@ for bmode in "$test8" "$test16" "$test32"; do
-8) bits=8; echo "---- Testing 8-bit library ----"; echo "";;
esac
# Set up directories for test output.
[ -d testoutput$bits ] || mkdir testoutput$bits
[ -d testoutput$bits$jitopt ] || mkdir testoutput$bits$jitopt
[ -d testoutput$bits-dfa ] || mkdir testoutput$bits-dfa
# Test 0 is a special test. Its output is not checked, because it will
# be different on different hardware and with different configurations.
# Running this test just exercises the code.
@@ -529,13 +540,16 @@ for bmode in "$test8" "$test16" "$test32"; do
echo $title0
echo '/abc/jit,memory,framesize' >testSinput
echo ' abc' >>testSinput
echo '' >testtry
checkspecial '-C'
checkspecial '--help'
echo '' >testSoutput
saverc=0
checkspecial '-C' || saverc=$?
checkspecial '--help' || saverc=$?
if [ $support_setstack -eq 0 ] ; then
checkspecial '-S 1 -t 10 testSinput'
checkspecial '-S 1 -t 10 testSinput' || saverc=$?
fi
if [ $saverc -eq 0 ] ; then
echo " OK"
fi
echo " OK"
fi
# Primary non-UTF test, compatible with JIT and all versions of Perl >= 5.8
@@ -543,7 +557,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ $do1 = yes ] ; then
echo $title1
for opt in "" $jitopt; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput1 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput1 testoutput$bits$opt/testoutput1
checkresult $? 1 "$opt"
done
fi
@@ -555,10 +569,10 @@ for bmode in "$test8" "$test16" "$test32"; do
echo $title2 "(excluding UTF-$bits)"
cp $testdata/testbtables .
for opt in "" $jitopt; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput2 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput2 testoutput$bits$opt/testoutput2
saverc=$?
if [ $saverc = 0 ] ; then
$sim $valgrind ${opt:+$vjs} $pcre2test -q $bmode $opt -error -80,-62,-2,-1,0,100,101,191,300 >>testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $bmode $opt -error -80,-62,-2,-1,0,100,101,191,300 >>testoutput$bits$opt/testoutput2
checkresult $? 2 "$opt"
else
checkresult $saverc 2 "$opt"
@@ -614,16 +628,16 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ "$locale" != "" ] ; then
echo $title3 "(using '$locale' locale)"
for opt in "" $jitopt; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $infile testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $infile testoutput$bits/testoutput3
if [ $? = 0 ] ; then
case "$opt" in
-jit) with=" with JIT";;
*) with="";;
esac
if $cf $outfile testtry >teststdout || \
$cf $outfile2 testtry >>teststdout || \
$cf $outfile3 testtry >>teststdout || \
$cf $outfile4 testtry >>teststdout
if $cf $outfile testoutput$bits/testoutput3 >teststdout || \
$cf $outfile2 testoutput$bits/testoutput3 >>teststdout || \
$cf $outfile3 testoutput$bits/testoutput3 >>teststdout || \
$cf $outfile4 testoutput$bits/testoutput3 >>teststdout
then
echo " OK$with"
else
@@ -633,9 +647,11 @@ for bmode in "$test8" "$test16" "$test32"; do
echo " $outfile, $outfile2, $outfile3 or $outfile4."
echo " This may mean that there is a problem with the locale settings rather"
echo " than a bug in PCRE2."
exit 1
yield=1
fi
else exit 1
else
echo "** pcre2test failed - check testoutput$bits/testoutput3"
yield=1
fi
done
else
@@ -654,7 +670,7 @@ for bmode in "$test8" "$test16" "$test32"; do
echo " Skipped because UTF-$bits support is not available"
else
for opt in "" $jitopt; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput4 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput4 testoutput$bits$opt/testoutput4
checkresult $? 4 "$opt"
done
fi
@@ -666,7 +682,7 @@ for bmode in "$test8" "$test16" "$test32"; do
echo " Skipped because UTF-$bits support is not available"
else
for opt in "" $jitopt; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput5 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput5 testoutput$bits$opt/testoutput5
checkresult $? 5 "$opt"
done
fi
@@ -676,7 +692,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ $do6 = yes ] ; then
echo $title6
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput6 testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput6 testoutput$bits/testoutput6
checkresult $? 6 ""
fi
@@ -685,7 +701,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ $utf -eq 0 ] ; then
echo " Skipped because UTF-$bits support is not available"
else
$sim $valgrind $pcre2test -q $setstack $bmode $opt $testdata/testinput7 testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput7 testoutput$bits/testoutput7
checkresult $? 7 ""
fi
fi
@@ -709,7 +725,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ $utf -eq 0 ] ; then
echo " Skipped because UTF-$bits support is not available"
else
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput8 testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput8 testoutput$bits/testoutput8-$bits-$bits_link_size
checkresult $? 8-$bits-$bits_link_size ""
fi
fi
@@ -722,7 +738,7 @@ for bmode in "$test8" "$test16" "$test32"; do
echo " Skipped when running 16/32-bit tests"
else
for opt in "" $jitopt; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput9 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput9 testoutput$bits$opt/testoutput9
checkresult $? 9 "$opt"
done
fi
@@ -738,7 +754,7 @@ for bmode in "$test8" "$test16" "$test32"; do
echo " Skipped because UTF-$bits support is not available"
else
for opt in "" $jitopt; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput10 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput10 testoutput$bits$opt/testoutput10
checkresult $? 10 "$opt"
done
fi
@@ -752,7 +768,7 @@ for bmode in "$test8" "$test16" "$test32"; do
echo " Skipped when running 8-bit tests"
else
for opt in "" $jitopt; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput11 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput11 testoutput$bits$opt/testoutput11-$bits
checkresult $? 11-$bits "$opt"
done
fi
@@ -769,7 +785,7 @@ for bmode in "$test8" "$test16" "$test32"; do
echo " Skipped because UTF-$bits support is not available"
else
for opt in "" $jitopt; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput12 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput12 testoutput$bits$opt/testoutput12-$bits
checkresult $? 12-$bits "$opt"
done
fi
@@ -782,7 +798,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ "$bits" = "8" ] ; then
echo " Skipped when running 8-bit tests"
else
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput13 testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput13 testoutput$bits/testoutput13
checkresult $? 13 ""
fi
fi
@@ -794,7 +810,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ $utf -eq 0 ] ; then
echo " Skipped because UTF-$bits support is not available"
else
$sim $valgrind $pcre2test -q $setstack $bmode $opt $testdata/testinput14 testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput14 testoutput$bits/testoutput14-$bits
checkresult $? 14-$bits ""
fi
fi
@@ -803,7 +819,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ $do15 = yes ] ; then
echo $title15
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput15 testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput15 testoutput$bits/testoutput15
checkresult $? 15 ""
fi
@@ -814,7 +830,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ $jit -ne 0 ] ; then
echo " Skipped because JIT is available"
else
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput16 testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput16 testoutput$bits/testoutput16
checkresult $? 16 ""
fi
fi
@@ -826,7 +842,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ $jit -eq 0 -o "$nojit" = "yes" ] ; then
echo " Skipped because JIT is not available or nojit was specified"
else
$sim $valgrind $vjs $pcre2test -q $setstack $bmode $testdata/testinput17 testtry
$sim $valgrind $vjs $pcre2test -q $setstack $bmode $testdata/testinput17 testoutput$bits/testoutput17
checkresult $? 17 ""
fi
fi
@@ -838,7 +854,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ "$bits" = "16" -o "$bits" = "32" ] ; then
echo " Skipped when running 16/32-bit tests"
else
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput18 testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput18 testoutput$bits/testoutput18
checkresult $? 18 ""
fi
fi
@@ -852,7 +868,7 @@ for bmode in "$test8" "$test16" "$test32"; do
elif [ $utf -eq 0 ] ; then
echo " Skipped because UTF-$bits support is not available"
else
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput19 testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput19 testoutput$bits/testoutput19
checkresult $? 19 ""
fi
fi
@@ -861,7 +877,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ $do20 = yes ] ; then
echo $title20
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput20 testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput20 testoutput$bits/testoutput20
checkresult $? 20 ""
fi
@@ -873,7 +889,7 @@ for bmode in "$test8" "$test16" "$test32"; do
echo " Skipped because \C is disabled"
else
for opt in "" $jitopt -dfa; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput21 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput21 testoutput$bits$opt/testoutput21
checkresult $? 21 "$opt"
done
fi
@@ -889,7 +905,7 @@ for bmode in "$test8" "$test16" "$test32"; do
echo " Skipped because UTF-$bits support is not available"
else
for opt in "" $jitopt; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput22 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput22 testoutput$bits$opt/testoutput22-$bits
checkresult $? 22-$bits "$opt"
done
fi
@@ -902,7 +918,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ $supportBSC -ne 0 ] ; then
echo " Skipped because \C is not disabled"
else
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput23 testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput23 testoutput$bits/testoutput23
checkresult $? 23 ""
fi
fi
@@ -911,7 +927,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ "$do24" = yes ] ; then
echo $title24
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput24 testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput24 testoutput$bits/testoutput24
checkresult $? 24 ""
fi
@@ -922,7 +938,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ $utf -eq 0 ] ; then
echo " Skipped because UTF-$bits support is not available"
else
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput25 testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinput25 testoutput$bits/testoutput25
checkresult $? 25 ""
fi
fi
@@ -935,7 +951,7 @@ for bmode in "$test8" "$test16" "$test32"; do
echo " Skipped because UTF-$bits support is not available"
else
for opt in "" $jitopt; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput26 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput26 testoutput$bits$opt/testoutput26
checkresult $? 26 "$opt"
done
fi
@@ -949,7 +965,7 @@ for bmode in "$test8" "$test16" "$test32"; do
echo " Skipped because UTF-$bits support is not available"
else
for opt in "" $jitopt; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput27 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput27 testoutput$bits$opt/testoutput27
checkresult $? 27 "$opt"
done
fi
@@ -963,7 +979,7 @@ for bmode in "$test8" "$test16" "$test32"; do
echo " Skipped when not targetting EBCDIC"
else
for opt in "" $jitopt "-dfa"; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput28 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput28 testoutput$bits$opt/testoutput28
checkresult $? 28 "$opt"
done
fi
@@ -979,7 +995,7 @@ for bmode in "$test8" "$test16" "$test32"; do
echo " Skipped because EBCDIC newline is not 0x25"
else
for opt in "" $jitopt "-dfa"; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput29 testtry
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput29 testoutput$bits$opt/testoutput29
checkresult $? 29 "$opt"
done
fi
@@ -990,7 +1006,7 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ $doheap = yes ] ; then
echo $titleheap
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinputheap testtry
$sim $valgrind $pcre2test -q $setstack $bmode $testdata/testinputheap testoutput$bits/testoutputheap-$bits
checkresult $? heap-$bits ""
fi
@@ -998,7 +1014,20 @@ for bmode in "$test8" "$test16" "$test32"; do
done
# Clean up local working files
rm -f testbtables testSinput test3input testsaved1 testsaved2 test3output test3outputA test3outputB teststdout teststderr testtry testtry2
if [ $yield -eq 0 ] ; then
echo ""
echo "All tests passed."
# Clean up local working files
rm -f testbtables testSinput testSoutput test3input test3output test3outputA test3outputB testsaved1 testsaved2 teststdout teststderr
rm -rf testoutput8 testoutput8-jit testoutput8-dfa \
testoutput16 testoutput16-jit testoutput16-dfa \
testoutput32 testoutput32-jit testoutput32-dfa
else
echo ""
echo "** Tests failed. See output above for details."
fi
exit $yield
# End

6
doc/html/README.txt
View File

@@ -637,9 +637,9 @@ NON-AUTOTOOLS-BUILD.
The RunTest script runs the pcre2test test program (which is documented in its
own man page) on each of the relevant testinput files in the testdata
directory, and compares the output with the contents of the corresponding
testoutput files. RunTest uses a file called testtry to hold the main output
from pcre2test. Other files whose names begin with "test" are used as working
files in some tests.
testoutput files. RunTest places its output in directories
testoutput{8,16,32}{,-jit,-dfa}. Other files whose names begin with "test" are
used as working files in some tests.
Some tests are relevant only when certain build-time options were selected. For
example, the tests for UTF-8/16/32 features are run only when Unicode support

8
maint/RunPerlTest
View File

@@ -28,10 +28,10 @@ echo "Perl test: main functionality (PCRE2 test 1)"
if ./perltest.sh $ARGS testdata/testinput1 testtry; then
tail -n +2 testtry > testtry2
diff -u testdata/testoutput1 testtry2 || RC=1
/bin/rm -rf testtry2
else
RC=33
fi
/bin/rm -f testtry testtry2
echo ""
fi
@@ -41,10 +41,10 @@ echo "Perl test: UTF-8 and Unicode property features (PCRE2 test 4)"
if ./perltest.sh $ARGS -utf8 testdata/testinput4 testtry; then
tail -n +2 testtry > testtry2
diff -u testdata/testoutput4 testtry2 || RC=4
/bin/rm -rf testtry2
else
RC=36
fi
/bin/rm -f testtry testtry2
echo ""
fi
@@ -60,10 +60,10 @@ else
if ./perltest.sh $ARGS testdata/testinput26 testtry; then
tail -n +2 testtry > testtry2
diff -u testdata/testoutput26 testtry2 || RC=26
/bin/rm -rf testtry2
else
RC=58
fi
/bin/rm -f testtry testtry2
echo ""
fi
fi
@@ -78,10 +78,10 @@ else
if ./perltest.sh $ARGS testdata/testinput27 testtry; then
tail -n +2 testtry > testtry2
diff -u testdata/testoutput27 testtry2 || RC=27
/bin/rm -rf testtry2
else
RC=59
fi
/bin/rm -f testtry testtry2
echo ""
fi
fi