From 79e4f4e933983ae771ee1a2c6744e35e6ab01400 Mon Sep 17 00:00:00 2001 From: Jaeden Amero Date: Fri, 5 Oct 2018 12:21:15 +0100 Subject: [PATCH 1/4] test: Print verbosely on failures in verbose mode Update the test runner to print detail about why the test failed when it fails, if the runner is running in verbose mode. --- tests/scripts/run-test-suites.pl | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl index 6fe6abfa5..73f33bb61 100755 --- a/tests/scripts/run-test-suites.pl +++ b/tests/scripts/run-test-suites.pl @@ -50,10 +50,20 @@ my ($failed_suites, $total_tests_run, $failed, $suite_cases_passed, $suite_cases_failed, $suite_cases_skipped, $total_cases_passed, $total_cases_failed, $total_cases_skipped ); +sub pad_print_center { + my( $width, $padchar, $string ) = @_; + my $padlen = ( $width - length( $string ) - 2 ) / 2; + print $padchar x( $padlen ), " $string ", $padchar x( $padlen ), "\n"; +} + for my $suite (@suites) { print "$suite ", "." x ( 72 - length($suite) - 2 - 4 ), " "; - my $result = `$prefix$suite`; + my $command = "$prefix$suite"; + if( $verbose ) { + $command .= ' -v'; + } + my $result = `$command`; $suite_cases_passed = () = $result =~ /.. PASS/g; $suite_cases_failed = () = $result =~ /.. FAILED/g; @@ -64,6 +74,11 @@ for my $suite (@suites) } else { $failed_suites++; print "FAIL\n"; + if( $verbose ) { + pad_print_center( 72, '-', "Begin $suite" ); + print $result; + pad_print_center( 72, '-', "End $suite" ); + } } my ($passed, $tests, $skipped) = $result =~ /([0-9]*) \/ ([0-9]*) tests.*?([0-9]*) skipped/; From f4b521dd10a4d336e1409af2458be2f216e2d9c5 Mon Sep 17 00:00:00 2001 From: Jaeden Amero Date: Fri, 5 Oct 2018 12:45:15 +0100 Subject: [PATCH 2/4] test: Use GetOpt::Long for argument parsing Simplify argument parsing by using a core perl library for parsing arguments. --- tests/scripts/run-test-suites.pl | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl index 73f33bb61..237326aff 100755 --- a/tests/scripts/run-test-suites.pl +++ b/tests/scripts/run-test-suites.pl @@ -24,14 +24,10 @@ use strict; use utf8; use open qw(:std utf8); -use constant FALSE => 0; -use constant TRUE => 1; +use Getopt::Long; my $verbose; -my $switch = shift; -if ( defined($switch) && ( $switch eq "-v" || $switch eq "--verbose" ) ) { - $verbose = TRUE; -} +GetOptions( "verbose|v" => \$verbose ); # All test suites = executable files, excluding source files, debug # and profiling information, etc. We can't just grep {! /\./} because From 8396a71449d9632aa962e0ab66de5b5de3c368df Mon Sep 17 00:00:00 2001 From: Jaeden Amero Date: Fri, 5 Oct 2018 13:23:35 +0100 Subject: [PATCH 3/4] test: Enable multiple levels of verbosity Enable passing a number to "-v" in order to set the level of verbosity. Print detailed test failure information at verbosity level 1 or higher. Display summary messages at the verbosity level 2 or higher. Print detailed test information at verbosity level 3 or higher, whether the test failed or not. This enables a more readable output style that includes detailed failure information when a failure occurs. --- tests/scripts/run-test-suites.pl | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl index 237326aff..d0d404621 100755 --- a/tests/scripts/run-test-suites.pl +++ b/tests/scripts/run-test-suites.pl @@ -26,8 +26,8 @@ use open qw(:std utf8); use Getopt::Long; -my $verbose; -GetOptions( "verbose|v" => \$verbose ); +my $verbose = 0; +GetOptions( "verbose|v:1" => \$verbose ); # All test suites = executable files, excluding source files, debug # and profiling information, etc. We can't just grep {! /\./} because @@ -67,6 +67,11 @@ for my $suite (@suites) if( $result =~ /PASSED/ ) { print "PASS\n"; + if( $verbose > 2 ) { + pad_print_center( 72, '-', "Begin $suite" ); + print $result; + pad_print_center( 72, '-', "End $suite" ); + } } else { $failed_suites++; print "FAIL\n"; @@ -80,7 +85,7 @@ for my $suite (@suites) my ($passed, $tests, $skipped) = $result =~ /([0-9]*) \/ ([0-9]*) tests.*?([0-9]*) skipped/; $total_tests_run += $tests - $skipped; - if ( $verbose ) { + if( $verbose > 1 ) { print "(test cases passed:", $suite_cases_passed, " failed:", $suite_cases_failed, " skipped:", $suite_cases_skipped, @@ -98,7 +103,7 @@ print "-" x 72, "\n"; print $failed_suites ? "FAILED" : "PASSED"; printf " (%d suites, %d tests run)\n", scalar @suites, $total_tests_run; -if ( $verbose ) { +if( $verbose > 1 ) { print " test cases passed :", $total_cases_passed, "\n"; print " failed :", $total_cases_failed, "\n"; print " skipped :", $total_cases_skipped, "\n"; From 60ca6e58b6e691f948772284d2633636c556cf81 Mon Sep 17 00:00:00 2001 From: Jaeden Amero Date: Fri, 7 Dec 2018 13:06:24 +0000 Subject: [PATCH 4/4] test: Make basic-build-test.sh see summary statuses We've changed the behavior of "-v" to no longer output test summary statuses. Update basic-build-test.sh to use the test runner's verbosity option "-v 2", so that the basic-build-test.sh script can get the summary statuses it needs. --- tests/scripts/basic-build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index b4058718a..fbe757d9e 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -76,7 +76,7 @@ TEST_OUTPUT=out_${PPID} cd tests # Step 2a - Unit Tests -perl scripts/run-test-suites.pl -v |tee unit-test-$TEST_OUTPUT +perl scripts/run-test-suites.pl -v 2 |tee unit-test-$TEST_OUTPUT echo # Step 2b - System Tests