Commit 14dc679c authored by Daniel Stenberg's avatar Daniel Stenberg
Browse files

test: offer "automake" output and check for perl better 

runtests.pl -am now uses the "PASS/FAIL: [desc]" output for each
executed test. You can run 'make test-am' in the root build directory to
invoke that. The reason for this output style is to better allow generic
test suite parsers to also grok our test output.

The test Makefile now also tests that perl was indeed found and that the
PERL variable points to an executable before it tries to run the main
test perl script runtests.pl,
parent ef6f0403

Makefile.am

+3 −0
Original line number Diff line number Diff line
@@ -81,6 +81,9 @@ test-full: 
test-torture:

	@(cd tests; $(MAKE) all torture-test)



test-am:

	@(cd tests; $(MAKE) all am-test)



endif



examples:

tests/Makefile.am

+12 −7
Original line number Diff line number Diff line
@@ -55,26 +55,31 @@ curl: 


if CROSSCOMPILING

TEST = @echo "NOTICE: we can't run the tests when cross-compiling!"

TEST_Q =

TEST_F =

TEST_T =

else # if not cross-compiling:

TEST = srcdir=$(srcdir) $(PERL) $(PERLFLAGS) $(srcdir)/runtests.pl

TEST_Q = -a -s

TEST_AM = -a -am

TEST_F = -a -p -r

TEST_T = -a -t

endif



test: all

# make sure that PERL is pointing to an executable

perlcheck:

	@if ! test -x "$(PERL)"; then echo "No perl!"; exit 2; fi



test: perhcheck all

	$(TEST)



quiet-test: all

quiet-test: perlcheck all

	$(TEST) $(TEST_Q)



full-test: all

am-test: perlcheck all

	$(TEST) $(TEST_AM)



full-test: perlcheck all

	$(TEST) $(TEST_F)



torture-test: all

torture-test: perlcheck all

	$(TEST) $(TEST_T)



.1.html:

tests/runtests.pl

+42 −31
Original line number Diff line number Diff line
@@ -262,6 +262,7 @@ my %oldenv; 
#



my $short;

my $automakestyle;

my $verbose;

my $debugprotocol;

my $anyway;
@@ -2148,18 +2149,25 @@ sub filteroff { 
#

sub compare {

    # filter off patterns _before_ this comparison!

    my ($subject, $firstref, $secondref)=@_;

    my ($testnum, $testname, $subject, $firstref, $secondref)=@_;



    my $result = compareparts($firstref, $secondref);



    if($result) {

        # timestamp test result verification end

        $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);



        if(!$short) {

            logmsg "\n $subject FAILED:\n";

            logmsg "\n $testnum: $subject FAILED:\n";

            logmsg showdiff($LOGDIR, $firstref, $secondref);

        }

        else {

        elsif(!$automakestyle) {

            logmsg "FAILED\n";

        }

        else {

            # automakestyle

            logmsg "FAIL: $testnum - $testname - $subject\n";

        }

    }

    return $result;

}
@@ -2874,7 +2882,7 @@ sub singletest { 
        timestampskippedevents($testnum);

        return -1;

    }

    logmsg sprintf("test %03d...", $testnum);

    logmsg sprintf("test %03d...", $testnum) if(!$automakestyle);



    # extract the reply data

    my @reply = getpart("reply", "data");
@@ -2916,12 +2924,9 @@ sub singletest { 


    # name of the test

    my @testname= getpart("client", "name");



    if(!$short) {

        my $name = $testname[0];

        $name =~ s/\n//g;

        logmsg "[$name]\n";

    }

    my $testname = $testname[0];

    $testname =~ s/\n//g;

    logmsg "[$testname]\n" if(!$short);



    if($listonly) {

        timestampskippedevents($testnum);
@@ -3333,10 +3338,8 @@ sub singletest { 
            chomp($validstdout[$#validstdout]);

        }



        $res = compare("stdout", \@actual, \@validstdout);

        $res = compare($testnum, $testname, "stdout", \@actual, \@validstdout);

        if($res) {

            # timestamp test result verification end

            $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);

            return 1;

        }

        $ok .= "s";
@@ -3357,10 +3360,8 @@ sub singletest { 
            map s/\r\n/\n/g, @out;

        }



        $res = compare("data", \@out, \@reply);

        $res = compare($testnum, $testname, "data", \@out, \@reply);

        if ($res) {

            # timestamp test result verification end

            $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);

            return 1;

        }

        $ok .= "d";
@@ -3372,10 +3373,8 @@ sub singletest { 
    if(@upload) {

        # verify uploaded data

        my @out = loadarray("$LOGDIR/upload.$testnum");

        $res = compare("upload", \@out, \@upload);

        $res = compare($testnum, $testname, "upload", \@out, \@upload);

        if ($res) {

            # timestamp test result verification end

            $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);

            return 1;

        }

        $ok .= "u";
@@ -3419,10 +3418,8 @@ sub singletest { 
            }

        }



        $res = compare("protocol", \@out, \@protstrip);

        $res = compare($testnum, $testname, "protocol", \@out, \@protstrip);

        if($res) {

            # timestamp test result verification end

            $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);

            return 1;

        }
@@ -3469,10 +3466,8 @@ sub singletest { 
            }

        }



        $res = compare("proxy", \@out, \@protstrip);

        $res = compare($testnum, $testname, "proxy", \@out, \@protstrip);

        if($res) {

            # timestamp test result verification end

            $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);

            return 1;

        }
@@ -3521,10 +3516,9 @@ sub singletest { 


            @outfile = fixarray(@outfile);



            $res = compare("output ($filename)", \@generated, \@outfile);

            $res = compare($testnum, $testname, "output ($filename)",

                           \@generated, \@outfile);

            if($res) {

                # timestamp test result verification end

                $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);

                return 1;

            }
@@ -3614,8 +3608,13 @@ sub singletest { 
            }

            my @e = valgrindparse($srcdir, $feature{'SSL'}, "$LOGDIR/$vgfile");

            if(@e && $e[0]) {

                if($automakestyle) {

                    logmsg "FAIL: $testnum - $testname - valgrind\n";

                }

                else {

                    logmsg " valgrind ERROR ";

                    logmsg @e;

                }

                # timestamp test result verification end

                $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);

                return 1;
@@ -3641,7 +3640,13 @@ sub singletest { 
    my $left=sprintf("remaining: %02d:%02d",

                     $estleft/60,

                     $estleft%60);



    if(!$automakestyle) {

        logmsg sprintf("OK (%-3d out of %-3d, %s)\n", $count, $total, $left);

    }

    else {

        logmsg "PASS: $testnum - $testname\n";

    }



    # the test succeeded, remove all log files

    if(!$keepoutfiles) {
@@ -4284,6 +4289,11 @@ while(@ARGV) { 
        # short output

        $short=1;

    }

    elsif($ARGV[0] eq "-am") {

        # automake-style output

        $short=1;

        $automakestyle=1;

    }

    elsif($ARGV[0] eq "-n") {

        # no valgrind

        undef $valgrind;
@@ -4360,6 +4370,7 @@ Usage: runtests.pl [options] [test selection(s)] 
  -r       run time statistics

  -rf      full run time statistics

  -s       short output

  -am      automake style output PASS/FAIL: [number] [name]

  -t[N]    torture (simulate memory alloc failures); N means fail Nth alloc

  -v       verbose output

  [num]    like "5 6 9" or " 5 to 22 " to run those tests only