Commit 067b8f23 authored by Daniel Stenberg's avatar Daniel Stenberg
Browse files

runtests: add -o to run test cases in scrambled order 

... instead of numerical order.

Closes #1466
parent 79bb0afd

tests/runtests.1

+3 −1
Original line number Diff line number Diff line
@@ -5,7 +5,7 @@ 
.\" *                            | (__| |_| |  _ <| |___

.\" *                             \___|\___/|_| \_\_____|

.\" *

.\" * Copyright (C) 1998 - 2013, Daniel Stenberg, <daniel@haxx.se>, et al.

.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.

.\" *

.\" * This software is licensed as described in the file COPYING, which

.\" * you should have received as part of this distribution. The terms
@@ -71,6 +71,8 @@ detected. Useful for debugging. 
Lists all test case names.

.IP "-n"

Disable the check for and use of valgrind.

.IP "-o"

Run the tests in a scrambled, or randomized, order instead of sequentially.

.IP "-p"

Prints out all files in "log/" to stdout when a test case fails. Very

practical when used in the automated and distributed tests since then the

tests/runtests.pl

+24 −5
Original line number Diff line number Diff line
@@ -194,7 +194,7 @@ my $pwd = getcwd(); # current working directory 


my $start;

my $ftpchecktime=1; # time it took to verify our test FTP server



my $scrambleorder;

my $stunnel = checkcmd("stunnel4") || checkcmd("tstunnel") || checkcmd("stunnel");

my $valgrind = checktestcmd("valgrind");

my $valgrind_logfile="--logfile";
@@ -4870,6 +4870,10 @@ while(@ARGV) { 
        # have the servers display protocol output

        $debugprotocol=1;

    }

    elsif($ARGV[0] eq "-e") {

        # run the tests cases event based if possible

        $run_event_based=1;

    }

    elsif ($ARGV[0] eq "-g") {

        # run this test with gdb

        $gdbthis=1;
@@ -4892,6 +4896,10 @@ while(@ARGV) { 
        # no valgrind

        undef $valgrind;

    }

    elsif ($ARGV[0] eq "-o") {

        # execute in scrambled order

        $scrambleorder=1;

    }

    elsif($ARGV[0] =~ /^-t(.*)/) {

        # torture

        $torture=1;
@@ -4907,10 +4915,6 @@ while(@ARGV) { 
        # continue anyway, even if a test fail

        $anyway=1;

    }

    elsif($ARGV[0] eq "-e") {

        # run the tests cases event based if possible

        $run_event_based=1;

    }

    elsif($ARGV[0] eq "-p") {

        $postmortem=1;

    }
@@ -4958,12 +4962,14 @@ Usage: runtests.pl [options] [test selection(s)] 
  -bN      use base port number N for test servers (default $base)

  -c path  use this curl executable

  -d       display server debug info

  -e       event-based execution

  -g       run the test case with gdb

  -gw      run the test case with gdb as a windowed application

  -h       this help text

  -k       keep stdout and stderr files present after tests

  -l       list all test case names/descriptions

  -n       no valgrind

  -o       scrambled order

  -p       print log file contents when a test fails

  -r       run time statistics

  -rf      full run time statistics
@@ -5188,6 +5194,19 @@ else { 
    $TESTCASES = $verified;

}



if($scrambleorder) {

    # scramble the order of the test cases

    my @rand;

    while($TESTCASES) {

        my @all = split(/ /, $TESTCASES);

        my $r = rand @all;

        push @rand, $all[$r];

        $all[$r]="";

        $TESTCASES = join(" ", @all);

    }

    $TESTCASES = join(" ", @rand);

}



#######################################################################

# Start the command line log

#