runtests.pl

        }

        my $strip;
        for $strip (@stripfile) {
            chomp $strip;
            for(@generated) {
                eval $strip;
            }
        }

        @outfile = fixarray(@outfile);

        $res = compare("output", \@generated, \@outfile);
        if($res) {
            return 1;
        }

        $ok .= "o";
    }
    else {
        $ok .= "-"; # output not checked
    }

    # accept multiple comma-separated error codes
    my @splerr = split(/ *, */, $errorcode);
    my $errok;
    foreach $e (@splerr) {
        if($e == $cmdres) {
            # a fine error code
            $errok = 1;
            last;
        }
    }

    if($errok) {
        $ok .= "e";
    }
    else {
        if(!$short) {
            printf "\ncurl returned $cmdres, %s was expected\n", $errorcode;
        }
        logmsg " exit FAILED\n";
        return 1;
    }

    @what = getpart("client", "killserver");
    for(@what) {
        my $serv = $_;
        chomp $serv;
        if($serv =~ /^ftp(\d*)(-ipv6|)/) {
            my ($id, $ext) = ($1, $2);
            #print STDERR "SERV $serv $id $ext\n";
            ftpkillslave($id, $ext, $verbose);
        }
        if($run{$serv}) {
            stopserver($run{$serv}); # the pid file is in the hash table
            $run{$serv}=0; # clear pid
        }
        else {
            logmsg "RUN: The $serv server is not running\n";
        }
    }

    if($curl_debug) {
        if(! -f $memdump) {
            logmsg "\n** ALERT! memory debugging with no output file?\n";
        }
        else {
            my @memdata=`$memanalyze $memdump`;
            my $leak=0;
            for(@memdata) {
                if($_ ne "") {
                    # well it could be other memory problems as well, but
                    # we call it leak for short here
                    $leak=1;
                }
            }
            if($leak) {
                logmsg "\n** MEMORY FAILURE\n";
                logmsg @memdata;
                return 1;
            }
            else {
                $ok .= "m";
            }
        }
    }
    else {
        $ok .= "-"; # memory not checked
    }

    if($valgrind) {
        # this is the valid protocol blurb curl should generate
        if($usevalgrind) {

            opendir(DIR, "log") ||
                return 0; # can't open log dir
            my @files = readdir(DIR);
            closedir(DIR);
            my $f;
            my $l;
            foreach $f (@files) {
                if($f =~ /^valgrind$testnum\.pid/) {
                    $l = $f;
                    last;
                }
            }
            my $src=$ENV{'srcdir'};
            if(!$src) {
                $src=".";
            }
            my @e = valgrindparse($src, $feature{'SSL'}, "log/$l");
            if($e[0]) {
                logmsg " valgrind ERROR ";
                logmsg @e;
                return 1;
            }
            $ok .= "v";
        }
        else {
            if(!$short) {
                logmsg " valgrind SKIPPED\n";
            }
            $ok .= "-"; # skipped
        }
    }
    else {
        $ok .= "-"; # valgrind not checked
    }

    logmsg "$ok " if(!$short);

    my $sofar= time()-$start;
    my $esttotal = $sofar/$count * $total;
    my $estleft = $esttotal - $sofar;
    my $left=sprintf("remaining: %02d:%02d",
                     $estleft/60,
                     $estleft%60);
    printf "OK (%-3d out of %-3d, %s)\n", $count, $total, $left;

    # the test succeeded, remove all log files
    if(!$keepoutfiles) {
        cleardir($LOGDIR);
    }

    unlink($FTPDCMD); # remove the instructions for this test

    return 0;
}

#######################################################################
# Stop all running test servers
sub stopservers {
    my ($verbose)=@_;
    for(keys %run) {
        my $server = $_;
        my $pids=$run{$server};
        my $pid;
        my $prev;

        foreach $pid (split(" ", $pids)) {
            if($pid != $prev) {
                # no need to kill same pid twice!
                logmsg sprintf("* kill pid for %s => %d\n",
                               $server, $pid) if($verbose);
                stopserver($pid);
            }
            $prev = $pid;
        }
    }
    ftpkillslaves($verbose);
}

#######################################################################
# startservers() starts all the named servers
#
# Returns: string with error reason or blank for success

sub startservers {
    my @what = @_;
    my ($pid, $pid2);
    for(@what) {
        my $what = lc($_);
        $what =~ s/[^a-z0-9-]//g;
        if($what eq "ftp") {
            if(!$run{'ftp'}) {
                ($pid, $pid2) = runftpserver("", $verbose);
                if($pid <= 0) {
                    return "failed starting FTP server";
                }
                printf ("* pid ftp => %d %d\n", $pid, $pid2) if($verbose);
                $run{'ftp'}="$pid $pid2";
            }
        }
        elsif($what eq "ftp2") {
            if(!$run{'ftp2'}) {
                ($pid, $pid2) = runftpserver("2", $verbose);
                if($pid <= 0) {
                    return "failed starting FTP2 server";
                }
                printf ("* pid ftp2 => %d %d\n", $pid, $pid2) if($verbose);
                $run{'ftp2'}="$pid $pid2";
            }
        }
        elsif($what eq "ftp-ipv6") {
            if(!$run{'ftp-ipv6'}) {
                ($pid, $pid2) = runftpserver("", $verbose, "ipv6");
                if($pid <= 0) {
                    return "failed starting FTP-IPv6 server";
                }
                logmsg sprintf("* pid ftp-ipv6 => %d %d\n", $pid,
                       $pid2) if($verbose);
                $run{'ftp-ipv6'}="$pid $pid2";
            }
        }
        elsif($what eq "http") {
            if(!$run{'http'}) {
                ($pid, $pid2) = runhttpserver($verbose);
                if($pid <= 0) {
                    return "failed starting HTTP server";
                }
                printf ("* pid http => %d %d\n", $pid, $pid2) if($verbose);
                $run{'http'}="$pid $pid2";
            }
        }
        elsif($what eq "http-ipv6") {
            if(!$run{'http-ipv6'}) {
                ($pid, $pid2) = runhttpserver($verbose, "IPv6");
                if($pid <= 0) {
                    return "failed starting HTTP-IPv6 server";
                }
                logmsg sprintf("* pid http-ipv6 => %d %d\n", $pid, $pid2)
                    if($verbose);
                $run{'http-ipv6'}="$pid $pid2";
            }
        }
        elsif($what eq "ftps") {
            if(!$stunnel) {
                # we can't run ftps tests without stunnel
                return "no stunnel";
            }
            if(!$ssl_version) {
                # we can't run ftps tests if libcurl is SSL-less
                return "curl lacks SSL support";
            }

            if(!$run{'ftp'}) {
                ($pid, $pid2) = runftpserver("", $verbose);
                if($pid <= 0) {
                    return "failed starting FTP server";
                }
                printf ("* pid ftp => %d %d\n", $pid, $pid2) if($verbose);
                $run{'ftp'}="$pid $pid2";
            }
            if(!$run{'ftps'}) {
                ($pid, $pid2) = runftpsserver($verbose);
                if($pid <= 0) {
                    return "failed starting FTPS server (stunnel)";
                }
                logmsg sprintf("* pid ftps => %d %d\n", $pid, $pid2)
                    if($verbose);
                $run{'ftps'}="$pid $pid2";
            }
        }
        elsif($what eq "file") {
            # we support it but have no server!
        }
        elsif($what eq "https") {
            if(!$stunnel) {
                # we can't run ftps tests without stunnel
                return "no stunnel";
            }
            if(!$ssl_version) {
                # we can't run ftps tests if libcurl is SSL-less
                return "curl lacks SSL support";
            }

            if(!$run{'http'}) {
                ($pid, $pid2) = runhttpserver($verbose);
                if($pid <= 0) {
                    return "failed starting HTTP server";
                }
                printf ("* pid http => %d %d\n", $pid, $pid2) if($verbose);
                $run{'http'}="$pid $pid2";
            }
            if(!$run{'https'}) {
                ($pid, $pid2) = runhttpsserver($verbose);
                if($pid <= 0) {
                    return "failed starting HTTPS server (stunnel)";
                }
                logmsg sprintf("* pid https => %d %d\n", $pid, $pid2)
                    if($verbose);
                $run{'https'}="$pid $pid2";
            }
        }
        elsif($what eq "tftp") {
            if(!$run{'tftp'}) {
                ($pid, $pid2) = runtftpserver("", $verbose);
                if($pid <= 0) {
                    return "failed starting TFTP server";
                }
                printf ("* pid tftp => %d %d\n", $pid, $pid2) if($verbose);
                $run{'tftp'}="$pid $pid2";
            }
        }
        elsif($what eq "tftp-ipv6") {
            if(!$run{'tftp-ipv6'}) {
                ($pid, $pid2) = runtftpserver("", $verbose, "IPv6");
                if($pid <= 0) {
                    return "failed starting TFTP-IPv6 server";
                }
                printf("* pid tftp-ipv6 => %d %d\n", $pid, $pid2) if($verbose);
                $run{'tftp-ipv6'}="$pid $pid2";
            }
        }
        elsif($what eq "sftp" || $what eq "scp" || $what eq "socks4" || $what eq "socks5" ) {
            if(!$run{'ssh'}) {
                ($pid, $pid2) = runsshserver("", $verbose);
                if($pid <= 0) {
                    return "failed starting SSH server";
                }
                printf ("* pid ssh => %d %d\n", $pid, $pid2) if($verbose);
                $run{'ssh'}="$pid $pid2";
            }
	    if ($what eq "socks4" || $what eq "socks5") {
                if (!checkcmd("ssh")) {
                   return "failed to find SSH client for socks support";
		}
            	if(!$run{'socks'}) {
		    my $sshversion=`ssh -V 2>&1`;
                    if ($sshversion =~ /SSH_(\d+)\.(\d+)/i) {
                       if ($1*10+$2 < 37) {
		       # need 3.7 for socks5 - http://www.openssh.com/txt/release-3.7
                           return "ssh version ($1.$2) insufficient; need at least 3.7";
		       }
                    } else {
                       return "Unsupported ssh client\n";
                    }

                    ($pid, $pid2) = runsocksserver("", $verbose);
                    if($pid <= 0) {
                        return "failed starting socks server";
                    }
                    printf ("* pid socks => %d %d\n", $pid, $pid2) if($verbose);
                    $run{'socks'}="$pid $pid2";
		}
	    }
        }
        elsif($what eq "none") {
            logmsg "* starts no server\n" if ($verbose);
        }
        else {
            warn "we don't support a server for $what";
            return "no server for $what";
        }
    }
    return 0;
}

##############################################################################
# This function makes sure the right set of server is running for the
# specified test case. This is a useful design when we run single tests as not
# all servers need to run then!
#
# Returns: a string, blank if everything is fine or a reason why it failed
#

sub serverfortest {
    my ($testnum)=@_;

    my @what = getpart("client", "server");

    if(!$what[0]) {
        warn "Test case $testnum has no server(s) specified";
        return "no server specified";
    }

    for (@what) {
	my $proto = lc($_);
	chomp $proto;
	if (! grep /^$proto$/, @protocols) {
	    if (substr($proto,0,5) ne "socks") {
		    return "curl lacks any $proto support";
	    }
	}
    }

    return &startservers(@what);
}

#######################################################################
# Check options to this test program
#

my $number=0;
my $fromnum=-1;
my @testthis;
my %disabled;
while(@ARGV) {
    if ($ARGV[0] eq "-v") {
        # verbose output
        $verbose=1;
    }
    elsif($ARGV[0] =~ /^-b(.*)/) {
        my $portno=$1;
        if($portno =~ s/(\d+)$//) {
            $base = int $1;
        }
    }
    elsif ($ARGV[0] eq "-c") {
        # use this path to curl instead of default
        $DBGCURL=$CURL=$ARGV[1];
        shift @ARGV;
    }
    elsif ($ARGV[0] eq "-d") {
        # have the servers display protocol output
        $debugprotocol=1;
    }
    elsif ($ARGV[0] eq "-f") {
        # run fork-servers, which makes the server fork for all new
        # connections This is NOT what you wanna do without knowing exactly
        # why and for what
        $forkserver=1;
    }
    elsif ($ARGV[0] eq "-g") {
        # run this test with gdb
        $gdbthis=1;
    }
    elsif($ARGV[0] eq "-s") {
        # short output
        $short=1;
    }
    elsif($ARGV[0] eq "-n") {
        # no valgrind
        undef $valgrind;
    }
    elsif($ARGV[0] =~ /^-t(.*)/) {
        # torture
        $torture=1;
        my $xtra = $1;

        if($xtra =~ s/(\d+)$//) {
            $tortalloc = $1;
        }
        # we undef valgrind to make this fly in comparison
        undef $valgrind;
    }
    elsif($ARGV[0] eq "-a") {
        # continue anyway, even if a test fail
        $anyway=1;
    }
    elsif($ARGV[0] eq "-p") {
        $postmortem=1;
    }
    elsif($ARGV[0] eq "-l") {
        # lists the test case names only
        $listonly=1;
    }
    elsif($ARGV[0] eq "-k") {
        # keep stdout and stderr files after tests
        $keepoutfiles=1;
    }
    elsif($ARGV[0] eq "-h") {
        # show help text
        print <<EOHELP
Usage: runtests.pl [options] [test number(s)]
  -a       continue even if a test fails
  -bN      use base port number N for test servers (default $base)
  -c path  use this curl executable
  -d       display server debug info
  -g       run the test case with gdb
  -h       this help text
  -k       keep stdout and stderr files present after tests
  -l       list all test case names/descriptions
  -n       no valgrind
  -p       print log file contents when a test fails
  -s       short output
  -t[N]    torture (simulate memory alloc failures); N means fail Nth alloc
  -v       verbose output
  [num]    like "5 6 9" or " 5 to 22 " to run those tests only
  [!num]   like "!5 !6 !9" to disable those tests
  [!keyword] like "!cookies !IPv6" to disable tests with those key words
EOHELP
    ;
        exit;
    }
    elsif($ARGV[0] =~ /^(\d+)/) {
        $number = $1;
        if($fromnum >= 0) {
            for($fromnum .. $number) {
                push @testthis, $_;
            }
            $fromnum = -1;
        }
        else {
            push @testthis, $1;
        }
    }
    elsif($ARGV[0] =~ /^to$/i) {
        $fromnum = $number+1;
    }
    elsif($ARGV[0] =~ /^!(\d+)/) {
        $fromnum = -1;
        $disabled{$1}=$1;
    }
    elsif($ARGV[0] =~ /^!(.+)/) {
        $disabled_keywords{$1}=$1;
    }
    else {
    	print "Unknown option: $ARGV[0]\n";
    	exit;
    }
    shift @ARGV;
} 

if($testthis[0] ne "") {
    $TESTCASES=join(" ", @testthis);
}

if($valgrind) {
    # we have found valgrind on the host, use it

    # verify that we can invoke it fine
    my $code = runclient("valgrind >/dev/null 2>&1");

    if(($code>>8) != 1) {
        #logmsg "Valgrind failure, disable it\n";
        undef $valgrind;
    } else {

	# since valgrind 2.1.x, '--tool' option is mandatory
	# use it, if it is supported by the version installed on the system
	runclient("valgrind --help 2>&1 | grep -- --tool > /dev/null 2>&1");
	if (($? >> 8)==0) {
	    $valgrind_tool="--tool=memcheck ";
	}
	open(C, "<$CURL");
	my $l = <C>;
	if($l =~ /^\#\!/) {
	    # A shell script. This is typically when built with libtool,
	    $valgrind="../libtool --mode=execute $valgrind";
	}
	close(C);

	# valgrind 3 renamed the --logfile option to --log-file!!!
	my $ver=join(' ', runclientoutput("valgrind --version"));
	# cut off all but digits and dots
	$ver =~ s/[^0-9.]//g;

	if($ver >= 3) {
	    $valgrind_logfile="--log-file";
	}
    }
}

if ($gdbthis) {
    # open the executable curl and read the first 4 bytes of it
    open(CHECK, "<$CURL");
    my $c;
    sysread CHECK, $c, 4;
    close(CHECK);
    if($c eq "#! /") {
	# A shell script. This is typically when built with libtool,
	$libtool = 1;
	$gdb = "libtool --mode=execute gdb";
    }
}

$HTTPPORT =  $base + 0; # HTTP server port
$HTTPSPORT = $base + 1; # HTTPS server port
$FTPPORT =   $base + 2; # FTP server port
$FTPSPORT =  $base + 3; # FTPS server port
$HTTP6PORT = $base + 4; # HTTP IPv6 server port (different IP protocol
                        # but we follow the same port scheme anyway)
$FTP2PORT =  $base + 5; # FTP server 2 port
$FTP6PORT =  $base + 6; # FTP IPv6 port
$TFTPPORT =  $base + 7; # TFTP (UDP) port
$TFTP6PORT =  $base + 8; # TFTP IPv6 (UDP) port
$SSHPORT =   $base + 9; # SSH (SCP/SFTP) port
$SOCKSPORT =   $base + 10; # SOCKS port

#######################################################################
# clear and create logging directory:
#

cleardir($LOGDIR);
mkdir($LOGDIR, 0777);

#######################################################################
# Output curl version and host info being tested
#

if(!$listonly) {
    checksystem();
}

#######################################################################
# If 'all' tests are requested, find out all test numbers
#

if ( $TESTCASES eq "all") {
    # Get all commands and find out their test numbers
    opendir(DIR, $TESTDIR) || die "can't opendir $TESTDIR: $!";
    my @cmds = grep { /^test([0-9]+)$/ && -f "$TESTDIR/$_" } readdir(DIR);
    closedir(DIR);

    open(D, "<$TESTDIR/DISABLED");
    while(<D>) {
        if(/^ *\#/) {
            # allow comments
            next;
        }
        if($_ =~ /(\d+)/) {
            $disabled{$1}=$1; # disable this test number
        }
    }
    close(D);

    $TESTCASES=""; # start with no test cases

    # cut off everything but the digits
    for(@cmds) {
        $_ =~ s/[a-z\/\.]*//g;
    }
    # the the numbers from low to high
    foreach my $n (sort { $a <=> $b } @cmds) {
        if($disabled{$n}) {
            # skip disabled test cases
            my $why = "configured as DISABLED";
            $skipped++;
            $skipped{$why}++;
            $teststat[$n]=$why; # store reason for this test case
            next;
        }
        $TESTCASES .= " $n";
    }
}

#######################################################################
# Start the command line log
#
open(CMDLOG, ">$CURLLOG") ||
    logmsg "can't log command lines to $CURLLOG\n";

#######################################################################

# Display the contents of the given file.  Line endings are canonicalized
# and excessively long files are truncated
sub displaylogcontent {
    my ($file)=@_;
    if(open(SINGLE, "<$file")) {
        my $lfcount;
        my $linecount = 0;
        my $truncate;
        my @tail;
        while(my $string = <SINGLE>) {
            $string =~ s/\r\n/\n/g;
            $string =~ s/[\r\f\032]/\n/g;
            $string .= "\n" unless ($string =~ /\n$/);
            $lfcount = $string =~ tr/\n//;
            if($lfcount == 1) {
                $string =~ s/\n//;
                $string =~ s/\s*\!$//;
                $linecount++;
                if ($truncate) {
                    push @tail, " $string\n";
                } else {
                    logmsg " $string\n";
                }
            }
            else {
                for my $line (split("\n", $string)) {
                    $line =~ s/\s*\!$//;
                    $linecount++;
                    if ($truncate) {
                        push @tail, " $line\n";
                    } else {
                        logmsg " $line\n";
		    }
                }
            }
            $truncate = $linecount > 1000;
        }
        if (@tail) {
            logmsg "=== File too long: lines here were removed\n";
            # This won't work properly if time stamps are enabled in logmsg
            logmsg join('',@tail[$#tail-200..$#tail]);
        }
        close(SINGLE);
    }
}

sub displaylogs {
    my ($testnum)=@_;
    opendir(DIR, "$LOGDIR") ||
        die "can't open dir: $!";
    my @logs = readdir(DIR);
    closedir(DIR);

    logmsg "== Contents of files in the log/ dir after test $testnum\n";
    foreach my $log (sort @logs) {
        if($log =~ /\.(\.|)$/) {
            next; # skip "." and ".."
        }
        if($log =~ /^\.nfs/) {
            next; # skip ".nfs"
        }
        if(($log eq "memdump") || ($log eq "core")) {
            next; # skip "memdump" and  "core"
        }
        if((-d "$LOGDIR/$log") || (! -s "$LOGDIR/$log")) {
            next; # skip directory and empty files
        }
        if(($log =~ /^stdout\d+/) && ($log !~ /^stdout$testnum/)) {
            next; # skip stdoutNnn of other tests
        }
        if(($log =~ /^stderr\d+/) && ($log !~ /^stderr$testnum/)) {
            next; # skip stderrNnn of other tests
        }
        if(($log =~ /^upload\d+/) && ($log !~ /^upload$testnum/)) {
            next; # skip uploadNnn of other tests
        }
        if(($log =~ /^curl\d+\.out/) && ($log !~ /^curl$testnum\.out/)) {
            next; # skip curlNnn.out of other tests
        }
        if(($log =~ /^test\d+\.txt/) && ($log !~ /^test$testnum\.txt/)) {
            next; # skip testNnn.txt of other tests
        }
        if(($log =~ /^file\d+\.txt/) && ($log !~ /^file$testnum\.txt/)) {
            next; # skip fileNnn.txt of other tests
        }
        if(($log =~ /^valgrind\d+/) && ($log !~ /^valgrind$testnum/)) {
            next; # skip valgrindNnn of other tests
        }
        logmsg "=== Start of file $log\n";
        displaylogcontent("$LOGDIR/$log");
        logmsg "=== End of file $log\n";
    }
}

#######################################################################
# The main test-loop
#

my $failed;
my $testnum;
my $ok=0;
my $total=0;
my $lasttest;
my @at = split(" ", $TESTCASES);
my $count=0;

$start = time();

foreach $testnum (@at) {

    $lasttest = $testnum if($testnum > $lasttest);
    $count++;

    my $error = singletest($testnum, $count, scalar(@at));
    if($error < 0) {
        # not a test we can run
        next;
    }

    $total++; # number of tests we've run

    if($error>0) {
        $failed.= "$testnum ";
        if($postmortem) {
            # display all files in log/ in a nice way
            displaylogs($testnum);
        }
        if(!$anyway) {
            # a test failed, abort
            logmsg "\n - abort tests\n";
            last;
        }
    }
    elsif(!$error) {
        $ok++; # successful test counter
    }

    # loop for next test
}

#######################################################################
# Close command log
#
close(CMDLOG);

# Tests done, stop the servers
stopservers($verbose);

my $all = $total + $skipped;

if($total) {
    logmsg sprintf("TESTDONE: $ok tests out of $total reported OK: %d%%\n",
                   $ok/$total*100);

    if($ok != $total) {
        logmsg "TESTFAIL: These test cases failed: $failed\n";
    }
}
else {
    logmsg "TESTFAIL: No tests were performed\n";
}

if($all) {
    my $sofar = time()-$start;
    logmsg "TESTDONE: $all tests were considered during $sofar seconds.\n";
}

if($skipped) {
    my $s=0;
    logmsg "TESTINFO: $skipped tests were skipped due to these restraints:\n";

    for(keys %skipped) {
        my $r = $_;
        printf "TESTINFO: \"%s\" %d times (", $r, $skipped{$_};

        # now show all test case numbers that had this reason for being
        # skipped
        my $c=0;
        for(0 .. scalar @teststat) {
            my $t = $_;
            if($teststat[$_] eq $r) {
                logmsg ", " if($c);
                logmsg $_;
                $c++;
            }
        }
        logmsg ")\n";
    }
}
if($total && ($ok != $total)) {
    exit 1;
}