Commit 208d721a authored by Richard Levitte's avatar Richard Levitte
Browse files

TAPify testutil 

With the perl test framework comes the output format TAP
(Test Anything Protocol, see http://testanything.org/

) with
extra extension for subtests.  This change extends that same
output format to any test program using testutil.

In this implementation, each test program is seen as a full test that
can be used as a subtest.  The perl framework passes on the subtest
level to the test programs with the environment variable
HARNESS_OSSL_LEVEL.  Furthermore, and series of tests added with
ADD_ALL_TESTS is regarded as another subtest level.

Reviewed-by: default avatarRich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3296)
parent 65d62488

test/gmdifftest.c

+1 −1
Original line number Diff line number Diff line
@@ -60,5 +60,5 @@ void register_tests(void) 
    if (sizeof(time_t) < 8)

        TEST_info("Skipping; time_t is less than 64-bits");

    else

        ADD_ALL_TESTS(test_gmtime, 1000000);

        ADD_ALL_TESTS_NOSUBTEST(test_gmtime, 1000000);

}

test/testlib/OpenSSL/Test.pm

+13 −1
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); 
$VERSION = "0.8";

@ISA = qw(Exporter);

@EXPORT = (@Test::More::EXPORT, qw(setup run indir cmd app fuzz test

                                   perlapp perltest));

                                   perlapp perltest subtest));

@EXPORT_OK = (@Test::More::EXPORT_OK, qw(bldtop_dir bldtop_file

                                         srctop_dir srctop_file

                                         data_file
@@ -65,6 +65,7 @@ use File::Spec::Functions qw/file_name_is_absolute curdir canonpath splitdir 
use File::Path 2.00 qw/rmtree mkpath/;

use File::Basename;



my $level = 0;



# The name of the test.  This is set by setup() and is used in the other

# functions to verify that setup() has been used.
@@ -454,6 +455,8 @@ sub run { 
        open STDERR, ">", devnull();

    }



    $ENV{HARNESS_OSSL_LEVEL} = $level + 1;



    # The dance we do with $? is the same dance the Unix shells appear to

    # do.  For example, a program that gets aborted (and therefore signals

    # SIGABRT = 6) will appear to exit with the code 134.  We mimic this
@@ -1153,4 +1156,13 @@ inspiration from Andy Polyakov E<lt>appro@openssl.org<gt>. 


=cut



no warnings 'redefine';

sub subtest {

    $level++;



    Test::More::subtest @_;



    $level--;

};



1;

test/testutil.h

+10 −2
Original line number Diff line number Diff line
@@ -53,7 +53,12 @@ 
 * Simple parameterized tests. Calls test_function(idx) for each 0 <= idx < num.

 */

# define ADD_ALL_TESTS(test_function, num) \

  add_all_tests(#test_function, test_function, num)

    add_all_tests(#test_function, test_function, num, 1)

/*

 * A variant of the same without TAP output.

 */

# define ADD_ALL_TESTS_NOSUBTEST(test_function, num) \

    add_all_tests(#test_function, test_function, num, 0)



/*-

 * Test cases that share common setup should use the helper
@@ -131,7 +136,8 @@ void setup_test(void); 
__owur int finish_test(int ret);



void add_test(const char *test_case_name, int (*test_fn) ());

void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num);

void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num,

                   int subtest);

__owur int run_tests(const char *test_prog_name);



/*
@@ -369,3 +375,5 @@ int test_flush_stderr(void); 


extern BIO *bio_out;

extern BIO *bio_err;



int subtest_level(void);

test/testutil/driver.c

+57 −21
Original line number Diff line number Diff line
@@ -23,6 +23,9 @@ typedef struct test_info { 
    int (*test_fn) ();

    int (*param_test_fn)(int idx);

    int num;



    /* flags */

    int subtest:1;

} TEST_INFO;



static TEST_INFO all_tests[1024];
@@ -45,16 +48,24 @@ void add_test(const char *test_case_name, int (*test_fn) ()) 
}



void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),

                   int num)

                   int num, int subtest)

{

    assert(num_tests != OSSL_NELEM(all_tests));

    all_tests[num_tests].test_case_name = test_case_name;

    all_tests[num_tests].param_test_fn = test_fn;

    all_tests[num_tests].num = num;

    all_tests[num_tests].subtest = subtest;

    ++num_tests;

    num_test_cases += num;

}



static int level = 0;



int subtest_level(void)

{

    return level;

}



#ifndef OPENSSL_NO_CRYPTO_MDEBUG

static int should_report_leaks()

{
@@ -71,7 +82,6 @@ static int should_report_leaks() 
}

#endif





static int err_cb(const char *str, size_t len, void *u)

{

    return test_puts_stderr(str);
@@ -79,8 +89,12 @@ static int err_cb(const char *str, size_t len, void *u) 


void setup_test()

{

    char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");



    test_open_streams();



    level = TAP_levels != NULL ? 4 * atoi(TAP_levels) : 0;



#ifndef OPENSSL_NO_CRYPTO_MDEBUG

    if (should_report_leaks()) {

        CRYPTO_set_mem_debug(1);
@@ -121,47 +135,69 @@ static void helper_printf_stdout(const char *fmt, ...) 
int run_tests(const char *test_prog_name)

{

    int num_failed = 0;

    char *verdict = NULL;

    int i, j;



    helper_printf_stdout("%s: %d test case%s\n", test_prog_name, num_test_cases,

                         num_test_cases == 1 ? "" : "s");

    helper_printf_stdout("%*s%d..%d\n", level, "", 1, num_tests);

    test_flush_stdout();



    for (i = 0; i != num_tests; ++i) {

        if (all_tests[i].num == -1) {

            int ret = all_tests[i].test_fn();



            verdict = "ok";

            if (!ret) {

                helper_printf_stdout("** %s failed **\n--------\n",

                                     all_tests[i].test_case_name);

                test_flush_stdout();

                verdict = "not ok";

                ++num_failed;

            }

            helper_printf_stdout("%*s%s %d - %s\n", level, "", verdict, i + 1,

                                     all_tests[i].test_case_name);

            test_flush_stdout();

            finalize(ret);

        } else {

            for (j = 0; j < all_tests[i].num; j++) {

                int ret = all_tests[i].param_test_fn(j);

            int num_failed_inner = 0;



                if (!ret) {

                    helper_printf_stdout("** %s failed test %d\n--------\n",

                                         all_tests[i].test_case_name, j);

            level += 4;

            if (all_tests[i].subtest) {

                helper_printf_stdout("%*s# Subtest: %s\n", level, "",

                                     all_tests[i].test_case_name);

                helper_printf_stdout("%*s%d..%d\n", level, "", 1,

                                     all_tests[i].num);

                test_flush_stdout();

                    ++num_failed;

            }



            for (j = 0; j < all_tests[i].num; j++) {

                int ret = all_tests[i].param_test_fn(j);



                if (!ret)

                    ++num_failed_inner;



                finalize(ret);



                if (all_tests[i].subtest) {

                    verdict = "ok";

                    if (!ret) {

                        verdict = "not ok";

                        ++num_failed_inner;

                    }

                    helper_printf_stdout("%*s%s %d\n", level, "", verdict, j + 1);

                    test_flush_stdout();

                }

            }



    if (num_failed != 0) {

        helper_printf_stdout("%s: %d test%s failed (out of %d)\n",

                             test_prog_name, num_failed,

                             num_failed != 1 ? "s" : "", num_test_cases);

        test_flush_stdout();

        return EXIT_FAILURE;

            level -= 4;

            verdict = "ok";

            if (num_failed_inner) {

                verdict = "not ok";

                ++num_failed;

            }

    helper_printf_stdout("  All tests passed.\n");

            helper_printf_stdout("%*s%s %d - %s\n", level, "", verdict, i + 1,

                                 all_tests[i].test_case_name);

            test_flush_stdout();

        }

    }

    if (num_failed != 0)

        return EXIT_FAILURE;

    return EXIT_SUCCESS;

}

test/testutil/tests.c

+2 −0
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@ 
static void test_fail_message(const char *prefix, const char *file, int line,

                              const char *type, const char *fmt, ...)

            PRINTF_FORMAT(5, 6);

int subtest_level(void);



static void helper_printf_stderr(const char *fmt, ...)

{
@@ -56,6 +57,7 @@ static void helper_printf_stderr(const char *fmt, ...) 
static void test_fail_message_va(const char *prefix, const char *file, int line,

                                 const char *type, const char *fmt, va_list ap)

{

    helper_printf_stderr("%*s# ", subtest_level(), "");

    test_puts_stderr(prefix != NULL ? prefix : "ERROR");

    test_puts_stderr(":");

    if (type)