Commit c5f7a996 authored by Pauli's avatar Pauli
Browse files

Test skip option. 



Provide C test cases with the option to skip tests and subtests.

Reviewed-by: default avatarMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/8695)
parent 1fb3c0af

test/test_test.c

+22 −0
Original line number Diff line number Diff line
@@ -531,6 +531,25 @@ static int test_bn_output(int n) 
    return 1;

}



static int test_skip_one(void)

{

    return TEST_skip("skip test");

}



static int test_skip_many(int n)

{

    return TEST_skip("skip tests: %d", n);

}



static int test_skip_null(void)

{

    /*

     * This is not a recommended way of skipping a test, a reason or

     * description should be included.

     */

    return TEST_skip(NULL);

}



int setup_tests(void)

{

    ADD_TEST(test_int);
@@ -553,5 +572,8 @@ int setup_tests(void) 
    ADD_TEST(test_single_eval);

    ADD_TEST(test_output);

    ADD_ALL_TESTS(test_bn_output, OSSL_NELEM(bn_output_tests));

    ADD_TEST(test_skip_one);

    ADD_TEST(test_skip_null);

    ADD_ALL_TESTS(test_skip_many, 3);

    return 1;

}

test/testutil.h

+5 −0
Original line number Diff line number Diff line
@@ -346,6 +346,9 @@ void test_info(const char *file, int line, const char *desc, ...) 
    PRINTF_FORMAT(3, 4);

void test_info_c90(const char *desc, ...) PRINTF_FORMAT(1, 2);

void test_note(const char *desc, ...) PRINTF_FORMAT(1, 2);

int test_skip(const char *file, int line, const char *desc, ...)

    PRINTF_FORMAT(3, 4);

int test_skip_c90(const char *desc, ...) PRINTF_FORMAT(1, 2);

void test_openssl_errors(void);

void test_perror(const char *s);
@@ -463,9 +466,11 @@ void test_perror(const char *s); 
# if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L

#  define TEST_error         test_error_c90

#  define TEST_info          test_info_c90

#  define TEST_skip          test_skip_c90

# else

#  define TEST_error(...)    test_error(__FILE__, __LINE__, __VA_ARGS__)

#  define TEST_info(...)     test_info(__FILE__, __LINE__, __VA_ARGS__)

#  define TEST_skip(...)     test_skip(__FILE__, __LINE__, __VA_ARGS__)

# endif

# define TEST_note           test_note

# define TEST_openssl_errors test_openssl_errors

test/testutil/driver.c

+22 −33
Original line number Diff line number Diff line
@@ -280,20 +280,20 @@ void set_test_title(const char *title) 
    test_title = title == NULL ? NULL : strdup(title);

}



PRINTF_FORMAT(2, 3) static void test_verdict(int pass, const char *extra, ...)

PRINTF_FORMAT(2, 3) static void test_verdict(int verdict,

                                             const char *description, ...)

{

    va_list ap;



    test_flush_stdout();

    test_flush_stderr();



    test_printf_stdout("%*s%s", level, "", pass ? "ok" : "not ok");

    if (extra != NULL) {

        test_printf_stdout(" ");

        va_start(ap, extra);

        test_vprintf_stdout(extra, ap);

    test_printf_stdout("%*s%s ", level, "", verdict != 0 ? "ok" : "not ok");

    va_start(ap, description);

    test_vprintf_stdout(description, ap);

    va_end(ap);

    }

    if (verdict == TEST_SKIP_CODE)

        test_printf_stdout(" # skipped");

    test_printf_stdout("\n");

    test_flush_stdout();

}
@@ -349,20 +349,14 @@ int run_tests(const char *test_prog_name) 
            }

            test_flush_stdout();

        } else if (all_tests[i].num == -1) {

            int ret = 0;



            set_test_title(all_tests[i].test_case_name);

            ret = all_tests[i].test_fn();

            verdict = 1;

            if (!ret) {

                verdict = 0;

                ++num_failed;

            }

            verdict = all_tests[i].test_fn();

            test_verdict(verdict, "%d - %s", ii + 1, test_title);

            finalize(ret);

            finalize(verdict != 0);

        } else {

            int num_failed_inner = 0;



            verdict = TEST_SKIP_CODE;

            level += 4;

            if (all_tests[i].subtest && single_iter == -1) {

                test_printf_stdout("%*s# Subtest: %s\n", level, "",
@@ -381,39 +375,34 @@ int run_tests(const char *test_prog_name) 
                while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);



            for (jj = 0; jj < all_tests[i].num; jj++) {

                int ret;

                int v;



                j = (j + jstep) % all_tests[i].num;

                if (single_iter != -1 && ((jj + 1) != single_iter))

                    continue;

                set_test_title(NULL);

                ret = all_tests[i].param_test_fn(j);

                v = all_tests[i].param_test_fn(j);



                if (!ret)

                if (v == 0) {

                    ++num_failed_inner;

                    verdict = 0;

                } else if (v != TEST_SKIP_CODE && verdict != 0) {

                    verdict = 1;

                }



                finalize(ret);

                finalize(v != 0);



                if (all_tests[i].subtest) {

                    verdict = 1;

                    if (!ret) {

                        verdict = 0;

                        ++num_failed_inner;

                    }

                    if (test_title != NULL)

                        test_verdict(verdict, "%d - %s", jj + 1, test_title);

                        test_verdict(v, "%d - %s", jj + 1, test_title);

                    else

                        test_verdict(verdict, "%d - iteration %d",

                                     jj + 1, j + 1);

                        test_verdict(v, "%d - iteration %d", jj + 1, j + 1);

                }

            }



            level -= 4;

            verdict = 1;

            if (num_failed_inner) {

                verdict = 0;

            if (verdict == 0)

                ++num_failed;

            }

            test_verdict(verdict, "%d - %s", ii + 1,

                         all_tests[i].test_case_name);

        }

test/testutil/tests.c

+23 −0
Original line number Diff line number Diff line
@@ -157,6 +157,29 @@ void test_note(const char *fmt, ...) 
    test_flush_stderr();

}





int test_skip(const char *file, int line, const char *desc, ...)

{

    va_list ap;



    va_start(ap, desc);

    test_fail_message_va("SKIP", file, line, NULL, NULL, NULL, NULL, desc, ap);

    va_end(ap);

    return TEST_SKIP_CODE;

}



int test_skip_c90(const char *desc, ...)

{

    va_list ap;



    va_start(ap, desc);

    test_fail_message_va("SKIP", NULL, -1, NULL, NULL, NULL, NULL, desc, ap);

    va_end(ap);

    test_printf_stderr("\n");

    return TEST_SKIP_CODE;

}





void test_openssl_errors(void)

{

    ERR_print_errors_cb(openssl_error_cb, NULL);

test/testutil/tu_local.h

+2 −0
Original line number Diff line number Diff line
@@ -12,6 +12,8 @@ 
#include <openssl/bio.h>

#include "../testutil.h"



#define TEST_SKIP_CODE  123



int subtest_level(void);

int openssl_error_cb(const char *str, size_t len, void *u);

const BIO_METHOD *BIO_f_tap(void);