Commit 5584fd7f authored by Pauli's avatar Pauli Committed by Richard Levitte
Browse files

Randomise the ordering of the C unit tests. 



Reviewed-by: default avatarRich Salz <rsalz@openssl.org>
Reviewed-by: default avatarRichard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3393)
parent e0011aa8

test/bioprinttest.c

+46 −29
Original line number Diff line number Diff line 
/*

 * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.

 * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.

 *

 * Licensed under the OpenSSL license (the "License").  You may not use

 * this file except in compliance with the License.  You can obtain a copy
@@ -20,17 +20,20 @@ 


static int justprint = 0;



static char *fpexpected[][5] = {

    /*   0 */ { "0.0000e+00", "0.0000", "0", "0.0000E+00", "0" },

    /*   1 */ { "6.7000e-01", "0.6700", "0.67", "6.7000E-01", "0.67" },

    /*   2 */ { "6.6667e-01", "0.6667", "0.6667", "6.6667E-01", "0.6667" },

    /*   3 */ { "6.6667e-04", "0.0007", "0.0006667", "6.6667E-04", "0.0006667" },

    /*   4 */ { "6.6667e-05", "0.0001", "6.667e-05", "6.6667E-05", "6.667E-05" },

    /*   5 */ { "6.6667e+00", "6.6667", "6.667", "6.6667E+00", "6.667" },

    /*   6 */ { "6.6667e+01", "66.6667", "66.67", "6.6667E+01", "66.67" },

    /*   7 */ { "6.6667e+02", "666.6667", "666.7", "6.6667E+02", "666.7" },

    /*   8 */ { "6.6667e+03", "6666.6667", "6667", "6.6667E+03", "6667" },

    /*   9 */ { "6.6667e+04", "66666.6667", "6.667e+04", "6.6667E+04", "6.667E+04" },

static char *fpexpected[][10][5] = {

    {

    /*  00 */ { "0.0000e+00", "0.0000", "0", "0.0000E+00", "0" },

    /*  01 */ { "6.7000e-01", "0.6700", "0.67", "6.7000E-01", "0.67" },

    /*  02 */ { "6.6667e-01", "0.6667", "0.6667", "6.6667E-01", "0.6667" },

    /*  03 */ { "6.6667e-04", "0.0007", "0.0006667", "6.6667E-04", "0.0006667" },

    /*  04 */ { "6.6667e-05", "0.0001", "6.667e-05", "6.6667E-05", "6.667E-05" },

    /*  05 */ { "6.6667e+00", "6.6667", "6.667", "6.6667E+00", "6.667" },

    /*  06 */ { "6.6667e+01", "66.6667", "66.67", "6.6667E+01", "66.67" },

    /*  07 */ { "6.6667e+02", "666.6667", "666.7", "6.6667E+02", "666.7" },

    /*  08 */ { "6.6667e+03", "6666.6667", "6667", "6.6667E+03", "6667" },

    /*  09 */ { "6.6667e+04", "66666.6667", "6.667e+04", "6.6667E+04", "6.667E+04" },

    },

    {

    /*  10 */ { "0.00000e+00", "0.00000", "0", "0.00000E+00", "0" },

    /*  11 */ { "6.70000e-01", "0.67000", "0.67", "6.70000E-01", "0.67" },

    /*  12 */ { "6.66667e-01", "0.66667", "0.66667", "6.66667E-01", "0.66667" },
@@ -41,6 +44,8 @@ static char *fpexpected[][5] = { 
    /*  17 */ { "6.66667e+02", "666.66667", "666.67", "6.66667E+02", "666.67" },

    /*  18 */ { "6.66667e+03", "6666.66667", "6666.7", "6.66667E+03", "6666.7" },

    /*  19 */ { "6.66667e+04", "66666.66667", "66667", "6.66667E+04", "66667" },

    },

    {

    /*  20 */ { "  0.0000e+00", "      0.0000", "           0", "  0.0000E+00", "           0" },

    /*  21 */ { "  6.7000e-01", "      0.6700", "        0.67", "  6.7000E-01", "        0.67" },

    /*  22 */ { "  6.6667e-01", "      0.6667", "      0.6667", "  6.6667E-01", "      0.6667" },
@@ -51,6 +56,8 @@ static char *fpexpected[][5] = { 
    /*  27 */ { "  6.6667e+02", "    666.6667", "       666.7", "  6.6667E+02", "       666.7" },

    /*  28 */ { "  6.6667e+03", "   6666.6667", "        6667", "  6.6667E+03", "        6667" },

    /*  29 */ { "  6.6667e+04", "  66666.6667", "   6.667e+04", "  6.6667E+04", "   6.667E+04" },

    },

    {

    /*  30 */ { " 0.00000e+00", "     0.00000", "           0", " 0.00000E+00", "           0" },

    /*  31 */ { " 6.70000e-01", "     0.67000", "        0.67", " 6.70000E-01", "        0.67" },

    /*  32 */ { " 6.66667e-01", "     0.66667", "     0.66667", " 6.66667E-01", "     0.66667" },
@@ -61,6 +68,8 @@ static char *fpexpected[][5] = { 
    /*  37 */ { " 6.66667e+02", "   666.66667", "      666.67", " 6.66667E+02", "      666.67" },

    /*  38 */ { " 6.66667e+03", "  6666.66667", "      6666.7", " 6.66667E+03", "      6666.7" },

    /*  39 */ { " 6.66667e+04", " 66666.66667", "       66667", " 6.66667E+04", "       66667" },

    },

    {

    /*  40 */ { "0e+00", "0", "0", "0E+00", "0" },

    /*  41 */ { "7e-01", "1", "0.7", "7E-01", "0.7" },

    /*  42 */ { "7e-01", "1", "0.7", "7E-01", "0.7" },
@@ -71,6 +80,8 @@ static char *fpexpected[][5] = { 
    /*  47 */ { "7e+02", "667", "7e+02", "7E+02", "7E+02" },

    /*  48 */ { "7e+03", "6667", "7e+03", "7E+03", "7E+03" },

    /*  49 */ { "7e+04", "66667", "7e+04", "7E+04", "7E+04" },

    },

    {

    /*  50 */ { "0.000000e+00", "0.000000", "0", "0.000000E+00", "0" },

    /*  51 */ { "6.700000e-01", "0.670000", "0.67", "6.700000E-01", "0.67" },

    /*  52 */ { "6.666667e-01", "0.666667", "0.666667", "6.666667E-01", "0.666667" },
@@ -81,6 +92,8 @@ static char *fpexpected[][5] = { 
    /*  57 */ { "6.666667e+02", "666.666667", "666.667", "6.666667E+02", "666.667" },

    /*  58 */ { "6.666667e+03", "6666.666667", "6666.67", "6.666667E+03", "6666.67" },

    /*  59 */ { "6.666667e+04", "66666.666667", "66666.7", "6.666667E+04", "66666.7" },

    },

    {

    /*  60 */ { "0.0000e+00", "000.0000", "00000000", "0.0000E+00", "00000000" },

    /*  61 */ { "6.7000e-01", "000.6700", "00000.67", "6.7000E-01", "00000.67" },

    /*  62 */ { "6.6667e-01", "000.6667", "000.6667", "6.6667E-01", "000.6667" },
@@ -91,6 +104,7 @@ static char *fpexpected[][5] = { 
    /*  67 */ { "6.6667e+02", "666.6667", "000666.7", "6.6667E+02", "000666.7" },

    /*  68 */ { "6.6667e+03", "6666.6667", "00006667", "6.6667E+03", "00006667" },

    /*  69 */ { "6.6667e+04", "66666.6667", "6.667e+04", "6.6667E+04", "6.667E+04" },

    },

};



typedef struct z_data_st {
@@ -170,7 +184,7 @@ static pw pw_params[] = { 
    { 4, "08" }

};



static int dofptest(int test, double val, const char *width, int prec)

static int dofptest(int test, int sub, double val, const char *width, int prec)

{

    static const char *fspecs[] = {

        "e", "f", "g", "E", "G"
@@ -190,12 +204,12 @@ static int dofptest(int test, double val, const char *width, int prec) 


        if (justprint) {

            if (i == 0)

                printf("    /* %d */  { \"%s\"", test, result);

                printf("    /*  %d%d */ { \"%s\"", test, sub, result);

            else

                printf(", \"%s\"", result);

        } else if (!TEST_str_eq(fpexpected[test][i], result)) {

        } else if (!TEST_str_eq(fpexpected[test][sub][i], result)) {

            TEST_info("test %d format=|%s| exp=|%s|, ret=|%s|",

                    test, format, fpexpected[test][i], result);

                    test, format, fpexpected[test][sub][i], result);

            ret = 0;

        }

    }
@@ -206,22 +220,25 @@ static int dofptest(int test, double val, const char *width, int prec) 


static int test_fp(int i)

{

    static int t = 0;

    int t = 0, r;

    const double frac = 2.0 / 3.0;

    const pw *pwp = &pw_params[i];



        if (!TEST_true(dofptest(t++, 0.0, pwp->w, pwp->p))

                || !TEST_true(dofptest(t++, 0.67, pwp->w, pwp->p))

                || !TEST_true(dofptest(t++, frac, pwp->w, pwp->p))

                || !TEST_true(dofptest(t++, frac / 1000, pwp->w, pwp->p))

                || !TEST_true(dofptest(t++, frac / 10000, pwp->w, pwp->p))

                || !TEST_true(dofptest(t++, 6.0 + frac, pwp->w, pwp->p))

                || !TEST_true(dofptest(t++, 66.0 + frac, pwp->w, pwp->p))

                || !TEST_true(dofptest(t++, 666.0 + frac, pwp->w, pwp->p))

                || !TEST_true(dofptest(t++, 6666.0 + frac, pwp->w, pwp->p))

                || !TEST_true(dofptest(t++, 66666.0 + frac, pwp->w, pwp->p)))

            return 0;

    return 1;

    if (justprint)

        printf("    {\n");

    r = TEST_true(dofptest(i, t++, 0.0, pwp->w, pwp->p))

        && TEST_true(dofptest(i, t++, 0.67, pwp->w, pwp->p))

        && TEST_true(dofptest(i, t++, frac, pwp->w, pwp->p))

        && TEST_true(dofptest(i, t++, frac / 1000, pwp->w, pwp->p))

        && TEST_true(dofptest(i, t++, frac / 10000, pwp->w, pwp->p))

        && TEST_true(dofptest(i, t++, 6.0 + frac, pwp->w, pwp->p))

        && TEST_true(dofptest(i, t++, 66.0 + frac, pwp->w, pwp->p))

        && TEST_true(dofptest(i, t++, 666.0 + frac, pwp->w, pwp->p))

        && TEST_true(dofptest(i, t++, 6666.0 + frac, pwp->w, pwp->p))

        && TEST_true(dofptest(i, t++, 66666.0 + frac, pwp->w, pwp->p));

    if (justprint)

        printf("    },\n");

    return r;

}



static int test_big(void)

test/testutil/driver.c

+52 −10
Original line number Diff line number Diff line
@@ -32,6 +32,7 @@ typedef struct test_info { 


static TEST_INFO all_tests[1024];

static int num_tests = 0;

static int seed = 0;

/*

 * A parameterised tests runs a loop of test cases.

 * |num_test_cases| counts the total number of test cases
@@ -84,14 +85,34 @@ static int should_report_leaks() 
}

#endif



static int gcd(int a, int b)

{

    while (b != 0) {

        int t = b;

        b = a % b;

        a = t;

    }

    return a;

}



void setup_test()

{

    char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");

    char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER");



    test_open_streams();



    level = TAP_levels != NULL ? 4 * atoi(TAP_levels) : 0;



    if (test_seed != NULL) {

        seed = atoi(test_seed);

        if (seed <= 0)

            seed = time(NULL);

        test_printf_stdout("%*s# RAND SEED %d\n", subtest_level(), "", seed);

        test_flush_stdout();

        srand(seed);

    }



#ifndef OPENSSL_NO_CRYPTO_MDEBUG

    if (should_report_leaks()) {

        CRYPTO_set_mem_debug(1);
@@ -133,7 +154,8 @@ int run_tests(const char *test_prog_name) 
{

    int num_failed = 0;

    char *verdict = NULL;

    int i, j;

    int ii, i, jj, j, jstep;

    int permute[OSSL_NELEM(all_tests)];



    if (num_tests < 1) {

        test_printf_stdout("%*s1..0 # Skipped: %s\n", level, "",
@@ -145,7 +167,18 @@ int run_tests(const char *test_prog_name) 
    }

    test_flush_stdout();



    for (i = 0; i != num_tests; ++i) {

    for (i = 0; i < num_tests; i++)

        permute[i] = i;

    if (seed != 0)

        for (i = num_tests - 1; i >= 1; i--) {

            j = rand() % (1 + i);

            ii = permute[j];

            permute[j] = permute[i];

            permute[i] = ii;

        }



    for (ii = 0; ii != num_tests; ++ii) {

        i = permute[ii];

        if (all_tests[i].num == -1) {

            int ret = 0;
@@ -160,7 +193,7 @@ int run_tests(const char *test_prog_name) 
                verdict = "not ok";

                ++num_failed;

            }

            test_printf_stdout("%*s%s %d - %s\n", level, "", verdict, i + 1,

            test_printf_stdout("%*s%s %d - %s\n", level, "", verdict, ii + 1,

                               test_title);

            test_flush_stdout();

            finalize(ret);
@@ -176,9 +209,18 @@ int run_tests(const char *test_prog_name) 
                test_flush_stdout();

            }



            for (j = 0; j < all_tests[i].num; j++) {

                int ret = 0;

            j = -1;

            if (seed == 0 || all_tests[i].num < 3)

                jstep = 1;

            else

                do

                    jstep = rand() % all_tests[i].num;

                while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);



            for (jj = 0; jj < all_tests[i].num; jj++) {

                int ret;



                j = (j + jstep) % all_tests[i].num;

                set_test_title(NULL);

                ret = all_tests[i].param_test_fn(j);
@@ -197,11 +239,11 @@ int run_tests(const char *test_prog_name) 
                        ++num_failed_inner;

                    }

                    if (test_title != NULL)

                        test_printf_stdout("%*s%s %d - %s\n", level, "", verdict,

                                           j + 1, test_title);

                        test_printf_stdout("%*s%s %d - %s\n", level, "",

                                           verdict, jj + 1, test_title);

                    else

                        test_printf_stdout("%*s%s %d\n", level, "", verdict,

                                           j + 1);

                        test_printf_stdout("%*s%s %d - iteration %d\n", level,

                                           "", verdict, jj + 1, j + 1);

                    test_flush_stdout();

                }

            }
@@ -212,7 +254,7 @@ int run_tests(const char *test_prog_name) 
                verdict = "not ok";

                ++num_failed;

            }

            test_printf_stdout("%*s%s %d - %s\n", level, "", verdict, i + 1,

            test_printf_stdout("%*s%s %d - %s\n", level, "", verdict, ii + 1,

                               all_tests[i].test_case_name);

            test_flush_stdout();

        }