Commit 3088f2b0 authored by Yann Ylavic's avatar Yann Ylavic
Browse files

event, worker: runtime pool. 

MPMs event and worker both need a dedicated pool to handle the creation of
the threads (listener, workers) and synchronization objects (queues, pollset,
mutexes...) in the start_threads() thread, with at least the lifetime of
the connections they handle, and thus survive pchild destruction (notably
in ONE_PROCCESS mode, but SIG_UNGRACEFUL is concerned too).

For instance, without this fix, the below backtrace can happen in ONE_PROCCESS
mode and a signal/^C is received (with active connections):

Thread 1 "httpd" received signal SIGSEGV, Segmentation fault.
(gdb) bt
#0  <BOOM>
#1  0x00007ffff7c7e016 in apr_file_write (thefile=0x0, ...)
                                          ^ NULL (cleared)
                       at file_io/unix/readwrite.c:230
#2  0x00007ffff7c7e4a7 in apr_file_putc (ch=1 '\001', thefile=0x0)
                                                      ^ NULL (cleared)
                       at file_io/unix/readwrite.c:377
#3  0x00007ffff7c8da4a in apr_pollset_wakeup (pollset=0x55555568b870)
                                              ^ already destroyed by pchild
                       at poll/unix/pollset.c:224
#4  0x00007ffff7fc16c7 in decrement_connection_count (cs_=0x7fff08000ea0)
                       at event.c:811
#5  0x00007ffff7c83e15 in run_cleanups (cref=0x7fffe4002b78)
                       at memory/unix/apr_pools.c:2672
#6  0x00007ffff7c82c2f in apr_pool_destroy (pool=0x7fffe4002b58)
                                            ^ master_conn
                       at memory/unix/apr_pools.c:1007
#7  0x00007ffff7c82c12 in apr_pool_destroy (pool=0x7fff08000c28)
                                            ^ ptrans
                       at memory/unix/apr_pools.c:1004
#8  0x00007ffff7c82c12 in apr_pool_destroy (pool=0x555555638698)
                                            ^ pconf
                       at memory/unix/apr_pools.c:1004
#9  0x00007ffff7c82c12 in apr_pool_destroy (pool=0x555555636688)
                                            ^ pglobal
                       at memory/unix/apr_pools.c:1004
#10 0x00005555555f4709 in ap_terminate ()
                       at unixd.c:522
#11 0x00007ffff6dbc8f1 in __run_exit_handlers (...)
                       at exit.c:108
#12 0x00007ffff6dbc9ea in __GI_exit (status=<optimized out>)
                       at exit.c:139
#13 0x00007ffff7fc1616 in clean_child_exit (code=0)
                       at event.c:774
                                  ^ pchild already destroyed here
#14 0x00007ffff7fc5ae4 in child_main (child_num_arg=0, child_bucket=0)
                       at event.c:2869
...


While at it, add comments about the lifetimes of MPMs pools and their objects,
and give each pool a tag (e.g. "pchild" accordingly to other MPMs).



git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1835845 13f79535-47bb-0310-9956-ffa450edef68
parent ab00fec2

server/mpm/event/event.c

+40 −11
Changes for server/mpm/event/event.c: 40 added lines, 11 removed lines.
Original line number Diff line number Diff line
@@ -2437,7 +2437,7 @@ static int check_signal(int signum) 






static void create_listener_thread(thread_starter * ts)

static void create_listener_thread(thread_starter * ts, apr_pool_t *pool)

{

    int my_child_num = ts->child_num_arg;

    apr_threadattr_t *thread_attr = ts->threadattr;
@@ -2448,7 +2448,7 @@ static void create_listener_thread(thread_starter * ts) 
    my_info->pslot = my_child_num;

    my_info->tslot = -1;      /* listener thread doesn't have a thread slot */

    rv = apr_thread_create(&ts->listener, thread_attr, listener_thread,

                           my_info, pchild);

                           my_info, pool);

    if (rv != APR_SUCCESS) {

        ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(00474)

                     "apr_thread_create: unable to create listener thread");
@@ -2469,6 +2469,7 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) 
    apr_thread_t **threads = ts->threads;

    apr_threadattr_t *thread_attr = ts->threadattr;

    int my_child_num = ts->child_num_arg;

    apr_pool_t *pruntime = NULL;

    proc_info *my_info;

    apr_status_t rv;

    int i;
@@ -2486,9 +2487,22 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) 
                                      (apr_uint32_t)threads_per_child *

                                      (async_factor > 2 ? async_factor : 2);



    /* All threads (listener, workers) and synchronization objects (queues,

     * pollset, mutexes...) created here should have at least the lifetime of

     * the connections they handle (i.e. ptrans). We can't use this thread's

     * self pool because all these objects survive it, nor use pchild or pconf

     * directly because this starter thread races with other modules' runtime,

     * nor finally pchild (or subpool thereof) because it is killed explicitely

     * before pconf (thus connections/ptrans can live longer, which matters in

     * ONE_PROCESS mode). So this leaves us with a subpool of pconf, created

     * before any ptrans hence destroyed after.

     */

    apr_pool_create(&pruntime, pconf);

    apr_pool_tag(pruntime, "mpm_runtime");



    /* We must create the fd queues before we start up the listener

     * and worker threads. */

    rv = ap_queue_create(&worker_queue, threads_per_child, pchild);

    rv = ap_queue_create(&worker_queue, threads_per_child, pruntime);

    if (rv != APR_SUCCESS) {

        ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03100)

                     "ap_queue_create() failed");
@@ -2502,7 +2516,7 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) 
         */

        max_recycled_pools = threads_per_child * 3 / 4 ;

    }

    rv = ap_queue_info_create(&worker_queue_info, pchild,

    rv = ap_queue_info_create(&worker_queue_info, pruntime,

                              threads_per_child, max_recycled_pools);

    if (rv != APR_SUCCESS) {

        ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03101)
@@ -2514,7 +2528,7 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) 
     * thread starts.

     */

    rv = apr_thread_mutex_create(&timeout_mutex, APR_THREAD_MUTEX_DEFAULT,

                                 pchild);

                                 pruntime);

    if (rv != APR_SUCCESS) {

        ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(03102)

                     "creation of the timeout mutex failed.");
@@ -2548,12 +2562,13 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) 
        clean_child_exit(APEXIT_CHILDFATAL);

    }



    worker_sockets = apr_pcalloc(pruntime, threads_per_child *

                                           sizeof(apr_socket_t *));



    ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf, APLOGNO(02471)

                 "start_threads: Using %s (%swakeable)",

                 apr_pollset_method_name(event_pollset),

                 listener_is_wakeable ? "" : "not ");

    worker_sockets = apr_pcalloc(pchild, threads_per_child

                                 * sizeof(apr_socket_t *));



    loops = prev_threads_created = 0;

    while (1) {
@@ -2577,7 +2592,7 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) 
             * done because it lets us deal with tid better.

             */

            rv = apr_thread_create(&threads[i], thread_attr,

                                   worker_thread, my_info, pchild);

                                   worker_thread, my_info, pruntime);

            if (rv != APR_SUCCESS) {

                ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf,

                             APLOGNO(03104)
@@ -2590,7 +2605,7 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) 


        /* Start the listener only when there are workers available */

        if (!listener_started && threads_created) {

            create_listener_thread(ts);

            create_listener_thread(ts, pruntime);

            listener_started = 1;

        }
@@ -2707,7 +2722,12 @@ static void child_main(int child_num_arg, int child_bucket) 
    ap_my_pid = getpid();

    ap_child_slot = child_num_arg;

    ap_fatal_signal_child_setup(ap_server_conf);



    /* Get a sub context for global allocations in this child, so that

     * we can have cleanups occur when the child exits.

     */

    apr_pool_create(&pchild, pconf);

    apr_pool_tag(pchild, "pchild");



    /* close unused listeners and pods */

    for (i = 0; i < retained->mpm->num_buckets; i++) {
@@ -2725,9 +2745,18 @@ static void child_main(int child_num_arg, int child_bucket) 
        clean_child_exit(APEXIT_CHILDFATAL);

    }



    apr_thread_mutex_create(&g_timer_skiplist_mtx, APR_THREAD_MUTEX_DEFAULT, pchild);

    /* Event's skiplist operations will happen concurrently with other modules'

     * runtime so they need their own pool for allocations, and its lifetime

     * should be at least the one of the connections (ptrans). Thus pskip is

     * created as a subpool of pconf like/before ptrans (before so that it's

     * destroyed after). In forked mode pconf is never destroyed so we are good

     * anyway, but in ONE_PROCESS mode this ensures that the skiplist works

     * from connection/ptrans cleanups (even after pchild is destroyed).

     */

    apr_pool_create(&pskip, pconf);

    apr_pool_tag(pskip, "mpm_skiplist");

    apr_thread_mutex_create(&g_timer_skiplist_mtx, APR_THREAD_MUTEX_DEFAULT, pskip);

    APR_RING_INIT(&timer_free_ring, timer_event_t, link);

    apr_pool_create(&pskip, pchild);

    apr_skiplist_init(&timer_skiplist, pskip);

    apr_skiplist_set_compare(timer_skiplist, timer_comp, timer_comp);

server/mpm/mpmt_os2/mpmt_os2_child.c

+1 −0
Changes for server/mpm/mpmt_os2/mpmt_os2_child.c: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -110,6 +110,7 @@ void ap_mpm_child_main(apr_pool_t *pconf) 


    /* Create pool for child */

    apr_pool_create(&pchild, pconf);

    apr_pool_tag(pchild, "pchild");



    ap_run_child_init(pchild, ap_server_conf);

server/mpm/winnt/child.c

+3 −0
Changes for server/mpm/winnt/child.c: 3 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -905,6 +905,9 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid) 
    int i;

    int num_events;



    /* Get a sub context for global allocations in this child, so that

     * we can have cleanups occur when the child exits.

     */

    apr_pool_create(&pchild, pconf);

    apr_pool_tag(pchild, "pchild");

server/mpm/worker/worker.c

+27 −8
Changes for server/mpm/worker/worker.c: 27 added lines, 8 removed lines.
Original line number Diff line number Diff line
@@ -864,7 +864,7 @@ static int check_signal(int signum) 
    return 0;

}



static void create_listener_thread(thread_starter *ts)

static void create_listener_thread(thread_starter *ts, apr_pool_t *pool)

{

    int my_child_num = ts->child_num_arg;

    apr_threadattr_t *thread_attr = ts->threadattr;
@@ -876,7 +876,7 @@ static void create_listener_thread(thread_starter *ts) 
    my_info->tid = -1; /* listener thread doesn't have a thread slot */

    my_info->sd = 0;

    rv = apr_thread_create(&ts->listener, thread_attr, listener_thread,

                           my_info, pchild);

                           my_info, pool);

    if (rv != APR_SUCCESS) {

        ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(00275)

                     "apr_thread_create: unable to create listener thread");
@@ -897,6 +897,7 @@ static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy) 
    apr_thread_t **threads = ts->threads;

    apr_threadattr_t *thread_attr = ts->threadattr;

    int my_child_num = ts->child_num_arg;

    apr_pool_t *pruntime = NULL;

    proc_info *my_info;

    apr_status_t rv;

    int i;
@@ -905,16 +906,29 @@ static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy) 
    int loops;

    int prev_threads_created;



    /* All threads (listener, workers) and synchronization objects (queues,

     * pollset, mutexes...) created here should have at least the lifetime of

     * the connections they handle (i.e. ptrans). We can't use this thread's

     * self pool because all these objects survive it, nor use pchild or pconf

     * directly because this starter thread races with other modules' runtime,

     * nor finally pchild (or subpool thereof) because it is killed explicitely

     * before pconf (thus connections/ptrans can live longer, which matters in

     * ONE_PROCESS mode). So this leaves us with a subpool of pconf, created

     * before any ptrans hence destroyed after.

     */

    apr_pool_create(&pruntime, pconf);

    apr_pool_tag(pruntime, "mpm_runtime");



    /* We must create the fd queues before we start up the listener

     * and worker threads. */

    rv = ap_queue_create(&worker_queue, threads_per_child, pchild);

    rv = ap_queue_create(&worker_queue, threads_per_child, pruntime);

    if (rv != APR_SUCCESS) {

        ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03140)

                     "ap_queue_create() failed");

        clean_child_exit(APEXIT_CHILDFATAL);

    }



    rv = ap_queue_info_create(&worker_queue_info, pchild,

    rv = ap_queue_info_create(&worker_queue_info, pruntime,

                              threads_per_child, -1);

    if (rv != APR_SUCCESS) {

        ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03141)
@@ -922,8 +936,8 @@ static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy) 
        clean_child_exit(APEXIT_CHILDFATAL);

    }



    worker_sockets = apr_pcalloc(pchild, threads_per_child

                                        * sizeof(apr_socket_t *));

    worker_sockets = apr_pcalloc(pruntime, threads_per_child *

                                           sizeof(apr_socket_t *));



    loops = prev_threads_created = 0;

    while (1) {
@@ -947,7 +961,7 @@ static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy) 
             * done because it lets us deal with tid better.

             */

            rv = apr_thread_create(&threads[i], thread_attr,

                                   worker_thread, my_info, pchild);

                                   worker_thread, my_info, pruntime);

            if (rv != APR_SUCCESS) {

                ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03142)

                             "apr_thread_create: unable to create worker thread");
@@ -958,7 +972,7 @@ static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy) 
        }

        /* Start the listener only when there are workers available */

        if (!listener_started && threads_created) {

            create_listener_thread(ts);

            create_listener_thread(ts, pruntime);

            listener_started = 1;

        }

        if (start_thread_may_exit || threads_created == threads_per_child) {
@@ -1087,7 +1101,12 @@ static void child_main(int child_num_arg, int child_bucket) 


    ap_my_pid = getpid();

    ap_fatal_signal_child_setup(ap_server_conf);



    /* Get a sub context for global allocations in this child, so that

     * we can have cleanups occur when the child exits.

     */

    apr_pool_create(&pchild, pconf);

    apr_pool_tag(pchild, "pchild");



    /* close unused listeners and pods */

    for (i = 0; i < retained->mpm->num_buckets; i++) {