Commit a61a4bd0 authored by Jim Jagielski's avatar Jim Jagielski
Browse files

Merge r1705922, r1706523, r1738464, r1738466, r1738486 from trunk: 

When shutting down a process, free resources early

Due to lingering connections, shutting down a process may take a very
long time. Free all recycled pools early in the hope that we can already
give some memory back to the OS.


rename some variables to be more descriptive

pid -> pslot
tid -> tslot
remove unused 'sd'


Terminate keep-alive connections when dying

When shutting down a process gracefully, terminate keep-alive connections so
that we don't get any new requests which may keep the dying process alive
longer.



Exit threads early during shutdown

During graceful shutdown, if there are more running worker threads than open
connections, terminate some threads. This frees resources faster, which may be
needed for new processes.



Exit threads early during shutdown, part 2

Follow up to r1738466: During graceful shutdown, when the listener thread is
closing a connection, it needs to wake up a worker thread so that it may
terminate.


Submitted by: sf
Reviewed/backported by: jim


git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/branches/2.4.x@1772334 13f79535-47bb-0310-9956-ffa450edef68
parent a489a662

CHANGES

+3 −0
Original line number Diff line number Diff line
@@ -15,6 +15,9 @@ Changes with Apache 2.4.24 
  *) mod_ssl: Fix quick renegotiation (OptRenegotiaton) with no intermediate

     in the client certificate chain.  PR 55786.  [Yann Ylavic]



  *) mpm_event: Free memory earlier when shutting down processes.

     [Stefan Fritsch]



  *) mod_status: Display the process slot number in the async connection

     overview. [Stefan Fritsch]

STATUS

+0 −19
Original line number Diff line number Diff line
@@ -117,25 +117,6 @@ RELEASE SHOWSTOPPERS: 
PATCHES ACCEPTED TO BACKPORT FROM TRUNK:

  [ start all new proposals below, under PATCHES PROPOSED. ]



  *) Improve mod_status view of async connections

     trunk patches:

       https://svn.apache.org/r1738628

       https://svn.apache.org/r1757009

       https://svn.apache.org/r1756848

       https://svn.apache.org/r1757029

     2.4.x patch: https://people.apache.org/~sf/PR53555_2_mod_status.diff

     +1: sf, jim, wrowe



  *) mpm_event: Free resources earlier during shutdown

     trunk patches:

       https://svn.apache.org/r1705922

       https://svn.apache.org/r1706523

       https://svn.apache.org/r1738464

       https://svn.apache.org/r1738466

       https://svn.apache.org/r1738486

     2.4.x patch: https://people.apache.org/~sf/PR53555_3_free_early.diff

     +1: sf, jim, wrowe



  *) mpm_event: Use all free scoreboard slots up to ServerLimit, but don't

     re-use scoreboard slots of still running, gracefully finishing processes.

     PR: 53555

server/mpm/event/event.c

+63 −17
Original line number Diff line number Diff line
@@ -181,6 +181,8 @@ static apr_uint32_t connection_count = 0; /* Number of open connections */ 
static apr_uint32_t lingering_count = 0;    /* Number of connections in lingering close */

static apr_uint32_t suspended_count = 0;    /* Number of suspended connections */

static apr_uint32_t clogged_count = 0;      /* Number of threads processing ssl conns */

static apr_uint32_t threads_shutdown = 0;   /* Number of threads that have shutdown

                                               early during graceful termination */

static int resource_shortage = 0;

static fd_queue_t *worker_queue;

static fd_queue_info_t *worker_queue_info;
@@ -288,9 +290,8 @@ static apr_pollset_t *event_pollset; 
/* The structure used to pass unique initialization info to each thread */

typedef struct

{

    int pid;

    int tid;

    int sd;

    int pslot;  /* process slot */

    int tslot;  /* worker slot of the thread */

} proc_info;



/* Structure used to pass information to the thread responsible for
@@ -911,6 +912,8 @@ static int start_lingering_close_nonblocking(event_conn_state_t *cs) 
        || apr_socket_shutdown(csd, APR_SHUTDOWN_WRITE) != APR_SUCCESS) {

        apr_socket_close(csd);

        ap_push_pool(worker_queue_info, cs->p);

        if (dying)

            ap_queue_interrupt_one(worker_queue);

        return 0;

    }

    return start_lingering_close_common(cs, 0);
@@ -934,6 +937,8 @@ static int stop_lingering_close(event_conn_state_t *cs) 
        AP_DEBUG_ASSERT(0);

    }

    ap_push_pool(worker_queue_info, cs->p);

    if (dying)

        ap_queue_interrupt_one(worker_queue);

    return 0;

}
@@ -1219,6 +1224,9 @@ static void close_listeners(int process_slot, int *closed) 
        }

        /* wake up the main thread */

        kill(ap_my_pid, SIGTERM);



        ap_free_idle_pools(worker_queue_info);

        ap_queue_interrupt_all(worker_queue);

    }

}
@@ -1439,6 +1447,8 @@ static void process_lingering_close(event_conn_state_t *cs, const apr_pollfd_t * 
    TO_QUEUE_ELEM_INIT(cs);



    ap_push_pool(worker_queue_info, cs->p);

    if (dying)

        ap_queue_interrupt_one(worker_queue);

}



/* call 'func' for all elements of 'q' with timeout less than 'timeout_time'.
@@ -1518,7 +1528,7 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) 
    timer_event_t *te;

    apr_status_t rc;

    proc_info *ti = dummy;

    int process_slot = ti->pid;

    int process_slot = ti->pslot;

    apr_pool_t *tpool = apr_thread_pool_get(thd);

    void *csd = NULL;

    apr_pool_t *ptrans;         /* Pool for per-transaction stuff */
@@ -1584,6 +1594,12 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) 
                             *keepalive_q->total,

                             apr_atomic_read32(&lingering_count),

                             apr_atomic_read32(&suspended_count));

                if (dying) {

                    ap_log_error(APLOG_MARK, APLOG_TRACE6, 0, ap_server_conf,

                                 "%u/%u workers shutdown",

                                 apr_atomic_read32(&threads_shutdown),

                                 threads_per_child);

                }

                apr_thread_mutex_unlock(timeout_mutex);

            }

        }
@@ -1818,7 +1834,8 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) 
            /* If all workers are busy, we kill older keep-alive connections so that they

             * may connect to another process.

             */

            if (workers_were_busy && *keepalive_q->total) {

            if ((workers_were_busy || dying) && *keepalive_q->total) {

                if (!dying)

                    ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, ap_server_conf,

                                 "All workers are busy, will close %d keep-alive "

                                 "connections",
@@ -1869,6 +1886,34 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) 
    return NULL;

}



/*

 * During graceful shutdown, if there are more running worker threads than

 * open connections, exit one worker thread.

 *

 * return 1 if thread should exit, 0 if it should continue running.

 */

static int worker_thread_should_exit_early(void)

{

    for (;;) {

        apr_uint32_t conns = apr_atomic_read32(&connection_count);

        apr_uint32_t dead = apr_atomic_read32(&threads_shutdown);

        apr_uint32_t newdead;



        AP_DEBUG_ASSERT(dead <= threads_per_child);

        if (conns >= threads_per_child - dead)

            return 0;



        newdead = dead + 1;

        if (apr_atomic_cas32(&threads_shutdown, newdead, dead) == dead) {

            /*

             * No other thread has exited in the mean time, safe to exit

             * this one.

             */

            return 1;

        }

    }

}



/* XXX For ungraceful termination/restart, we definitely don't want to

 *     wait for active connections to finish but we may want to wait

 *     for idle workers to get out of the queue code and release mutexes,
@@ -1879,8 +1924,8 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) 
static void *APR_THREAD_FUNC worker_thread(apr_thread_t * thd, void *dummy)

{

    proc_info *ti = dummy;

    int process_slot = ti->pid;

    int thread_slot = ti->tid;

    int process_slot = ti->pslot;

    int thread_slot = ti->tslot;

    apr_socket_t *csd = NULL;

    event_conn_state_t *cs;

    apr_pool_t *ptrans;         /* Pool for per-transaction stuff */
@@ -1916,6 +1961,9 @@ static void *APR_THREAD_FUNC worker_thread(apr_thread_t * thd, void *dummy) 
        if (workers_may_exit) {

            break;

        }

        if (dying && worker_thread_should_exit_early()) {

            break;

        }



        te = NULL;

        rv = ap_queue_pop_something(worker_queue, &csd, &cs, &ptrans, &te);
@@ -1993,9 +2041,8 @@ static void create_listener_thread(thread_starter * ts) 
    apr_status_t rv;



    my_info = (proc_info *) ap_malloc(sizeof(proc_info));

    my_info->pid = my_child_num;

    my_info->tid = -1;          /* listener thread doesn't have a thread slot */

    my_info->sd = 0;

    my_info->pslot = my_child_num;

    my_info->tslot = -1;      /* listener thread doesn't have a thread slot */

    rv = apr_thread_create(&ts->listener, thread_attr, listener_thread,

                           my_info, pchild);

    if (rv != APR_SUCCESS) {
@@ -2108,9 +2155,8 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) 
            }



            my_info = (proc_info *) ap_malloc(sizeof(proc_info));

            my_info->pid = my_child_num;

            my_info->tid = i;

            my_info->sd = 0;

            my_info->pslot = my_child_num;

            my_info->tslot = i;



            /* We are creating threads right now */

            ap_update_child_status_from_indexes(my_child_num, i,

server/mpm/event/fdqueue.c

+28 −2
Original line number Diff line number Diff line
@@ -280,6 +280,19 @@ void ap_pop_pool(apr_pool_t ** recycled_pool, fd_queue_info_t * queue_info) 
    }

}



void ap_free_idle_pools(fd_queue_info_t *queue_info)

{

    apr_pool_t *p;



    queue_info->max_recycled_pools = 0;

    do {

        ap_pop_pool(&p, queue_info);

        if (p != NULL)

            apr_pool_destroy(p);

    } while (p != NULL);

}





apr_status_t ap_queue_info_term(fd_queue_info_t * queue_info)

{

    apr_status_t rv;
@@ -477,17 +490,30 @@ apr_status_t ap_queue_pop_something(fd_queue_t * queue, apr_socket_t ** sd, 
    return rv;

}



apr_status_t ap_queue_interrupt_all(fd_queue_t * queue)

static apr_status_t queue_interrupt(fd_queue_t * queue, int all)

{

    apr_status_t rv;



    if ((rv = apr_thread_mutex_lock(queue->one_big_mutex)) != APR_SUCCESS) {

        return rv;

    }

    if (all)

        apr_thread_cond_broadcast(queue->not_empty);

    else

        apr_thread_cond_signal(queue->not_empty);

    return apr_thread_mutex_unlock(queue->one_big_mutex);

}



apr_status_t ap_queue_interrupt_all(fd_queue_t * queue)

{

    return queue_interrupt(queue, 1);

}



apr_status_t ap_queue_interrupt_one(fd_queue_t * queue)

{

    return queue_interrupt(queue, 0);

}



apr_status_t ap_queue_term(fd_queue_t * queue)

{

    apr_status_t rv;

server/mpm/event/fdqueue.h

+2 −0
Original line number Diff line number Diff line
@@ -52,6 +52,7 @@ apr_status_t ap_queue_info_wait_for_idler(fd_queue_info_t * queue_info, 
                                          int *had_to_block);

apr_status_t ap_queue_info_term(fd_queue_info_t * queue_info);

apr_uint32_t ap_queue_info_get_idlers(fd_queue_info_t * queue_info);

void ap_free_idle_pools(fd_queue_info_t *queue_info);



struct fd_queue_elem_t

{
@@ -98,6 +99,7 @@ apr_status_t ap_queue_pop_something(fd_queue_t * queue, apr_socket_t ** sd, 
                                    event_conn_state_t ** ecs, apr_pool_t ** p,

                                    timer_event_t ** te);

apr_status_t ap_queue_interrupt_all(fd_queue_t * queue);

apr_status_t ap_queue_interrupt_one(fd_queue_t * queue);

apr_status_t ap_queue_term(fd_queue_t * queue);



#endif /* FDQUEUE_H */