Commit 8c0c69c5 authored by Mladen Turk's avatar Mladen Turk
Browse files

Remove proxy_runtime_worker and make the things simpler by 

sharing runtime status accross child processes. Also make sure
that shared data is initialized after the scoreboard is created.


git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@105293 13f79535-47bb-0310-9956-ffa450edef68
parent 882f6214

modules/proxy/mod_proxy.c

+48 −28
Original line number Diff line number Diff line
@@ -189,11 +189,15 @@ static const char *set_worker_param(apr_pool_t *p, 
    else if (!strcasecmp(key, "route")) {

        /* Worker route.

         */

        if (strlen(val) > PROXY_WORKER_MAX_ROUTE_SIZ)

            return "Route length must be < 64 characters";

        worker->route = apr_pstrdup(p, val);

    }

    else if (!strcasecmp(key, "redirect")) {

        /* Worker redirection route.

         */

        if (strlen(val) > PROXY_WORKER_MAX_ROUTE_SIZ)

            return "Redirect length must be < 64 characters";

        worker->redirect = apr_pstrdup(p, val);

    }

    else {
@@ -1194,9 +1198,6 @@ static const char * 
        psf->forward->name     = "proxy:forward";

        psf->forward->hostname = "*";

        psf->forward->scheme   = "*";



        /* Do not disable worker in case of errors */

        psf->forward->status = PROXY_WORKER_IGNORE_ERRORS;

    }

    return NULL;

}
@@ -1663,24 +1664,10 @@ PROXY_DECLARE(int) ap_proxy_ssl_disable(conn_rec *c) 
static int proxy_post_config(apr_pool_t *pconf, apr_pool_t *plog,

                             apr_pool_t *ptemp, server_rec *s)

{

    proxy_server_conf *conf =

    (proxy_server_conf *) ap_get_module_config(s->module_config, &proxy_module);

    proxy_worker *worker;

    int i;



    proxy_ssl_enable = APR_RETRIEVE_OPTIONAL_FN(ssl_proxy_enable);

    proxy_ssl_disable = APR_RETRIEVE_OPTIONAL_FN(ssl_engine_disable);



    /* Initialize workers */

    worker = (proxy_worker *)conf->workers->elts;

    for (i = 0; i < conf->workers->nelts; i++) {

        ap_proxy_initialize_worker(worker, s);

        worker++;

    }

    /* Initialize forward worker if defined */

    if (conf->forward)

        ap_proxy_initialize_worker(conf->forward, s);



    return OK;

}
@@ -1713,7 +1700,7 @@ static int proxy_status_hook(request_rec *r, int flags) 
    proxy_server_conf *conf = (proxy_server_conf *)

        ap_get_module_config(sconf, &proxy_module);

    proxy_balancer *balancer = NULL;

    proxy_runtime_worker *worker = NULL;

    proxy_worker *worker = NULL;



    if (flags & AP_STATUS_SHORT || conf->balancers->nelts == 0 ||

        conf->proxy_status == status_off)
@@ -1736,26 +1723,26 @@ static int proxy_status_hook(request_rec *r, int flags) 
                 "<th>F</th><th>Acc</th><th>Wr</th><th>Rd</th>"

                 "</tr>\n", r);



        worker = (proxy_runtime_worker *)balancer->workers->elts;

        worker = (proxy_worker *)balancer->workers->elts;

        for (n = 0; n < balancer->workers->nelts; n++) {



            ap_rvputs(r, "<tr>\n<td>", worker->w->scheme, "</td>", NULL);

            ap_rvputs(r, "<td>", worker->w->hostname, "</td><td>", NULL);

            if (worker->w->status & PROXY_WORKER_DISABLED)

            ap_rvputs(r, "<tr>\n<td>", worker->scheme, "</td>", NULL);

            ap_rvputs(r, "<td>", worker->hostname, "</td><td>", NULL);

            if (worker->s->status & PROXY_WORKER_DISABLED)

                ap_rputs("Dis", r);

            else if (worker->w->status & PROXY_WORKER_IN_ERROR)

            else if (worker->s->status & PROXY_WORKER_IN_ERROR)

                ap_rputs("Err", r);

            else if (worker->w->status & PROXY_WORKER_INITIALIZED)

            else if (worker->s->status & PROXY_WORKER_INITIALIZED)

                ap_rputs("Ok", r);

            else

                ap_rputs("-", r);

            ap_rvputs(r, "</td><td>", worker->w->route, NULL);

            ap_rvputs(r, "</td><td>", worker->w->redirect, NULL);

            ap_rprintf(r, "</td><td>%.2f</td>", worker->s->lbfactor);

            ap_rvputs(r, "</td><td>", worker->s->route, NULL);

            ap_rvputs(r, "</td><td>", worker->s->redirect, NULL);

            ap_rprintf(r, "</td><td>%d</td>", worker->s->lbfactor);

            ap_rprintf(r, "<td>%d</td><td>", (int)(worker->s->elected));

            format_byte_out(r, worker->s->transfered);

            ap_rputs("</td><td>", r);

            format_byte_out(r, worker->s->transfered);

            format_byte_out(r, worker->s->readed);

            ap_rputs("</td>\n", r);



            /* TODO: Add the rest of dynamic worker data */
@@ -1783,6 +1770,31 @@ static int proxy_status_hook(request_rec *r, int flags) 
    return OK;

}



static void child_init(apr_pool_t *p, server_rec *s)

{

    void *sconf = s->module_config;

    proxy_server_conf *conf = (proxy_server_conf *)

        ap_get_module_config(sconf, &proxy_module);

    proxy_worker *worker;

    int i;

    

    /* Initialize worker's shared scoreboard data */ 

    worker = (proxy_worker *)conf->workers->elts;

    for (i = 0; i < conf->workers->nelts; i++) {

        ap_proxy_initialize_worker_share(conf, worker);

        ap_proxy_initialize_worker(worker, s);

        worker++;

    }

    /* Initialize forward worker if defined */

    if (conf->forward) {

        ap_proxy_initialize_worker_share(conf, conf->forward);

        ap_proxy_initialize_worker(conf->forward, s);

        /* Do not disable worker in case of errors */

        conf->forward->s->status |= PROXY_WORKER_IGNORE_ERRORS;

    }



}



/*

 * This routine is called before the server processes the configuration

 * files.  There is no return value.
@@ -1803,6 +1815,11 @@ static void register_hooks(apr_pool_t *p) 
#ifndef FIX_15207

    static const char * const aszSucc[]={ "mod_rewrite.c", NULL };

#endif

    /* Only the mpm_winnt has child init hook handler.

     * make sure that we are called after the mpm

     * initializes.

     */

    static const char *const aszPred[] = { "mpm_winnt.c", NULL};

    

    APR_REGISTER_OPTIONAL_FN(ap_proxy_lb_workers);

    /* handler */
@@ -1821,6 +1838,9 @@ static void register_hooks(apr_pool_t *p) 
    ap_hook_pre_config(proxy_pre_config, NULL, NULL, APR_HOOK_MIDDLE); 

    /* post config handling */

    ap_hook_post_config(proxy_post_config, NULL, NULL, APR_HOOK_MIDDLE);

    /* child init handling */

    ap_hook_child_init(child_init, aszPred, NULL, APR_HOOK_MIDDLE); 



}



module AP_MODULE_DECLARE_DATA proxy_module =

modules/proxy/mod_proxy.h

+32 −26
Original line number Diff line number Diff line
@@ -218,7 +218,6 @@ struct proxy_conn_pool { 
#if APR_HAS_THREADS

    apr_reslist_t  *res;    /* Connection resource list */

#endif

    int            nfree;   /* Balancer free count number */

    proxy_conn_rec *conn;   /* Single connection for prefork mpm's */

};
@@ -229,17 +228,30 @@ struct proxy_conn_pool { 
#define PROXY_WORKER_DISABLED       0x0020

#define PROXY_WORKER_IN_ERROR       0x0040



#define PROXY_WORKER_IS_USABLE(f)   (!((f)->status & 0x00F0))

#define PROXY_WORKER_IS_USABLE(f)   (!((f)->s->status & 0x00F0))



/* default worker retry timeout in seconds */

#define PROXY_WORKER_DEFAULT_RETRY  60

#define PROXY_WORKER_MAX_ROUTE_SIZ  63



/* Worker configuration */

struct proxy_worker {

/* Runtime worker status informations. Shared in scoreboard */

typedef struct {

    int             status;

    apr_time_t      error_time; /* time of the last error */

    apr_interval_time_t retry;  /* retry interval */

    int             retries;    /* number of retries on this worker */

    int             lbstatus;   /* Current lbstatus */

    int             lbfactor;   /* dynamic lbfactor */

    apr_off_t       transfered; /* Number of bytes transfered to remote */

    apr_off_t       readed;     /* Number of bytes readed from remote */

    apr_size_t      elected;    /* Number of times the worker was elected */

    char            route[PROXY_WORKER_MAX_ROUTE_SIZ+1];

    char            redirect[PROXY_WORKER_MAX_ROUTE_SIZ+1];

} proxy_worker_stat;



/* Worker configuration */

struct proxy_worker {

    int             id;         /* scoreboard id */

    apr_interval_time_t retry;  /* retry interval */

    int             lbfactor;   /* initial load balancing factor */

    const char      *name;

    const char      *scheme;    /* scheme to use ajp|http|https */
@@ -263,28 +275,12 @@ struct proxy_worker { 
    char                keepalive;

    char                keepalive_set;

    proxy_conn_pool     *cp;        /* Connection pool to use */

    proxy_worker_stat   *s;         /* Shared data */

    void                *opaque;    /* per scheme worker data */

};



/* Runtime worker status informations. Shared in scoreboard */

typedef struct {

    double          lbstatus;   /* Current lbstatus */

    double          lbfactor;   /* dynamic lbfactor */

    apr_size_t      transfered; /* Number of bytes transfered to remote */

    apr_size_t      readed;     /* Number of bytes readed from remote */

    apr_size_t      elected;    /* Number of times the worker was elected */

} proxy_runtime_stat;



/* Runtime worker. */

typedef struct {

    int                id;      /* scoreboard id */

    proxy_balancer     *b;      /* balancer containing this worker */

    proxy_worker       *w;

    proxy_runtime_stat *s;

} proxy_runtime_worker;



struct proxy_balancer {

    apr_array_header_t *workers; /* array of proxy_runtime_workers */

    apr_array_header_t *workers; /* array of proxy_workers */

    const char *name;            /* name of the load balancer */

    const char *sticky;          /* sticky session identifier */

    int         sticky_force;    /* Disable failover for sticky sessions */
@@ -421,9 +417,19 @@ PROXY_DECLARE(const char *) ap_proxy_add_worker(proxy_worker **worker, 
 */

PROXY_DECLARE(proxy_worker *) ap_proxy_create_worker(apr_pool_t *p);



/**

 * Initize the worker's shared data

 * @param conf   current proxy server configuration

 * @param s      current server record

 * @param worker worker to initialize

 */

PROXY_DECLARE(void) ap_proxy_initialize_worker_share(proxy_server_conf *conf,

                                                     proxy_worker *worker);





/**

 * Initize the worker

 * @param worker the new worker

 * @param worker worker to initialize

 * @param p      memory pool to allocate worker from 

 * @param s      current server record

 * @return       APR_SUCCESS or error code

modules/proxy/proxy_util.c

+44 −18
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 
/* Utility routines for Apache proxy */

#include "mod_proxy.h"

#include "ap_mpm.h"

#include "scoreboard.h"

#include "apr_version.h"



#if APR_HAVE_UNISTD_H
@@ -1038,7 +1039,7 @@ PROXY_DECLARE(const char *) ap_proxy_add_balancer(proxy_balancer **balancer, 
    memset(*balancer, 0, sizeof(proxy_balancer));



    (*balancer)->name = uri;

    (*balancer)->workers = apr_array_make(p, 5, sizeof(proxy_runtime_worker));

    (*balancer)->workers = apr_array_make(p, 5, sizeof(proxy_worker));

    /* XXX Is this a right place to create mutex */

#if APR_HAS_THREADS

    if (apr_thread_mutex_create(&((*balancer)->mutex),
@@ -1141,7 +1142,9 @@ PROXY_DECLARE(const char *) ap_proxy_add_worker(proxy_worker **worker, 
    if (port == -1)

        port = apr_uri_port_of_scheme((*worker)->scheme);

    (*worker)->port = port;



    (*worker)->id   = lb_workers;

    /* Increase the total worker count */

    ++lb_workers;

    init_conn_pool(p, *worker);



    return NULL;
@@ -1152,15 +1155,19 @@ PROXY_DECLARE(proxy_worker *) ap_proxy_create_worker(apr_pool_t *p) 


    proxy_worker *worker;

    worker = (proxy_worker *)apr_pcalloc(p, sizeof(proxy_worker));

    worker->id = lb_workers;

    /* Increase the total worker count */

    ++lb_workers;

    init_conn_pool(p, worker);



    return worker;

}



PROXY_DECLARE(void) 

ap_proxy_add_worker_to_balancer(apr_pool_t *pool, proxy_balancer *balancer, proxy_worker *worker)

ap_proxy_add_worker_to_balancer(apr_pool_t *pool, proxy_balancer *balancer,

                                proxy_worker *worker)

{

    proxy_runtime_worker *runtime;

    proxy_worker *runtime;



#if PROXY_HAS_SCOREBOARD

    int mpm_daemons;
@@ -1179,10 +1186,8 @@ ap_proxy_add_worker_to_balancer(apr_pool_t *pool, proxy_balancer *balancer, prox 
    }

#endif

    runtime = apr_array_push(balancer->workers);

    runtime->w  = worker;

    runtime->b  = balancer;

    memcpy(runtime, worker, sizeof(proxy_worker));

    runtime->id = lb_workers;

    runtime->s  = NULL;

    /* Increase the total runtime count */

    ++lb_workers;
@@ -1374,6 +1379,28 @@ static apr_status_t connection_destructor(void *resource, void *params, 
    return APR_SUCCESS;

}



PROXY_DECLARE(void) ap_proxy_initialize_worker_share(proxy_server_conf *conf,

                                                     proxy_worker *worker)

{

#if PROXY_HAS_SCOREBOARD

    lb_score *score = NULL;

#else

    void *score = NULL;

#endif

#if PROXY_HAS_SCOREBOARD

        /* Get scoreboard slot */

    if (ap_scoreboard_image)

        score = ap_get_scoreboard_lb(worker->id);

#endif

    if (!score)

        score = apr_pcalloc(conf->pool, sizeof(proxy_worker_stat));

    worker->s = (proxy_worker_stat *)score;

    if (worker->route)

        strcpy(worker->s->route, worker->route);

    if (worker->redirect)

        strcpy(worker->s->redirect, worker->redirect);

}



PROXY_DECLARE(apr_status_t) ap_proxy_initialize_worker(proxy_worker *worker, server_rec *s)

{

    apr_status_t rv;
@@ -1391,7 +1418,6 @@ PROXY_DECLARE(apr_status_t) ap_proxy_initialize_worker(proxy_worker *worker, ser 
        /* Set min to be lower then smax */

        if (worker->min > worker->smax)

            worker->min = worker->smax; 

        worker->cp->nfree = worker->hmax;

    }

    else {

        /* This will supress the apr_reslist creation */
@@ -1428,7 +1454,7 @@ PROXY_DECLARE(apr_status_t) ap_proxy_initialize_worker(proxy_worker *worker, ser 
                      worker->hostname);

    }

    if (rv == APR_SUCCESS)

        worker->status |= PROXY_WORKER_INITIALIZED;

        worker->s->status |= PROXY_WORKER_INITIALIZED;

    /* Set default parameters */

    if (!worker->retry)

        worker->retry = apr_time_from_sec(PROXY_WORKER_DEFAULT_RETRY);
@@ -1439,13 +1465,13 @@ PROXY_DECLARE(int) ap_proxy_retry_worker(const char *proxy_function, 
                                         proxy_worker *worker,

                                         server_rec *s)

{

    if (worker->status & PROXY_WORKER_IN_ERROR) {

    if (worker->s->status & PROXY_WORKER_IN_ERROR) {

        ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s,

                    "proxy: %s: retrying the worker for (%s)",

                     proxy_function, worker->hostname);

        if (apr_time_now() > worker->error_time + worker->retry) {

            ++worker->retries;

            worker->status &= ~PROXY_WORKER_IN_ERROR;

        if (apr_time_now() > worker->s->error_time + worker->retry) {

            ++worker->s->retries;

            worker->s->status &= ~PROXY_WORKER_IN_ERROR;

            ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s,

                         "proxy: %s: worker for (%s) has been marked for retry",

                         proxy_function, worker->hostname);
@@ -1739,16 +1765,16 @@ PROXY_DECLARE(int) ap_proxy_connect_backend(const char *proxy_function, 
     * no further connections to the worker could be made

     */

    if (!connected && PROXY_WORKER_IS_USABLE(worker) &&

        !(worker->status & PROXY_WORKER_IGNORE_ERRORS)) {

        worker->status |= PROXY_WORKER_IN_ERROR;

        worker->error_time = apr_time_now();

        !(worker->s->status & PROXY_WORKER_IGNORE_ERRORS)) {

        worker->s->status |= PROXY_WORKER_IN_ERROR;

        worker->s->error_time = apr_time_now();

        ap_log_error(APLOG_MARK, APLOG_ERR, 0, s,

            "ap_proxy_connect_backend disabling worker for (%s)",

            worker->hostname);

    }

    else {

        worker->error_time = 0;

        worker->retries = 0;

        worker->s->error_time = 0;

        worker->s->retries = 0;

    }

    return connected ? OK : DECLINED;

}