From 8a77bee4ebaba3ecceb21fa87c925c8bcc8d9545 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Michal=20Voc=C5=AF?= Date: Fri, 7 Dec 2007 21:31:54 +0000 Subject: [PATCH] reference counting scheme for gss credentials, avoids IL thread blocking others when opening connections --- org.glite.lb.logger/src/interlogd.c | 37 +++++++++++++++++++++++++++++++++--- org.glite.lb.logger/src/interlogd.h | 11 +++++++++-- org.glite.lb.logger/src/recover.c | 23 +++++++++++++--------- org.glite.lb.logger/src/send_event.c | 23 +++++++++++++++++++--- 4 files changed, 77 insertions(+), 17 deletions(-) diff --git a/org.glite.lb.logger/src/interlogd.c b/org.glite.lb.logger/src/interlogd.c index 274898e..390d6c9 100644 --- a/org.glite.lb.logger/src/interlogd.c +++ b/org.glite.lb.logger/src/interlogd.c @@ -34,8 +34,9 @@ int killflg = 0; int TIMEOUT = DEFAULT_TIMEOUT; -edg_wll_GssCred cred_handle = NULL; +cred_handle_t *cred_handle = NULL; pthread_mutex_t cred_handle_lock = PTHREAD_MUTEX_INITIALIZER; +pthread_key_t cred_handle_key; time_t key_mtime = 0, cert_mtime = 0; @@ -253,6 +254,25 @@ void handle_signal(int num) { killflg++; } + +/* this is called when thread exists */ +void cred_handle_destroy(void *handle) { + cred_handle_t *h = (cred_handle_t*)handle; + if(!h) + return; + il_log(LOG_DEBUG, "Thread exiting, releasing credentials.\n"); + if(pthread_mutex_lock(&cred_handle_lock) < 0) + abort(); + if(--(h->counter) == 0) { + edg_wll_gss_release_cred(&h->creds, NULL); + free(h); + il_log(LOG_DEBUG, "Freed credentials, not used anymore.\n"); + } + if(pthread_mutex_unlock(&cred_handle_lock) < 0) + abort(); +} + + int main (int argc, char **argv) { @@ -314,11 +334,22 @@ main (int argc, char **argv) il_log(LOG_DEBUG, " using lazy mode when closing connections, timeout %d\n", default_close_timeout); + /* initialize credential key and get credentials */ + /* IMPORTANT: no other threads may run at the time, the key initialization + has to be done exactly once */ + if(pthread_key_create(&cred_handle_key, cred_handle_destroy) < 0) + abort(); if (CAcert_dir) setenv("X509_CERT_DIR", CAcert_dir, 1); - edg_wll_gss_watch_creds(cert_file,&cert_mtime); - ret = edg_wll_gss_acquire_cred_gsi(cert_file, key_file, &cred_handle, &gss_stat); + cred_handle = malloc(sizeof(*cred_handle)); + if(cred_handle == NULL) { + il_log(LOG_CRIT, "Failed to allocate structure for credentials.\n"); + exit(EXIT_FAILURE); + } + cred_handle->creds = NULL; + cred_handle->counter = 0; + ret = edg_wll_gss_acquire_cred_gsi(cert_file, key_file, &cred_handle->creds, &gss_stat); if (ret) { char *gss_err = NULL; char *str; diff --git a/org.glite.lb.logger/src/interlogd.h b/org.glite.lb.logger/src/interlogd.h index 0af27ff..60e10c3 100644 --- a/org.glite.lb.logger/src/interlogd.h +++ b/org.glite.lb.logger/src/interlogd.h @@ -62,9 +62,13 @@ extern int TIMEOUT; #define INPUT_TIMEOUT (60) - -extern edg_wll_GssCred cred_handle; +typedef struct cred_handle { + edg_wll_GssCred creds; + int counter; +} cred_handle_t; +extern cred_handle_t *cred_handle; extern pthread_mutex_t cred_handle_lock; +extern pthread_key_t cred_handle_key; extern char *cert_file; extern char *key_file; extern char *CAcert_dir; @@ -146,6 +150,9 @@ struct event_queue { }; +/* credential destructor */ +void cred_handle_destroy(void *); + /* server msg methods */ struct server_msg *server_msg_create(il_octet_string_t *, long); struct server_msg *server_msg_copy(struct server_msg *); diff --git a/org.glite.lb.logger/src/recover.c b/org.glite.lb.logger/src/recover.c index b1b7154..928efe1 100644 --- a/org.glite.lb.logger/src/recover.c +++ b/org.glite.lb.logger/src/recover.c @@ -33,22 +33,27 @@ recover_thread(void *q) exit(1); } il_log(LOG_INFO, "Reloading certificate...\n"); - if(pthread_mutex_lock(&cred_handle_lock) < 0) - abort(); if (edg_wll_gss_watch_creds(cert_file, &cert_mtime) > 0) { - edg_wll_GssCred new_cred_handle = NULL; + edg_wll_GssCred new_creds = NULL; int ret; ret = edg_wll_gss_acquire_cred_gsi(cert_file,key_file, - &new_cred_handle, NULL); - if (new_cred_handle != NULL) { - edg_wll_gss_release_cred(&cred_handle, NULL); - cred_handle = new_cred_handle; + &new_creds, NULL); + if (new_creds != NULL) { + if(pthread_mutex_lock(&cred_handle_lock) < 0) + abort(); + cred_handle = malloc(sizeof(*cred_handle)); + if(cred_handle == NULL) { + il_log(LOG_CRIT, "Failed to allocate structure for credentials.\n"); + exit(EXIT_FAILURE); + } + cred_handle->creds = new_creds; + cred_handle->counter = 0; + if(pthread_mutex_unlock(&cred_handle_lock) < 0) + abort(); il_log(LOG_INFO, "New certificate found and deployed.\n"); } } - if(pthread_mutex_unlock(&cred_handle_lock) < 0) - abort(); sleep(INPUT_TIMEOUT); } } diff --git a/org.glite.lb.logger/src/send_event.c b/org.glite.lb.logger/src/send_event.c index f74827a..2bea9a2 100644 --- a/org.glite.lb.logger/src/send_event.c +++ b/org.glite.lb.logger/src/send_event.c @@ -167,6 +167,7 @@ event_queue_connect(struct event_queue *eq) int ret; struct timeval tv; edg_wll_GssStatus gss_stat; + cred_handle_t *local_cred_handle; assert(eq != NULL); @@ -178,12 +179,28 @@ event_queue_connect(struct event_queue *eq) tv.tv_sec = TIMEOUT; tv.tv_usec = 0; + /* get thread specific pointer to credentials */ + local_cred_handle = pthread_getspecific(cred_handle_key); + /* check if there are new credentials */ if(pthread_mutex_lock(&cred_handle_lock) < 0) abort(); - il_log(LOG_DEBUG, " trying to connect to %s:%d\n", eq->dest_name, eq->dest_port); - ret = edg_wll_gss_connect(cred_handle, eq->dest_name, eq->dest_port, &tv, &eq->gss, &gss_stat); - if(pthread_mutex_unlock(&cred_handle_lock) < 0) + if(local_cred_handle != cred_handle) { + il_log(LOG_DEBUG, " new credentials were found, discarding old\n"); + /* decrement counter in credentials, if it goes to zero, deallocate */ + if(--(local_cred_handle->counter) == 0) { + edg_wll_gss_release_cred(&local_cred_handle->creds, &gss_stat); + free(local_cred_handle); + il_log(LOG_DEBUG, " freed old credentials, not used anymore\n"); + } + /* use the new credentials, increment usage count */ + local_cred_handle = cred_handle; + local_cred_handle->counter++; + pthread_setspecific(cred_handle_key, local_cred_handle); + } + if(pthread_mutex_unlock(&cred_handle_lock) < 0) abort(); + il_log(LOG_DEBUG, " trying to connect to %s:%d\n", eq->dest_name, eq->dest_port); + ret = edg_wll_gss_connect(local_cred_handle->creds, eq->dest_name, eq->dest_port, &tv, &eq->gss, &gss_stat); if(ret < 0) { char *gss_err = NULL; -- 1.8.2.3