From 0259f1bca5c7216c171b3fd2019d574e3bf64ac7 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Milo=C5=A1=20Mula=C4=8D?= Date: Wed, 9 Jan 2008 15:36:19 +0000 Subject: [PATCH] merge from branch_RC31_3 (merge_313_3_src - merge_313_2_src) --- org.glite.lb.client/Makefile | 20 +- org.glite.lb.client/doc/glite-lb-dump.8 | 62 +++++ org.glite.lb.client/doc/glite-lb-load.8 | 50 ++++ org.glite.lb.client/doc/glite-lb-logevent.1 | 28 +- org.glite.lb.client/doc/glite-lb-purge.8 | 91 +++++++ org.glite.lb.client/examples/multiple_user_jobs.c | 128 +++++++++ org.glite.lb.client/interface/JobStatus.h.T | 5 +- org.glite.lb.client/project/ChangeLog | 15 ++ org.glite.lb.client/project/version.properties | 2 +- org.glite.lb.client/src/connection.c | 5 +- org.glite.lb.client/src/export.sh | 4 +- org.glite.lb.client/src/prod_proto.c | 4 +- org.glite.lb.common/interface/Event.h.T | 7 +- org.glite.lb.common/interface/context-int.h | 2 + org.glite.lb.common/project/ChangeLog | 9 + org.glite.lb.common/src/EventAttrNames.pl | 1 + org.glite.lb.common/src/context.c | 3 + org.glite.lb.common/src/xml_conversions.c | 17 +- org.glite.lb.common/src/xml_parse.c.T | 42 +-- org.glite.lb.logger/Makefile | 17 +- org.glite.lb.logger/doc/glite-lb-interlogd.8 | 175 ++++++++++++ org.glite.lb.logger/doc/glite-lb-logd.8 | 162 +++++++++++ org.glite.lb.logger/project/ChangeLog | 10 + org.glite.lb.logger/project/version.properties | 2 +- org.glite.lb.logger/src/event_queue.c | 14 +- org.glite.lb.logger/src/il_master.c | 30 ++- org.glite.lb.logger/src/interlogd.h | 7 +- org.glite.lb.logger/src/queue_mgr.c | 60 +++-- org.glite.lb.logger/src/queue_thread.c | 114 +++++--- org.glite.lb.logger/src/server_msg.c | 14 +- org.glite.lb.server/Makefile | 16 +- org.glite.lb.server/config/startup | 14 +- org.glite.lb.server/doc/glite-lb-bkindex.8 | 171 ++++++++++++ org.glite.lb.server/doc/glite-lb-bkserverd.8 | 300 +++++++++++++++++++++ org.glite.lb.server/project/ChangeLog | 20 ++ org.glite.lb.server/project/version.properties | 2 +- org.glite.lb.server/src/bkserverd.c | 1 - org.glite.lb.server/src/il_lbproxy.h | 2 +- org.glite.lb.server/src/il_notification.c | 34 ++- org.glite.lb.server/src/il_notification.h | 7 +- org.glite.lb.server/src/jobstat.c | 4 + org.glite.lb.server/src/jobstat.h | 3 + org.glite.lb.server/src/jobstat_supp.c | 3 + org.glite.lb.server/src/lb_xml_parse.c.T | 27 +- org.glite.lb.server/src/notif_match.c | 26 +- org.glite.lb.server/src/notification.c | 16 +- org.glite.lb.server/src/process_event.c | 20 +- org.glite.lb.server/src/query.c | 1 + org.glite.lb.server/src/srv_purge.c | 11 +- org.glite.lb.server/src/store.c.T | 11 +- org.glite.lb.server/test/test_query_events.cpp | 4 +- org.glite.lb.types/events.T | 1 + org.glite.lb.ws-interface/project/ChangeLog | 2 + .../project/version.properties | 7 +- org.glite.lb/project/dependencies.properties | 20 -- org.glite.lb/project/version.properties | 5 +- org.glite.security.gsoap-plugin/Makefile | 20 +- .../project/libtoolhack/gcc | 1 + .../project/version.properties | 5 +- 59 files changed, 1621 insertions(+), 233 deletions(-) create mode 100644 org.glite.lb.client/doc/glite-lb-dump.8 create mode 100644 org.glite.lb.client/doc/glite-lb-load.8 create mode 100644 org.glite.lb.client/doc/glite-lb-purge.8 create mode 100644 org.glite.lb.client/examples/multiple_user_jobs.c create mode 100644 org.glite.lb.client/project/ChangeLog create mode 100644 org.glite.lb.common/project/ChangeLog create mode 100644 org.glite.lb.logger/doc/glite-lb-interlogd.8 create mode 100644 org.glite.lb.logger/doc/glite-lb-logd.8 create mode 100644 org.glite.lb.logger/project/ChangeLog create mode 100644 org.glite.lb.server/doc/glite-lb-bkindex.8 create mode 100644 org.glite.lb.server/doc/glite-lb-bkserverd.8 create mode 100644 org.glite.lb.server/project/ChangeLog create mode 100644 org.glite.lb.ws-interface/project/ChangeLog delete mode 100644 org.glite.lb/project/dependencies.properties create mode 100644 org.glite.security.gsoap-plugin/project/libtoolhack/gcc diff --git a/org.glite.lb.client/Makefile b/org.glite.lb.client/Makefile index 6df58e8..628e29e 100644 --- a/org.glite.lb.client/Makefile +++ b/org.glite.lb.client/Makefile @@ -153,7 +153,7 @@ EXAMPLES:=${EXAMPLES_SRC:.c=} # EXAMPLES_PLUS_SRC:=indexed_attrs_plus.cpp job_status_plus.cpp query_events_plus.cpp listener.cpp user_jobs_plus.cpp job_log_plus.cpp notify_plus.cpp # EXAMPLES_PLUS:=${EXAMPLES_PLUS_SRC:.cpp=} -EXAMPLES_CL_SRC:=user_jobs.c job_status.c +EXAMPLES_CL_SRC:=user_jobs.c job_status.c multiple_user_jobs.c EXAMPLES_CL:=${EXAMPLES_CL_SRC:.c=} EXAMPLES_CL_THR_SRC:=user_jobs_threaded.c @@ -162,7 +162,9 @@ EXAMPLES_CL_THR:=${EXAMPLES_CL_THR_SRC:.c=} FAKE_EXAMPLES:=job_log_fake MAN_GZ:=glite-lb-logevent.1.gz +MAN8_GZ:=glite-lb-dump.8.gz glite-lb-load.8.gz glite-lb-purge.8.gz MAN = $(MAN_GZ:.gz=) +MAN8 = $(MAN8_GZ:.gz=) PLUS_EXTRA_LIB:=-lglite_jobid @@ -176,9 +178,9 @@ version_info:=-version-info ${shell \ perl -e '$$,=":"; @F=split "\\.","${version}"; print $$F[0]+$$F[1]+${offset},$$F[2],$$F[1]' } ifdef LB_STANDALONE -compile all: generate ${LIB} ${THRLIB} ${TOOLS} logevent examples ${MAN_GZ} +compile all: generate ${LIB} ${THRLIB} ${TOOLS} logevent examples ${MAN_GZ} ${MAN8_GZ} else -compile all: check_version generate ${LIB} ${THRLIB} ${PLUSLIB} ${THRPLUSLIB} ${TOOLS} logevent examples ${MAN_GZ} +compile all: check_version generate ${LIB} ${THRLIB} ${PLUSLIB} ${THRPLUSLIB} ${TOOLS} logevent examples ${MAN_GZ} ${MAN8_GZ} endif generate: ${GEN_HDRS} @@ -256,9 +258,13 @@ logevent_fake.o: logevent.c ${COMPILE} -c $< -o $@ ${MAN_GZ}: ${MAN} + rm -f ${MAN_GZ} ${MAN} cp $? . - gzip $(notdir $?) + gzip -f $(notdir $?) +${MAN8_GZ}: ${MAN8} + cp $? . + gzip $(notdir $?) perftest_logjobs.o: perftest_logjobs.c ${CC} ${CFLAGS} -DLB_PERF_DROP -c $< @@ -306,7 +312,7 @@ producer_test: producer_test.o prod_proto_test.o producer_test.o: %.o: %.cpp ${CXX} -c ${CXXFLAGS} ${TEST_INC} $< -man: ${MAN_GZ} +man: ${MAN_GZ} ${MAN8_GZ} stage: compile ${FAKELIB} ${FAKETHRLIB} $(MAKE) install PREFIX=${stagedir} @@ -329,6 +335,7 @@ install: doc mkdir -p ${PREFIX}/share/doc/${package}-${version}/examples mkdir -p ${PREFIX}/share/man/man1 mkdir -p ${PREFIX}/examples + mkdir -p ${PREFIX}/share/man/man8 ifdef LB_STANDALONE ${INSTALL} -m 644 ${LIB} ${THRLIB} ${PREFIX}/lib else @@ -360,10 +367,11 @@ endif ${INSTALL} -m 644 ${top_srcdir}/examples/README.examples "${PREFIX}/share/doc/${package}-${version}/examples/" ${INSTALL} -m 755 ${top_srcdir}/src/export.sh "${PREFIX}/sbin/glite-lb-export.sh" ${INSTALL} -m 644 ${MAN_GZ} ${PREFIX}/share/man/man1 + ${INSTALL} -m 644 ${MAN8_GZ} ${PREFIX}/share/man/man8 clean: rm -rvf *.o *.lo .libs lib* *.c *.cpp *.h *.dox producer_test C/ CPP/ - rm -rvf ${LIB} ${THRLIB} ${TOOLS} logevent ${PLUSLIB} ${THRPLUSLIB} ${MAN_GZ} + rm -rvf ${LIB} ${THRLIB} ${TOOLS} logevent ${PLUSLIB} ${THRPLUSLIB} ${MAN_GZ} ${MAN8_GZ} rm -rvf ${EXAMPLES} ${EXAMPLES_CL} ${EXAMPLES_CL_THR} ${sh_PROGS} rm -vf ${globalprefix} ${lbprefix} rm -rvf log.xml project/ rpmbuild/ RPMS/ tgz/ diff --git a/org.glite.lb.client/doc/glite-lb-dump.8 b/org.glite.lb.client/doc/glite-lb-dump.8 new file mode 100644 index 0000000..a963698 --- /dev/null +++ b/org.glite.lb.client/doc/glite-lb-dump.8 @@ -0,0 +1,62 @@ +.TH EDG-WL-BKDUMP 8 "May 2003" "DataGrid Project" "Logging&Bookkeeping" + +.SH NAME +glite-lb-dump - utility for dumping job events from L&B database + +.SH SYNOPSIS +.B glite-lb-dump +.RI [ options ] +.br + +.SH DESCRIPTION +.B glite-lb-dump +is utility for dumping job events from L&B database. It is primarilly used for statistical purposes. By default, it dumps events from the time when the last dum ended upto now. + +.B glite-lb-dump +may be run periodically, however the periodic dumps are done by server itself, so check the configuration of the server first. + +.SH OPTIONS +.TP +.BI \-m \fR,\fP --server mkserver +L&B server machine name. If not specified, the environment variable EDG_WL_QUERY_SERVER is beeing checked. + +.TP +.BI \-f \fR,\fP --from YYYYMMDDHHmmss +Beginning of the time interval for events to be dumped. + +.TP +.BI \-t \fR,\fP --to YYYYMMDDHHmmss +End of the time interval for events to be dumped. + +.TP +.BI \-h \fR,\fP --help +Display help. + +.TP +.BI \-v \fR,\fP --version +Display version. + +.TP +.B "-d\fR,\fP --debug" +Print diagnostics on the +.I stderr\fR.\fP + +.\".SH USAGE +.\" Add any additional description here + +.PP + +.SH FILES +No configuration files needed. + +.SH ENVIRONMENT +No environment variables needed. + +.SH REPORTING BUGS +Please, report all bugs to DataGrid Bug Tracking System located at http://marianne.in2p3.fr/datagrid/bugzilla + +.SH SEE ALSO +.B glite-lb-bkindex\fR(8),\fP glite-lb-bkserverd\fR(8),\fP glite-lb-interlogd\fR(8),\fP glite-lb-logd\fR(8),\fP glite-lb-logevent\fR(1),\fP + +.SH AUTHOR +DataGrid Work Package 1, CESNET group. diff --git a/org.glite.lb.client/doc/glite-lb-load.8 b/org.glite.lb.client/doc/glite-lb-load.8 new file mode 100644 index 0000000..7b7bd62 --- /dev/null +++ b/org.glite.lb.client/doc/glite-lb-load.8 @@ -0,0 +1,50 @@ +.TH EDG-WL-BKLOAD 8 "May 2003" "DataGrid Project" "Logging&Bookkeeping" + +.SH NAME +glite-lb-load - utility for loading job events into the L&B database + +.SH SYNOPSIS +.B glite-lb-load +.RI [ options ] +.br + +.SH DESCRIPTION +.B glite-lb-load +is utility for loading job events into the L&B database. It is primarilly used for recovery purposes. By default, it loads events from the last dump. + +.SH OPTIONS +.IP "\-m, --server \fIbkserver\fR" +L&B server machine name. If not specified, the environment variable EDG_WL_QUERY_SERVER is beeing checked. + +.IP "\-f, --file \fIfilename\fR" +File with dumped data to be loaded. + +.IP "\-h, --help" +Display help. + +.IP "\-v, --version" +Display version. + +.IP "\-d, --debug" +Print diagnostics on the +.I stderr\fR.\fP + +.\".SH USAGE +.\" Add any additional description here + +.PP + +.SH FILES +No configuration files needed. + +.SH ENVIRONMENT +No environment variables needed. + +.SH REPORTING BUGS +Please, report all bugs to DataGrid Bug Tracking System located at http://marianne.in2p3.fr/datagrid/bugzilla + +.SH SEE ALSO +.B glite-lb-bkindex\fR(8),\fP glite-lb-bkserverd\fR(8),\fP glite-lb-dump\fR(8),\fP glite-lb-purge\fR(8),\fP + +.SH AUTHOR +DataGrid Work Package 1, CESNET group. diff --git a/org.glite.lb.client/doc/glite-lb-logevent.1 b/org.glite.lb.client/doc/glite-lb-logevent.1 index 4a79400..43394d0 100644 --- a/org.glite.lb.client/doc/glite-lb-logevent.1 +++ b/org.glite.lb.client/doc/glite-lb-logevent.1 @@ -1,10 +1,10 @@ -.TH GLITE-LB-LOGEV 1 "May 2003" "EU EGEE Project" "Logging&Bookkeeping" +.TH EDG-WL-LOGEV 1 "May 2003" "EU DataGrid Project" "Logging&Bookkeeping" .SH NAME -glite-lb-logev - program for logging events to L&B subsystem +glite-lb-logevent - program for logging events to L&B subsystem .SH SYNOPSIS -.B glite-lb-logev +.B glite-lb-logevent .B -s Application -e UserTag .B -j .I JOBID @@ -17,10 +17,10 @@ glite-lb-logev - program for logging events to L&B subsystem .br .SH DESCRIPTION -.B glite-lb-logev +.B glite-lb-logevent is a low-level program for logging events to the L&B service. -It is used internally by WMS components. -At the user level it is supposed to be used only for logging user tag events and changing job ACL's in L&B enventually. +It is used internally by WMS components and has many more options, that are not recomend to be used at the user level. +At the user level it is supposed to be used only for logging user tag events. .SH USAGE Besides @@ -37,18 +37,18 @@ The user application is always executed from within a .I JobWrapper script. The wrapper sets the appropriate JobId in the environment variable EDG_WL_JOBID. The user should pass this value to the -j option of -.B edg-wl-logev. +.B glite-lb-logevent. Similarly, the wrapper sets an initial value of the event sequence code in the environment variable EDG_WL_SEQUENCE_CODE. If the user application calls -.B edg-wl-logev +.B glite-lb-logevent just once, it is sufficient to pass this value to the -c option. However, if there are more subsequent calls, the user is responsible for capturing an updated sequence code from the .B stdout of -.B edg-wl-logev +.B glite-lb-logevent and using it in subsequent calls. The L&B design requires the sequence codes in order to be able to sort events correctly while not relying on strictly synchronized clocks. @@ -67,7 +67,7 @@ as a part of the job status returned by L&B. for p in `seq 1 100`; do # log the UserTag event - EDG_WL_SEQUENCE_CODE=`edg-wl-logev -s Application + EDG_WL_SEQUENCE_CODE=`glite-lb-logevent -s Application -e UserTag -j $EDG_WL_JOBID -c $EDG_WL_SEQUENCE_CODE --name=phase --value=$p` @@ -80,8 +80,8 @@ done .SH OTHER OPTIONS -See command line help (edg-wl-logev -h) for extensive help on all options. -However, GLITE users should follow the usage shown in SYNOPSIS strictly. +See command line help (glite-lb-logevent -h) for extensive help on all options. +However, EDG users should should follow the usage shown in SYNOPSIS strictly. Logging other events may confuse the L&B service and cause wrong job status information to be reported. @@ -89,10 +89,10 @@ information to be reported. No configuration files needed. .SH REPORTING BUGS -Please, report all bugs to EU EGEE Bug Tracking System located at https://savannah.cern.ch/bugs/?func=additem&group=jra1mdw +Please, report all bugs to EU DataGrid Bug Tracking System located at http://marianne.in2p3.fr/datagrid/bugzilla .SH SEE ALSO -.B edg-wl-bkserver\fR(8),\fP edg-wl-interlogd\fR(8),\fP edg-wl-logd\fR(8) +.B glite-lb-bkserverd\fR(8),\fP glite-lb-interlogd\fR(8),\fP glite-lb-logd\fR(8) L&B API Reference, DataGrid-01-TED-0139 diff --git a/org.glite.lb.client/doc/glite-lb-purge.8 b/org.glite.lb.client/doc/glite-lb-purge.8 new file mode 100644 index 0000000..7069af4 --- /dev/null +++ b/org.glite.lb.client/doc/glite-lb-purge.8 @@ -0,0 +1,91 @@ +.TH EDG-WL-BKPURGE 8 "May 2003" "DataGrid Project" "Logging&Bookkeeping" + + +.SH NAME +edg-wl-bkpurge - utility for removing inactive jobs from the L&B database + + +.SH SYNOPSIS +.B glite-lb-purge +.RI [ options ] +.br + + +.SH DESCRIPTION +.B glite-lb-purge +is utility for removing inactive jobs from the L&B database. By default, it removes jobs that reached CLEARED, ABORTED and CANCELLED state and also OTHER jobs that are in another state and are inactive for specific (rather long) time. It is a command line frontend to server internal purge. It is used by administrators of bkserver and it is not supposed to be used at the user level. + +.B glite-lb-purge +may be run periodically in order to avoid database overflow, however the periodic purges are done by server itself, so check the configuration of the server first. + + +.SH OPTIONS +.IP "-m, --server \fIbkserver\fR" +L&B server machine name. If not specified, the environment variable EDG_WL_QUERY_SERVER is beeing checked. + +.IP "-a, --aborted \fiNNN[smhd]\fR" +Purge ABORTED jobs older than NNN seconds/minutes/hours/days (only one number and one letter allowed). If the time is not specified, the default timeout is determined by the server. + +.IP "-c, --cleared \fiNNN[smhd]\fR" +Purge CLEARED jobs older than given time. + +.IP "-n, --cancelled \fiNNN[smhd]\fR" +Purge CANCELLED jobs older than given time. + +.IP "-o, --other \fiNNN[smhd]\fR" +Purge OTHER (i.e. job in other state than ABORTED, CLEARED or CANCELLED) jobs older than given time. + +.IP "-r, --dry-run" +Do not really purge. + +.IP "-j, --jobs \fIfilename\fR" +Input file with jobIds of jobs to purge. + +.IP "-l, --return-list" +Return list of jobid matching the purge/dump criteria. + +.IP "-s, --server-dump" +Dump jobs at the server into a file (see bkserverd). + +.IP "-i, --client-dump" +Receive stream of dumped jobs (NOT implemented yet!). + +.IP "-f, --dump-full" +Dump full database records of purged jobs. + +.IP "-h, --help" +Display help. + +.IP "-v, --version" +Display version. + +.IP "-d, --debug" +Print diagnostics on the \fIstderr\fR. + + +.\".SH EXAMPLES +.\"To appear :o( + + +.SH FILES +No configuration files needed. + + +.SH ENVIRONMENT +.TP +.B EDG_WL_QUERY_SERVER +If +.RB $ EDG_WL_QUERY_SERVER +is set, its value is used as the bkserver address. + + +.SH REPORTING BUGS +Please, report all bugs to DataGrid Bug Tracking System located at http://marianne.in2p3.fr/datagrid/bugzilla + + +.SH SEE ALSO +\fBedg-wl-bkindex\fP(8), \fBedg-wl-bkserver\fP(8), \fBedg-wl-interlogd\fP(8), \fBedg-wl-logd\fP(8), \fBedg-wl-logev\fP(1), \fBedg-wl-bkdump\fP(8), \fBedg-wl-bkload\fP(8) + + +.SH AUTHOR +DataGrid Work Package 1, CESNET group. diff --git a/org.glite.lb.client/examples/multiple_user_jobs.c b/org.glite.lb.client/examples/multiple_user_jobs.c new file mode 100644 index 0000000..b4efbb9 --- /dev/null +++ b/org.glite.lb.client/examples/multiple_user_jobs.c @@ -0,0 +1,128 @@ +#include +#include +#include + +#include + +#include "glite/lb/context.h" +#include "glite/lb/xml_conversions.h" +#include "glite/lb/consumer.h" + +int use_proxy = 0; + +int (*user_jobs)(edg_wll_Context, edg_wlc_JobId **, edg_wll_JobStat **); + + +void +usage(char *me) +{ + fprintf(stderr,"This example demonstrates the use of several user identities provided byi\n" + "proxy files to access a single bkserver and retrieve appropriate information.\n\n" + "usage: %s [-h] \n" + "\t-h, --help\t show this help\n" + "\t\t A list of proxy files to use to contact the bkserver\n" + "\t \t Give \"default\" for default (EDG_WLL_PARAM_X509_PROXY == NULL)\n" + "\n" + ,me); + +} + +int main(int argc,char **argv) +{ + edg_wll_Context *p_ctx; + char *errt,*errd; + edg_wlc_JobId **jobs = NULL; + edg_wll_JobStat **states = NULL; + int i,j,k; + int proxy_file_no = 0; + int no_of_runs; + + if ((argc<2) || !strcmp(argv[1], "-h")) {usage(argv[0]); exit(0);} + + no_of_runs = argc-1; + + p_ctx = (edg_wll_Context*) calloc (sizeof(edg_wll_Context), no_of_runs); + jobs = (edg_wlc_JobId**) calloc (sizeof(edg_wlc_JobId*), no_of_runs); + states = (edg_wll_JobStat**) calloc (sizeof(edg_wll_JobStat*), no_of_runs); + + user_jobs = edg_wll_UserJobs; + for ( i = 1; i <= no_of_runs; i++ ) { + printf ("Proxy file No. %d: %s\n",i,argv[i]); + + edg_wll_InitContext(&p_ctx[i-1]); + if (strcmp(argv[i],"default")) edg_wll_SetParam(p_ctx[i-1], EDG_WLL_PARAM_X509_PROXY, argv[i]); + if (user_jobs(p_ctx[i-1],&jobs[i-1],&states[i-1])) goto err; + + } + + for (k=0; k < no_of_runs; k++) { + printf("Jobs retrieved using file No. %d (%s)\n" + "------------------------------------------\n", k + 1, argv[k + 1]); + for (i=0; states[k][i].state != EDG_WLL_JOB_UNDEF; i++) { + char *id = edg_wlc_JobIdUnparse(states[k][i].jobId), + *st = edg_wll_StatToString(states[k][i].state); + + if (!states[k][i].parent_job) { + if (states[k][i].jobtype == EDG_WLL_STAT_SIMPLE) { + printf(" %s .... %s %s\n", id, st, (states[k][i].state==EDG_WLL_JOB_DONE) ? edg_wll_done_codeToString(states[k][i].done_code) : "" ); + } + else if ((states[k][i].jobtype == EDG_WLL_STAT_DAG) || + (states[k][i].jobtype == EDG_WLL_STAT_COLLECTION)) { + printf("%s %s .... %s %s\n", (states[k][i].jobtype==EDG_WLL_STAT_DAG)?"DAG ":"COLL",id, st, (states[k][i].state==EDG_WLL_JOB_DONE) ? edg_wll_done_codeToString(states[k][i].done_code) : ""); + for (j=0; states[k][j].state != EDG_WLL_JOB_UNDEF; j++) { + if (states[k][j].parent_job) { + char *par_id = edg_wlc_JobIdUnparse(states[k][j].parent_job); + + if (!strcmp(id,par_id)) { + char *sub_id = edg_wlc_JobIdUnparse(states[k][j].jobId), + *sub_st = edg_wll_StatToString(states[k][j].state); + + printf(" `- %s .... %s %s\n", sub_id, sub_st, (states[k][j].state==EDG_WLL_JOB_DONE) ? edg_wll_done_codeToString(states[k][j].done_code) : ""); + free(sub_id); + free(sub_st); + } + free(par_id); + } + } + } + } + + free(id); + free(st); + } + } + + printf("\nFound %d jobs\n",i); + +err: + if (jobs) { + for (k=0; k < no_of_runs; k++) { + if (jobs[k]) + for (i=0; jobs[k][i]; i++) edg_wlc_JobIdFree(jobs[i]); + } + free(jobs); + } + + if (states) { + for (k=0; k < no_of_runs; k++) { + if (states[k]) + for (i=0; states[k][i].state; i++) edg_wll_FreeStatus(&states[i]); + } + free(states); + } + + for (k=0; k < no_of_runs; k++) { + if (edg_wll_Error(p_ctx[k],&errt,&errd)) { + fprintf(stderr,"%s: %s (%s)\n",argv[0],errt,errd); + return 1; + } + } + + + for (k=0; k < no_of_runs; k++) { + edg_wll_FreeContext(p_ctx[k]); + } + + return 0; +} + diff --git a/org.glite.lb.client/interface/JobStatus.h.T b/org.glite.lb.client/interface/JobStatus.h.T index 5e77bc0..d8e1128 100644 --- a/org.glite.lb.client/interface/JobStatus.h.T +++ b/org.glite.lb.client/interface/JobStatus.h.T @@ -66,6 +66,7 @@ public: $StatusAttrNums{$_} = $stno++; } + my $max = 0; selectType $status '_common_'; for my $u (sort {$a cmp $b} getAllFields $status) { selectField $status $u; @@ -76,8 +77,10 @@ public: gen defined($StatusAttrNums{$u}) ? "\t/** $f->{comment} */\n\t\t$u = $StatusAttrNums{$u},\n" : "#error \"$u not defined in StatusAttrNames\"\n"; + $max = $StatusAttrNums{$u} if defined($StatusAttrNums{$u}) && $StatusAttrNums{$u} > $max; } - gen "\tATTR_MAX = $stno /**< Limit for range checking. */ "; + $max++; + gen "\t\tATTR_MAX = $max /**< Limit for range checking. */\n"; @@@} }; diff --git a/org.glite.lb.client/project/ChangeLog b/org.glite.lb.client/project/ChangeLog new file mode 100644 index 0000000..8691429 --- /dev/null +++ b/org.glite.lb.client/project/ChangeLog @@ -0,0 +1,15 @@ +3.1.0-1 (lb-1.7.0-1) +- key connections in the pool also with certificate; allows using LB client + while swithing among multiple identities safely +- don't keep proxy connections at all (workaround bug #25153 with little + perfomance impact only) + +3.1.1-1 (lb-1.7.1-1) +- check for loaded credentials in connection pool management (avoid SEGV) +- minor fixes to manpage + +3.1.2-1 (lb-1.7.2-1) +- allow aditional options for purger (variables GLITE_LB_PURGE_OTHER_OPTIONS, + GLITE_LB_DUMP_EXPORTER_OTHER_OPTIONS +- pass errors from credential loading to upper layers properly + diff --git a/org.glite.lb.client/project/version.properties b/org.glite.lb.client/project/version.properties index 888ecb1..c591a16 100644 --- a/org.glite.lb.client/project/version.properties +++ b/org.glite.lb.client/project/version.properties @@ -1,3 +1,3 @@ # $Header$ -module.version=3.1.0 +module.version=3.1.2 module.age=1 diff --git a/org.glite.lb.client/src/connection.c b/org.glite.lb.client/src/connection.c index 5492a34..2f1e919 100644 --- a/org.glite.lb.client/src/connection.c +++ b/org.glite.lb.client/src/connection.c @@ -69,8 +69,9 @@ int ConnectionIndex(edg_wll_Context ctx, const char *name, int port) !strcmp(name, ctx->connections->connPool[i].peerName) && // Server names must be equal (port == ctx->connections->connPool[i].peerPort) && // Ports must be equal (!using_certfile || // we are either using the default cert file - ((ctx->connections->connPool[i].certfile->st_ino == statinfo.st_ino) && // or checking which file - (ctx->connections->connPool[i].certfile->st_dev == statinfo.st_dev)))) { // this conn uses to auth. + ((ctx->connections->connPool[i].certfile) && // or checking which file + (ctx->connections->connPool[i].certfile->st_ino == statinfo.st_ino) && // this conn + (ctx->connections->connPool[i].certfile->st_dev == statinfo.st_dev)))) { // uses to auth. /* TryLock (next line) is in fact used only diff --git a/org.glite.lb.client/src/export.sh b/org.glite.lb.client/src/export.sh index 362d119..58503df 100644 --- a/org.glite.lb.client/src/export.sh +++ b/org.glite.lb.client/src/export.sh @@ -54,14 +54,14 @@ GLITE_LB_EXPORT_ENABLED=${GLITE_LB_EXPORT_ENABLED:-true} [ -d $GLITE_LB_EXPORT_JOBSDIR ] || mkdir -p $GLITE_LB_EXPORT_JOBSDIR if [ x"$GLITE_LB_PURGE_ENABLED" = x"true" ]; then - X509_USER_CERT="$X509_USER_CERT" X509_USER_KEY="$X509_USER_KEY" $PREFIX/bin/glite-lb-purge $GLITE_LB_EXPORT_PURGE_ARGS -l -m $GLITE_LB_EXPORT_BKSERVER -s + X509_USER_CERT="$X509_USER_CERT" X509_USER_KEY="$X509_USER_KEY" $PREFIX/bin/glite-lb-purge $GLITE_LB_EXPORT_PURGE_ARGS -l -m $GLITE_LB_EXPORT_BKSERVER -s $GLITE_LB_PURGE_OTHER_OPTIONS fi if [ x"$GLITE_LB_EXPORT_ENABLED" = x"true" ]; then list=`ls $GLITE_LB_EXPORT_PURGEDIR/* 2>/dev/null` for file in $list; do if [ -s $file ]; then - $PREFIX/bin/glite-lb-dump_exporter -d $file -s $GLITE_LB_EXPORT_JOBSDIR -m $GLITE_LB_EXPORT_JPDUMP_MAILDIR + $PREFIX/bin/glite-lb-dump_exporter -d $file -s $GLITE_LB_EXPORT_JOBSDIR -m $GLITE_LB_EXPORT_JPDUMP_MAILDIR $GLITE_LB_DUMP_EXPORTER_OTHER_OPTIONS if [ -n "$GLITE_LB_EXPORT_PURGEDIR_KEEP" ]; then mv $file $GLITE_LB_EXPORT_PURGEDIR_KEEP else diff --git a/org.glite.lb.client/src/prod_proto.c b/org.glite.lb.client/src/prod_proto.c index a51a9f2..00d102a 100644 --- a/org.glite.lb.client/src/prod_proto.c +++ b/org.glite.lb.client/src/prod_proto.c @@ -279,7 +279,7 @@ int edg_wll_log_connect(edg_wll_Context ctx, int *conn) &ctx->connections->connPool[index].gsiCred, &gss_stat); /* give up if unable to acquire prescribed credentials, otherwise go on anonymously */ if (ret && ctx->p_proxy_filename) { - edg_wll_SetErrorGss(ctx, "edg_wll_gss_acquire_cred_gsi(): failed to load GSI credentials", &gss_stat); + answer = edg_wll_SetErrorGss(ctx, "edg_wll_gss_acquire_cred_gsi(): failed to load GSI credentials", &gss_stat); goto edg_wll_log_connect_err; } my_subject_name = ctx->connections->connPool[index].gsiCred->name; @@ -670,7 +670,7 @@ int edg_wll_log_direct_connect(edg_wll_Context ctx, edg_wll_GssConnection *conn) &cred, &gss_stat); /* give up if unable to acquire prescribed credentials, otherwise go on anonymously */ if (ret && ctx->p_proxy_filename) { - edg_wll_SetErrorGss(ctx, "edg_wll_gss_acquire_cred_gsi(): failed to load GSI credentials", &gss_stat); + answer = edg_wll_SetErrorGss(ctx, "edg_wll_gss_acquire_cred_gsi(): failed to load GSI credentials", &gss_stat); goto edg_wll_log_direct_connect_end; } my_subject_name = cred->name; diff --git a/org.glite.lb.common/interface/Event.h.T b/org.glite.lb.common/interface/Event.h.T index da1e635..b6b7285 100644 --- a/org.glite.lb.common/interface/Event.h.T +++ b/org.glite.lb.common/interface/Event.h.T @@ -70,6 +70,7 @@ public: $EventAttrNums{$_} = $evno++; } + my $max = 0; for (sort {$a cmp $b} getAllFields $event) { my $u = $_; # $u =~ s/([a-z])([A-Z])/$1_$2/g; @@ -87,10 +88,12 @@ public: gen defined($EventAttrNums{$u}) ? "$c\t\t$u = $EventAttrNums{$u},\n" : "#error \"$u not defined in EventAttrNames\"\n"; + + $max = $EventAttrNums{$u} if defined($EventAttrNums{$u}) && $EventAttrNums{$u} > $max; } + $max++; + gen "\t\tATTR_MAX = $max /**< Limit for checking attribute code validity. */\n"; @@@} - ATTR_MAX /**< Limit for checking attribute code - validity. */ }; @@@{ diff --git a/org.glite.lb.common/interface/context-int.h b/org.glite.lb.common/interface/context-int.h index bc8edc9..360e9cb 100644 --- a/org.glite.lb.common/interface/context-int.h +++ b/org.glite.lb.common/interface/context-int.h @@ -44,7 +44,9 @@ glite_lb_padded_struct(_edg_wll_ConnProxy,12, typedef struct _edg_wll_ConnProxy edg_wll_ConnProxy; +/* !!! if adding something malloc-able, update edg_wll_FreeContext too !!! */ glite_lb_padded_struct(_edg_wll_Context,150, +/// XXX: branch value: glite_lb_padded_struct(_edg_wll_Context,120, /* Error handling */ int errCode; /* recent error code */ char *errDesc; /* additional error description */ diff --git a/org.glite.lb.common/project/ChangeLog b/org.glite.lb.common/project/ChangeLog new file mode 100644 index 0000000..ecbe5ee --- /dev/null +++ b/org.glite.lb.common/project/ChangeLog @@ -0,0 +1,9 @@ +6.1.0-1 (lb-1.7.0-1) +- key connections in the pool also with certificate; allows using LB client + while swithing among multiple identities safely +- support for FQAN-based super-user authorization in LB server +- resurrected and extended some unit tests + +6.1.1-1 (lb-1.7.1-1) +- fixed memleaks in LB query protocol + diff --git a/org.glite.lb.common/src/EventAttrNames.pl b/org.glite.lb.common/src/EventAttrNames.pl index 1fc8620..a2720e1 100644 --- a/org.glite.lb.common/src/EventAttrNames.pl +++ b/org.glite.lb.common/src/EventAttrNames.pl @@ -80,4 +80,5 @@ USER_ID_TYPE VALUE WN_SEQ + EXPIRES /; diff --git a/org.glite.lb.common/src/context.c b/org.glite.lb.common/src/context.c index 639638f..535c743 100644 --- a/org.glite.lb.common/src/context.c +++ b/org.glite.lb.common/src/context.c @@ -140,6 +140,9 @@ void edg_wll_FreeContext(edg_wll_Context ctx) free(ctx->fqans); ctx->fqans = NULL; } + + if (ctx->jpreg_dir) free(ctx->jpreg_dir); + if (ctx->serverIdentity) free(ctx->serverIdentity); edg_wll_FreeParams(ctx); diff --git a/org.glite.lb.common/src/xml_conversions.c b/org.glite.lb.common/src/xml_conversions.c index 20006b3..4befc08 100644 --- a/org.glite.lb.common/src/xml_conversions.c +++ b/org.glite.lb.common/src/xml_conversions.c @@ -95,8 +95,10 @@ void edg_wll_freeXMLCtx(edg_wll_XML_ctx *c) { void edg_wll_freeBuf(edg_wll_XML_ctx *c) { - free(c->char_buf); - c->char_buf = NULL; + if (c->char_buf) { + free(c->char_buf); + c->char_buf = NULL; + } c->char_buf_len = 0; } @@ -369,12 +371,14 @@ void edg_wll_add_intlist_to_XMLBody(char **body, const int *toAdd, const char *t char **list = NULL; int i, len, tot_len = 0; int *len_list = NULL; - i = from; while (i <= to) { + char *tag2 = indexToTag(i-1); len = trio_asprintf(&pomA,"%s\t<%s>%|Xd\r\n", - indent, indexToTag(i-1),toAdd[i],indexToTag(i-1)); + indent,tag2,toAdd[i],tag2); + + if (tag2) free(tag2); i++; tot_len += len; @@ -476,8 +480,11 @@ void edg_wll_add_time_t_list_to_XMLBody(char **body, const time_t *toAdd, const i = from; while (i < to) { + char *tag2 = indexToTag(i); len = trio_asprintf(&pomA,"%s\t<%s>%|Xld\r\n", - indent, indexToTag(i),toAdd[i],indexToTag(i)); + indent,tag2,toAdd[i],tag2); + + if (tag2) free(tag2); i++; tot_len += len; diff --git a/org.glite.lb.common/src/xml_parse.c.T b/org.glite.lb.common/src/xml_parse.c.T index ca9a256..97c22e4 100644 --- a/org.glite.lb.common/src/xml_parse.c.T +++ b/org.glite.lb.common/src/xml_parse.c.T @@ -782,8 +782,7 @@ static void endQueryJobs(void *data, const char *el) break; } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); memset(&(XMLCtx->element), 0, sizeof(XMLCtx->element)); XMLCtx->level--; } @@ -860,8 +859,7 @@ static void endQueryEvents(void *data, const char *el UNUSED_VAR) break; } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); memset(&(XMLCtx->element), 0, sizeof(XMLCtx->element)); XMLCtx->level--; @@ -1002,8 +1000,7 @@ static void endJobStat(void *data, const char *el) break; } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); memset(&(XMLCtx->element), 0, sizeof(XMLCtx->element)); XMLCtx->level--; } @@ -1036,8 +1033,7 @@ static void endStrList(void *data, const char *el UNUSED_VAR) break; } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); memset(&(XMLCtx->element), 0, sizeof(XMLCtx->element)); XMLCtx->level--; } @@ -1070,8 +1066,7 @@ static void endIntList(void *data, const char *el UNUSED_VAR) break; } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); memset(&(XMLCtx->element), 0, sizeof(XMLCtx->element)); XMLCtx->level--; } @@ -1103,8 +1098,7 @@ static void endTagList(void *data, const char *el UNUSED_VAR) break; } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); memset(&(XMLCtx->element), 0, sizeof(XMLCtx->element)); XMLCtx->level--; } @@ -1140,8 +1134,7 @@ static void endStsList(void *data, const char *el) break; } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); memset(&(XMLCtx->element), 0, sizeof(XMLCtx->element)); XMLCtx->level--; } @@ -1175,8 +1168,7 @@ static void endPurgeResult(void *data, const char *el UNUSED_VAR) } } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } @@ -1203,8 +1195,7 @@ static void endDumpResult(void *data, const char *el UNUSED_VAR) } } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } @@ -1224,8 +1215,7 @@ static void endLoadResult(void *data, const char *el UNUSED_VAR) } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } @@ -1259,8 +1249,7 @@ static void endIndexedAttrs(void *data, const char *el) } } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } @@ -1275,8 +1264,7 @@ static void endNotifResult(void *data, const char *el UNUSED_VAR) XMLCtx->notifValidity = edg_wll_from_string_to_time_t(XMLCtx); } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } @@ -1291,8 +1279,7 @@ static void endQuerySequenceCodeResult(void *data, const char *el UNUSED_VAR) XMLCtx->seqCode = edg_wll_from_string_to_string(XMLCtx); } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } @@ -1321,8 +1308,7 @@ static void endStatsResult(void *data, const char *el UNUSED_VAR) XMLCtx->statsResTo = edg_wll_from_string_to_int(XMLCtx); } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } diff --git a/org.glite.lb.logger/Makefile b/org.glite.lb.logger/Makefile index fd11f89..0fba2f8 100644 --- a/org.glite.lb.logger/Makefile +++ b/org.glite.lb.logger/Makefile @@ -17,7 +17,7 @@ version=${module.version} CC=gcc -VPATH:=${top_srcdir}/src:${top_srcdir}/test +VPATH:=${top_srcdir}/src:${top_srcdir}/test:${top_srcdir}/doc VERSION=-DVERSION=\"GLite-${version}\" @@ -100,9 +100,12 @@ INTERLOG_TEST_OBJS:= \ IlTestBase.o \ il_test.o +MAN_GZ:=glite-lb-interlogd.8.gz glite-lb-logd.8.gz +MAN = $(MAN_GZ:.gz=) + default: all -all compile: $(LOGD) $(INTERLOGD) $(NOTIF_INTERLOGD) +all compile: $(LOGD) $(INTERLOGD) $(NOTIF_INTERLOGD) ${MAN_GZ} glite-lb-logd: ${LOGD_OBJS} ${LINK} -o $@ ${LOGD_OBJS} ${COMMON_LIB}_${nothrflavour} ${TRIO_LIB} @@ -132,6 +135,12 @@ glite-lb-interlogd-perf-empty: ${INTERLOG_EMPTY_OBJS} # ${LINK} -o $@ ${INTERLOG_INLINE_EMPTY_OBJS} \ # ${COMMON_LIB}_${thrflavour} +${MAN_GZ}: ${MAN} + cp $? . + gzip $(notdir $?) + +man: ${MAN_GZ} + stage: compile $(MAKE) install PREFIX=${stagedir} DOSTAGE=yes @@ -153,6 +162,7 @@ install: -mkdir -p ${PREFIX}/bin -mkdir -p ${PREFIX}/etc/init.d -mkdir -p ${PREFIX}/share/doc/${package}-${version} + -mkdir -p ${PREFIX}/share/man/man8 ${INSTALL} -m 755 ${LOGD} ${PREFIX}/bin ${INSTALL} -m 755 ${INTERLOGD} ${PREFIX}/bin if [ x${DOSTAGE} = xyes ]; then \ @@ -164,6 +174,7 @@ ifdef LB_PERF endif ${INSTALL} -m 755 ${top_srcdir}/config/startup ${PREFIX}/etc/init.d/glite-lb-locallogger ${INSTALL} -m 644 ${top_srcdir}/LICENSE ${PREFIX}/share/doc/${package}-${version} + ${INSTALL} -m 644 ${MAN_GZ} ${PREFIX}/share/man/man8 ${INTERLOG_NOBJS}: %.no: %.c ${CC} ${CFLAGS} -DIL_NOTIFICATIONS -c $< -o $@ @@ -196,5 +207,5 @@ il_test.o IlTestBase.o server_msgTest.o event_queueTest.o input_queue_socketTest ${CXX} ${CFLAGS} ${TEST_INC} -c $< -o $@ clean: - rm -rvf .libs/ *.o *.no ${LOGD} ${INTERLOGD} ${NOTIF_INTERLOGD} + rm -rvf .libs/ *.o *.no ${LOGD} ${INTERLOGD} ${NOTIF_INTERLOGD} {MAN_GZ} rm -rvf log.xml project/ rpmbuild/ RPMS/ tgz/ diff --git a/org.glite.lb.logger/doc/glite-lb-interlogd.8 b/org.glite.lb.logger/doc/glite-lb-interlogd.8 new file mode 100644 index 0000000..20732a0 --- /dev/null +++ b/org.glite.lb.logger/doc/glite-lb-interlogd.8 @@ -0,0 +1,175 @@ +.TH EDG-WL-INTERLOGD 8 "May 2003" "EU DataGrid Project" "Logging&Bookkeeping" + +.SH NAME +glite-lb-interlogd - interlogger daemon + +.SH SYNOPSIS +.B glite-lb-interlogd +.RI [ options ] +.br + +.SH DESCRIPTION +.B glite-lb-interlogd +takes over L&B events from glite-lb-logd and +it is responsible for reliable transfer to their target storage - +bookkeeping and logging servers. +Interlogger maintains message queues for each target server +and implements reliable handling in environment with possible communication +failures. + +On startup (and then periodically during operation) interlogger checks the +event files (created by glite-lb-logd, see also the -f option), +spawns a thread for each target server, and populates the queues +with events read from the files. +Then it starts listening for further incoming events on the UNIX socket +(see the -s option). + +The queue threads periodically attempt to contact the destination servers +and to deliver the pending events. + +Eventually, when all events from a file are delivered the file is removed +after a certain timeout by a clean-up thread. +Possible race conditions wrt. glite-lb-logd are prevented by +locking the files appropriately. + +.SH OPTIONS +.TP +.B "-b\fR,\fP --book +Send events only to bookkeeping server. This option has effect only if +the logging server address is specified through the -l option. There is no +implementation of logging server currently. + +.TP +.BI \-c " FILE" "\fR,\fP --cert " FILE +Get certificate from +.I FILE\fR.\fP +This option overrides the +.B \fR$\fPX509_USER_CERT +environment variable. + +.TP +.BI \-k " FILE" "\fR,\fP --key " FILE +Get server private key from +.I FILE\fR.\fP +This option overrides the +.B \fR$\fPX509_USER_KEY +environment variable. + +.TP +.BI \-C " DIR" "\fR,\fP --CAdir " DIR +Look for trusted CA's certificates in +.I DIR\fR.\fP +This option overrides the +.B \fR$\fPX509_CERT_DIR +environment variable. + +.TP +.B "-d\fR,\fP --debug" +Don't run as daemon (do not fork and put itself into background). + +.TP +.BI \-f " PREFIX" "\fR,\fP --file-prefix " PREFIX +.I PREFIX +is path prefix of the event files. + +For each job (jobid) a specific file is created that stores all events +associated with this job. +The job filename is constructed by stripping the +protocol+server prefix from the jobid and appending the rest to this +.I PREFIX. + +.I PREFIX +defaults to /tmp/dglogd.log. + +On startup, interlogger checks all files matching the pattern +.I PREFIX*. + +The value has to be same as used in the cooperating glite-lb-logd. + +.TP +.BI \-k " FILE" "\fR,\fP --key " FILE +Get private key from +.I FILE\fR.\fP +This option overrides the +.B \fR$\fPX509_USER_KEY +environment variable. + +.TP +.BI -l " HOST" "\fR,\fP --log-server " HOST +Use +.I HOST +as address of logging server. +As the logging is not implemented yet this option should not be used. + +.TP +.BI -s " PATH" "\fR,\fP --socket " PATH +Listen for events sent by glite-lb-logd at the UNIX socket +.I PATH. + +.TP +.B "-v\fR,\fP --verbose" +Print extensive debug output. + +.TP +.B "-V\fR,\fP --version" +Print version and exit. + +.TP +.BI -L " TIMEOUT" "\fR,\fP --lazy " TIMEOUT +Be lazy when closing connections to servers (default, +.I TIMEOUT\fR +==0 means turn lazy off). + +.\".SH USAGE +.\" Add any additional description here + +.PP + +.SH FILES +.TP +.I /tmp/interlogger.sock +Default name of local socket. +.TP +.I /tmp/dglogd.log* +Default location of event files. + +.I /tmp/dglogd.log*.ctl +Interlogger's control files keeping the information on status of +the corresponding event file wrt. delivery to the target server. + +.TP +No configuration files needed. + +.SH ENVIRONMENT +.TP +.B X509_USER_KEY +If +.B \fR$\fPX509_USER_KEY +is set, it is used to locate private key file. + +.TP +.B X509_USER_CERT +If +.B \fR$\fPX509_USER_CERT +is set, it is used to locate certificate file. + +.TP +.B X509_CERT_DIR +If +.B \fR$\fPX509_CERT_DIR +is set, it is used to locate the trusted CA's certificates and ca-signing-policy files. + +.TP +.B X509_USER_PROXY +If +.B \fR$\fPX509USER_PROXY +is set, it is used to locate proxy certificate file. + +.SH REPORTING BUGS +Please, report all bugs to EU DataGrid Bug Tracking System located at http://marianne.in2p3.fr/datagrid/bugzilla + +.SH SEE ALSO +.B te-lb-bkserverd\fR(8),\fP glite-lb-logd\fR(8),\fP glite-lb-logevent\fR(1),\fP + +.SH AUTHOR +EU DataGrid Work Package 1, CESNET group. diff --git a/org.glite.lb.logger/doc/glite-lb-logd.8 b/org.glite.lb.logger/doc/glite-lb-logd.8 new file mode 100644 index 0000000..c9bf108 --- /dev/null +++ b/org.glite.lb.logger/doc/glite-lb-logd.8 @@ -0,0 +1,162 @@ +.TH EDG-WL-LOGD 8 "May 2003" "EU DataGrid Project" "Logging&Bookkeeping" + +.SH NAME +glite-lb-logd - local logger daemon + +.SH SYNOPSIS +.B glite-lb-logd +.RI [ options ] +.br + +.SH DESCRIPTION +.B glite-lb-logd +accepts the L&B events from their sources (via the producer L&B API). +It is responsible for fast acceptance of the events and their reliable storage +in local files. + +Under normal conditions, the events are forwared to the +.B glite-lb-interlogd +immediately. +The file storage allows the interlogger crash recovery. + +.SH OPTIONS +.TP +.BI \-c " FILE" "\fR,\fP --cert " FILE +Get certificate from +.I FILE\fR.\fP +This option overrides the +.B \fR$\fPX509_USER_CERT +environment variable. + +.TP +.BI \-C " DIR" "\fR,\fP --CAdir " DIR +Look for trusted CA's certificates in +.I DIR\fR.\fP +This option overrides the +.B \fR$\fPX509_CERT_DIR +environment variable. + +.TP +.B "-d\fR,\fP --debug" +Don't run as daemon, print additional diagnostics. + +.TP +.B "-v\fR,\fP --verbose" +Print extensive debug output. + +.TP +.BI \-p " NUM" "\fR,\fP --port " NUM +Listen on the port +.I NUM\fR.\fP + +.TP +.BI \-f " PREFIX" "\fR,\fP --file-prefix " PREFIX +.I PREFIX +is a path prefix of the event files. + +For each job (jobid) a specific file is created that stores all events +associated with this job. +The job filename is constructed by stripping the +protocol+server prefix from the jobid and appending the rest to this +.I PREFIX. + +.I PREFIX +defaults to /tmp/dglogd.log. + +The value has to be same as used in the cooperating glite-lb-logd. + +.TP +.BI \-k " FILE" "\fR,\fP --key " FILE +Get private key from +.I FILE\fR.\fP +This option overrides the +.B \fR$\fPX509_USER_KEY +environment variable. + + +.TP +.BI -s " PATH" "\fR,\fP --socket " PATH +Send the messages to interlogger through the UNIX socket +.I PATH\fR.\fP + +The value has to be same as used in the cooperating glite-lb-interlogd. + +.TP +.B "-V\fR,\fP --version" +Print version and exit. + +.TP +.B --noAuth +Don't require valid X509 credentials to run the daemon. +Used for debugging only. + +.TP +.B --noIPC +Don't send messages to interlogger. + +.TP +.B --noParse +Don't parse messages for correctness. +Dangerous, for debugging only! Don't use at all. + + +.\".SH USAGE +.\" Add any additional description here + +.PP + +.SH FILES +.TP +.I /tmp/interlogger.sock +Default name of local socket. + +.TP +.I /tmp/dglogd.log* +Default location of the event storage files. +.TP +No configuration files needed. + +.SH ENVIRONMENT +.TP +.B X509_USER_KEY +If +.B \fR$\fPX509_USER_KEY +is set, it is used to locate private key file. + +.TP +.B X509_USER_CERT +If +.B \fR$\fPX509_USER_CERT +is set, it is used to locate certificate file. + +.TP +.B X509_CERT_DIR +If +.B \fR$\fPX509_CERT_DIR +is set, it is used to locate the trusted CA's certificates and ca-signing-policy files. + +.TP +.B X509_USER_PROXY +If +.B \fR$\fPX509USER_PROXY +is set, it is used to locate proxy certificate file. + + +.SH SIGNALS +.TP +.B USR1 +Increase verbosity of the program. + +.TP +.B USR2 +Decrease verbosity of the program. + + +.SH REPORTING BUGS +Please, report all bugs to DataGrid Bug Tracking System located at http://marianne.in2p3.fr/datagrid/bugzilla + +.SH SEE ALSO +.B glite-lb-bkserverd\fR(8),\fP glite-lb-interlogd\fR(8),\fP glite-lb-logevent\fR(1),\fP + +.SH AUTHOR +EU DataGrid Work Package 1, CESNET group. diff --git a/org.glite.lb.logger/project/ChangeLog b/org.glite.lb.logger/project/ChangeLog new file mode 100644 index 0000000..bb564d6 --- /dev/null +++ b/org.glite.lb.logger/project/ChangeLog @@ -0,0 +1,10 @@ +1.4.6-1 (lb-1.7.0-1) +- let empty notification message get through the output queue to keep + event_store commits in sync +- don't keep event queue mutex while sleeping on error + +1.4.7-1 (lb-1.7.1-1) +- support for message expiration + +1.4.8-1 (lb-1.7.2-1) +- support for notification expiration diff --git a/org.glite.lb.logger/project/version.properties b/org.glite.lb.logger/project/version.properties index 8012667..a9bb99b 100644 --- a/org.glite.lb.logger/project/version.properties +++ b/org.glite.lb.logger/project/version.properties @@ -1,3 +1,3 @@ # $Header$ -module.version=1.4.6 +module.version=1.4.8 module.age=1 diff --git a/org.glite.lb.logger/src/event_queue.c b/org.glite.lb.logger/src/event_queue.c index 2d5b64a..65a22a8 100644 --- a/org.glite.lb.logger/src/event_queue.c +++ b/org.glite.lb.logger/src/event_queue.c @@ -270,15 +270,15 @@ event_queue_remove(struct event_queue *eq) return(0); } -#if defined(IL_NOTIFICATIONS) - int -event_queue_move_events(struct event_queue *eq_s, struct event_queue *eq_d, char *notif_id) +event_queue_move_events(struct event_queue *eq_s, + struct event_queue *eq_d, + int (*cmp_func)(struct server_msg *, void *), + void *data) { struct event_queue_msg *p, **source_prev, **dest_tail; assert(eq_s != NULL); - assert(notif_id != NULL); event_queue_lock(eq_s); if(eq_d) { @@ -290,9 +290,10 @@ event_queue_move_events(struct event_queue *eq_s, struct event_queue *eq_d, char p = *source_prev; eq_s->tail = NULL; while(p) { - if(strcmp(p->msg->job_id_s, notif_id) == 0) { + if((*cmp_func)(p->msg, data)) { il_log(LOG_DEBUG, " moving event at offset %d from %s:%d to %s:%d\n", - p->msg->offset, eq_s->dest_name,eq_s->dest_port, eq_d ? eq_d->dest_name : "trash",eq_d ? eq_d->dest_port : -1); + p->msg->offset, eq_s->dest_name, eq_s->dest_port, + eq_d ? eq_d->dest_name : "trash", eq_d ? eq_d->dest_port : -1); il_log(LOG_DEBUG, " current: %x, next: %x\n", p, p->prev); /* remove the message from the source list */ *source_prev = p->prev; @@ -319,4 +320,3 @@ event_queue_move_events(struct event_queue *eq_s, struct event_queue *eq_d, char return(0); } -#endif diff --git a/org.glite.lb.logger/src/il_master.c b/org.glite.lb.logger/src/il_master.c index 9e00759..1a326da 100644 --- a/org.glite.lb.logger/src/il_master.c +++ b/org.glite.lb.logger/src/il_master.c @@ -15,6 +15,27 @@ #include "glite/lb/lb_perftest.h" #endif +static +int +cmp_jobid(struct server_msg *msg, void *data) +{ + char *job_id_s = (char*)data; + return strcmp(msg->job_id_s, job_id_s) == 0; +} + +static +int +cmp_jobid_set_exp(struct server_msg *msg, void *data) +{ + struct server_msg *m = (struct server_msg *)data; + + if(strcmp(msg->job_id_s, m->job_id_s) == 0) { + msg->expires = m->expires; + } + return 0; +} + + int enqueue_msg(struct event_queue *eq, struct server_msg *msg) { @@ -31,10 +52,17 @@ enqueue_msg(struct event_queue *eq, struct server_msg *msg) return(-1); /* move all events with this notif_id from eq_known to eq */ if(eq_known != NULL) { - event_queue_move_events(eq_known, eq, msg->job_id_s); + event_queue_move_events(eq_known, eq, cmp_jobid, msg->job_id_s); /* XXX - we should kill the old queue too */ } } + + /* if the expiration changed, set new one */ + if(msg->expires != notifid_map_get_expiration(msg->job_id_s)) { + notifid_map_set_expiration(msg->job_id_s, msg->expires); + /* set expiration for all events with this notif id */ + event_queue_move_events(eq, NULL, cmp_jobid_set_exp, msg); + } #endif /* fire thread to take care of this queue */ diff --git a/org.glite.lb.logger/src/interlogd.h b/org.glite.lb.logger/src/interlogd.h index 60e10c3..63e4985 100644 --- a/org.glite.lb.logger/src/interlogd.h +++ b/org.glite.lb.logger/src/interlogd.h @@ -61,12 +61,14 @@ // #define TIMEOUT 5 extern int TIMEOUT; #define INPUT_TIMEOUT (60) +#define EXIT_TIMEOUT (1*60) typedef struct cred_handle { edg_wll_GssCred creds; int counter; } cred_handle_t; extern cred_handle_t *cred_handle; + extern pthread_mutex_t cred_handle_lock; extern pthread_key_t cred_handle_key; extern char *cert_file; @@ -119,6 +121,7 @@ struct server_msg { int dest_port; char *dest; #endif + time_t expires; /* time (in seconds from epoch) the message expires */ }; @@ -172,6 +175,7 @@ int event_queue_remove(struct event_queue *); int event_queue_enqueue(struct event_queue *, char *); /* helper */ int enqueue_msg(struct event_queue *, struct server_msg *); +int event_queue_move_events(struct event_queue *, struct event_queue *, int (*)(struct server_msg *, void *), void *); /* protocol event queue methods */ int event_queue_connect(struct event_queue *); @@ -206,7 +210,8 @@ int queue_list_is_log(struct event_queue *); #if defined(IL_NOTIFICATIONS) struct event_queue *notifid_map_get_dest(const char *); int notifid_map_set_dest(const char *, struct event_queue *); -int event_queue_move_events(struct event_queue *, struct event_queue *, char *); +time_t notifid_map_get_expiration(const char *); +int notifid_map_set_expiration(const char *, time_t); #endif /* event store functions */ diff --git a/org.glite.lb.logger/src/queue_mgr.c b/org.glite.lb.logger/src/queue_mgr.c index ba01aec..2134a32 100644 --- a/org.glite.lb.logger/src/queue_mgr.c +++ b/org.glite.lb.logger/src/queue_mgr.c @@ -13,6 +13,9 @@ struct queue_list { struct event_queue *queue; char *dest; struct queue_list *next; +#if defined(IL_NOTIFICATIONS) + time_t expires; +#endif }; static struct event_queue *log_queue; @@ -92,24 +95,6 @@ queue_list_add(struct queue_list **ql, const char *dest, struct event_queue *eq) } -/* -static -int -queue_list_remove(struct queue_list *el, struct queue_list *prev) -{ - assert(el != NULL); - - if(prev) - prev->next = el->next; - else - queues = el->next; - - free(el); - return(1); -} -*/ - - #if !defined(IL_NOTIFICATIONS) static char * @@ -214,6 +199,17 @@ queue_list_next() return(current ? current->queue : NULL); } + +int +queue_list_remove_queue(struct event_queue *eq) +{ + assert(eq != NULL); + + free(eq); + return(1); +} + + #if defined(IL_NOTIFICATIONS) static struct queue_list *notifid_map = NULL; @@ -242,4 +238,32 @@ notifid_map_set_dest(const char *notif_id, struct event_queue *eq) } } + +time_t +notifid_map_get_expiration(const char * notif_id) +{ + struct queue_list *q; + + queue_list_find(notifid_map, notif_id, &q, NULL); + return(q ? q->expires : 0); +} + + +int +notifid_map_set_expiration(const char *notif_id, time_t exp) +{ + struct queue_list *q; + + if(queue_list_find(notifid_map, notif_id, &q, NULL)) { + q->expires = exp; + return(1); + } else { + return(0); + } +} + #endif + +/* Local Variables: */ +/* c-indentation-style: gnu */ +/* End: */ diff --git a/org.glite.lb.logger/src/queue_thread.c b/org.glite.lb.logger/src/queue_thread.c index cb5424e..b75c501 100644 --- a/org.glite.lb.logger/src/queue_thread.c +++ b/org.glite.lb.logger/src/queue_thread.c @@ -27,6 +27,16 @@ queue_thread_cleanup(void *q) } +static time_t now; + +static +int +cmp_expires(struct server_msg *msg, void *data) +{ + time_t *t = (time_t*)data; + return msg->expires < *t; +} + static void * queue_thread(void *q) @@ -34,7 +44,8 @@ queue_thread(void *q) struct event_queue *eq = (struct event_queue *)q; int ret, exit; int retrycnt; - int close_timeout; + int close_timeout = 0; + int exit_timeout = EXIT_TIMEOUT; if(init_errors(0) < 0) { il_log(LOG_ERR, "Error initializing thread specific data, exiting!"); @@ -68,8 +79,15 @@ queue_thread(void *q) eq->dest_name, eq->dest_port); } close_timeout = 0; - } else - ret = event_queue_wait(eq, 0); + } else { + ret = event_queue_wait(eq, exit_timeout); + if(ret == 1) { + il_log(LOG_INFO, " thread idle for more than %d seconds, exiting\n", exit_timeout); + event_queue_close(eq); + event_queue_cond_unlock(eq); + pthread_exit((void*)0); + } + } if(ret < 0) { /* error waiting */ il_log(LOG_ERR, "queue_thread: %s\n", error_get_msg()); @@ -78,56 +96,64 @@ queue_thread(void *q) } } /* END while(empty) */ - il_log(LOG_DEBUG, " attempting delivery to %s:%d\n", eq->dest_name, eq->dest_port); /* allow other threads to signal us, ie. insert new events while * we are sending or request flush operation */ event_queue_cond_unlock(eq); - /* connect to server */ - if((ret=event_queue_connect(eq)) == 0) { - /* not connected */ - if(error_get_maj() != IL_OK) - il_log(LOG_ERR, "queue_thread: %s\n", error_get_msg()); + /* discard expired events */ + il_log(LOG_DEBUG, " discarding expired events\n"); + now = time(NULL); + event_queue_move_events(eq, NULL, cmp_expires, &now); + if(!event_queue_empty(eq)) { + + /* deliver pending events */ + il_log(LOG_DEBUG, " attempting delivery to %s:%d\n", eq->dest_name, eq->dest_port); + /* connect to server */ + if((ret=event_queue_connect(eq)) == 0) { + /* not connected */ + if(error_get_maj() != IL_OK) + il_log(LOG_ERR, "queue_thread: %s\n", error_get_msg()); #if defined(IL_NOTIFICATIONS) - il_log(LOG_INFO, " could not connect to client %s, waiting for retry\n", eq->dest_name); + il_log(LOG_INFO, " could not connect to client %s, waiting for retry\n", eq->dest_name); #else - il_log(LOG_INFO, " could not connect to bookkeeping server %s, waiting for retry\n", eq->dest_name); + il_log(LOG_INFO, " could not connect to bookkeeping server %s, waiting for retry\n", eq->dest_name); #endif - retrycnt++; - } else { - retrycnt = 0; - /* connected, send events */ - switch(ret=event_queue_send(eq)) { - - case 0: - /* there was an error and we still have events to send */ - if(error_get_maj() != IL_OK) + retrycnt++; + } else { + retrycnt = 0; + /* connected, send events */ + switch(ret=event_queue_send(eq)) { + + case 0: + /* there was an error and we still have events to send */ + if(error_get_maj() != IL_OK) + il_log(LOG_ERR, "queue_thread: %s\n", error_get_msg()); + il_log(LOG_DEBUG, " events still waiting\n"); + break; + + case 1: + /* hey, we are done for now */ + il_log(LOG_DEBUG, " all events for %s sent\n", eq->dest_name); + break; + + default: + /* internal error */ il_log(LOG_ERR, "queue_thread: %s\n", error_get_msg()); - il_log(LOG_DEBUG, " events still waiting\n"); - break; - - case 1: - /* hey, we are done for now */ - il_log(LOG_DEBUG, " all events for %s sent\n", eq->dest_name); - break; - - default: - /* internal error */ - il_log(LOG_ERR, "queue_thread: %s\n", error_get_msg()); - exit = 1; - break; - - } /* switch */ + exit = 1; + break; + + } /* switch */ - /* we are done for now anyway, so close the queue */ - if((ret == 1) && lazy_close) - close_timeout = default_close_timeout; - else { - event_queue_close(eq); - il_log(LOG_DEBUG, " connection to %s:%d closed\n", - eq->dest_name, eq->dest_port); + /* we are done for now anyway, so close the queue */ + if((ret == 1) && lazy_close) + close_timeout = default_close_timeout; + else { + event_queue_close(eq); + il_log(LOG_DEBUG, " connection to %s:%d closed\n", + eq->dest_name, eq->dest_port); + } } } @@ -394,3 +420,7 @@ int event_queue_cond_unlock(struct event_queue *eq) return(0); } + +/* Local Variables: */ +/* c-indentation-style: linux */ +/* End: */ diff --git a/org.glite.lb.logger/src/server_msg.c b/org.glite.lb.logger/src/server_msg.c index 7f23e98..1eff856 100644 --- a/org.glite.lb.logger/src/server_msg.c +++ b/org.glite.lb.logger/src/server_msg.c @@ -11,7 +11,7 @@ static int -create_msg(il_octet_string_t *ev, char **buffer, long *receipt) +create_msg(il_octet_string_t *ev, char **buffer, long *receipt, time_t *expires) { char *p; int len; char *event = ev->data; @@ -56,6 +56,12 @@ create_msg(il_octet_string_t *ev, char **buffer, long *receipt) } #endif + if(p = strstr(event, "DG.EXPIRES")) { + int n; + + p += 11; + *expires = atoi(p); + } len = encode_il_msg(buffer, ev); if(len < 0) { set_error(IL_NOMEM, ENOMEM, "create_msg: out of memory allocating message"); @@ -116,6 +122,7 @@ server_msg_copy(struct server_msg *src) msg->dest_port = src->dest_port; msg->dest = strdup(src->dest); #endif + msg->expires = src->expires; return(msg); } @@ -153,15 +160,16 @@ server_msg_init(struct server_msg *msg, il_octet_string_t *event) msg->job_id_s = edg_wll_NotifIdUnparse(notif_event->notification.notifId); if(notif_event->notification.jobstat && (strlen(notif_event->notification.jobstat) > 0)) { - msg->len = create_msg(event, &msg->msg, &msg->receipt_to); + msg->len = create_msg(event, &msg->msg, &msg->receipt_to, &msg->expires); } + msg->expires = notif_event->notification.expires; edg_wll_FreeEvent(notif_event); free(notif_event); if(msg->len < 0) { return(-1); } #else - msg->len = create_msg(event, &msg->msg, &msg->receipt_to); + msg->len = create_msg(event, &msg->msg, &msg->receipt_to, &msg->expires); if(msg->len < 0) { return(-1); } diff --git a/org.glite.lb.server/Makefile b/org.glite.lb.server/Makefile index 270540e..758789c 100644 --- a/org.glite.lb.server/Makefile +++ b/org.glite.lb.server/Makefile @@ -236,6 +236,9 @@ LIB_OBJS_BK:= \ MONDB_OBJS:=mon-db.o ${LIB_OBJS_BK} MONDB_LIBS:=${COMMON_LIBS} ${LB_UTILS_DB_LIB} ${EXT_LIBS} +MAN_GZ:= glite-lb-bkindex.8.gz glite-lb-bkserverd.8.gz +MAN = $(MAN_GZ:.gz=) + glite-lb-bkserverd: ${NSMAP} ${BKSERVER_OBJS} ${LINKXX} -o $@ ${BKSERVER_OBJS} ${BKSERVER_LIBS} @@ -258,6 +261,10 @@ else endif endif +${MAN_GZ}: ${MAN} + cp $? . + gzip $(notdir $?) + compile: generate glite-lb-bkserverd glite-lb-bkindex glite-lb-mon-db ${STATIC_LIB_BK} ${PLUGIN_LIB} ${MAN_GZ} generate: store.c index.c jp_job_attrs.h @@ -299,7 +306,7 @@ test_xml: test_xml.cpp ${LINKXX} -o $@ test_xml.o lb_xml_parse.o ${LB_COMMON_LIB} ${TEST_LIBS} test.query: test_query_events - ./test_query_events + # XXX coredumps ./test_query_events #query_events_objs:=`echo ${BKSERVER_BASE_OBJS} |sed 's/bkserverd\.o//;s/db_store\.o//;s/stored_master\.o//;s/store\.o//;s/srv_purge\.o//;s/load\.o//;s/dump\.o//;s/lb_proto\.o//;s/lb_html\.o//;s/lb_http\.o//;'` @@ -344,6 +351,8 @@ ${MAN_GZ}: ${MAN} cp $? . gzip -f $(notdir $?) +man: ${MAN_GZ} + stage: compile $(MAKE) install PREFIX=${stagedir} DOSTAGE=yes @@ -353,6 +362,7 @@ install: -mkdir -p ${PREFIX}/share/man/man1 -mkdir -p ${PREFIX}/lib -mkdir -p ${PREFIX}/include/${globalprefix}/${lbprefix} + mkdir -p ${PREFIX}/share/man/man8 ${INSTALL} -m 644 ${top_srcdir}/LICENSE ${PREFIX}/share/doc/${package}-${version} ${INSTALL} -m 644 ${MAN_GZ} ${PREFIX}/share/man/man1 for p in bkserverd bkindex mon-db; do \ @@ -376,9 +386,9 @@ install: ${INSTALL} -m 644 jp_job_attrs.h ${PREFIX}/include/${globalprefix}/${lbprefix} ; \ fi ${INSTALL} -m 644 ${top_srcdir}/interface/srv_perf.h ${PREFIX}/include/${globalprefix}/${lbprefix} - + ${INSTALL} -m 644 ${MAN_GZ} ${PREFIX}/share/man/man8 clean: - rm -rvf *.c *.h *.ch *.xh *.xml *.nsmap *.o *.lo .libs glite-lb-* ${STATIC_LIB_BK} ${PLUGIN_LIB} test* + rm -rvf *.c *.h *.ch *.xh *.xml *.nsmap *.o *.lo .libs glite-lb-* ${STATIC_LIB_BK} ${PLUGIN_LIB} test* ${MAN_GZ} rm -rvf log.xml project/ rpmbuild/ RPMS/ tgz/ %.c: %.c.T diff --git a/org.glite.lb.server/config/startup b/org.glite.lb.server/config/startup index 4ab7f4e..b42a39c 100755 --- a/org.glite.lb.server/config/startup +++ b/org.glite.lb.server/config/startup @@ -58,19 +58,29 @@ start() [ -n "$GLITE_LB_SERVER_PORT" ] && port="-p $GLITE_LB_SERVER_PORT" [ -n "$GLITE_LB_SERVER_WPORT" ] && wport="-w $GLITE_LB_SERVER_WPORT" [ -z "$GLITE_LB_NOTIF_FPREFIX" ] && GLITE_LB_NOTIF_FPREFIX="/var/tmp/glite-lb-notif" + if [ -n "$GLITE_LB_SERVER_DEBUG" ] ; then + if which tscat >/dev/null; then + GLITE_LB_SERVER_OTHER_OPTIONS="$GLITE_LB_SERVER_OTHER_OPTIONS -d -s 1 2>&1 | tscat >> $GLITE_LOCATION_VAR/lb.log &" + GLITE_LB_NOTIF_OTHER_OPTIONS="$GLITE_LB_NOTIF_OTHER_OPTIONS -v -d 2>&1 | tscat >> $GLITE_LOCATION_VAR/notif-il.log &" + else + GLITE_LB_SERVER_OTHER_OPTIONS="$GLITE_LB_SERVER_OTHER_OPTIONS -d -s 1 >> $GLITE_LOCATION_VAR/lb.log 2>&1 &" + GLITE_LB_NOTIF_OTHER_OPTIONS="$GLITE_LB_NOTIF_OTHER_OPTIONS -v -d >> $GLITE_LOCATION_VAR/notif-il.log 2>&1 &" + fi + fi echo -n Starting glite-lb-bkserver ... su - $GLITE_USER -c "$GLITE_LOCATION/bin/glite-lb-bkserverd \ --notif-il-sock=$GLITE_LB_NOTIF_SOCK \ --notif-il-fprefix=$GLITE_LB_NOTIF_FPREFIX \ - $super $creds -i $pidfile $port $wport $dumpdir $purgedir $maildir" \ + $super $creds -i $pidfile $port $wport $dumpdir $purgedir $maildir \ + $GLITE_LB_SERVER_OTHER_OPTIONS" \ && echo " done" || echo " FAILED" if test -x $GLITE_LOCATION/bin/glite-lb-notif-interlogd; then echo -n Starting glite-lb-notif-interlogd ... su - $GLITE_USER -c "$GLITE_LOCATION/bin/glite-lb-notif-interlogd \ -f $GLITE_LB_NOTIF_FPREFIX -s $GLITE_LB_NOTIF_SOCK \ - $creds" && echo " done" || echo " FAILED" + $creds $GLITE_LB_NOTIF_OTHER_OPTIONS" && echo " done" || echo " FAILED" else echo Warning: glite-lb-notif-interlogd not installed, LB notifications will not work fi diff --git a/org.glite.lb.server/doc/glite-lb-bkindex.8 b/org.glite.lb.server/doc/glite-lb-bkindex.8 new file mode 100644 index 0000000..d95a568 --- /dev/null +++ b/org.glite.lb.server/doc/glite-lb-bkindex.8 @@ -0,0 +1,171 @@ +.TH EDG-WL-BKINDEX 8 "May 2003" "EU DataGrid Project" "Logging&Bookkeeping" + +.SH NAME +glite-lb-bkindex - dump or rebuild bookkeeping server indices + +.SH SYNOPSIS +.B glite-lb-bkindex +.RB [\| -mrv\| ] +.RI [ FILE ] +.br +.B glite-lb-bkindex +.RB [\| -dmv\| ] + + +.SH DESCRIPTION +The bookkeeping server +.B glite-lb-bkserverd +uses database indices to support various types of user queries efficiently. +.B glite-lb-bkindex +is a tool for dumping and manipulating the indices. +Both single- and multiple-column indices are supported. + +Due to performance reasons (reindexing may take rather long time) +the default invocation performs only "dry run" - the input is checked and intended actions +reported but not really performed. +The option -r has to be specified to do the real work. + +.B glite-lb-bkindex +should not be run while there is a running +.B glite-lb-bkserverd +on top of the same database. + +.PP +By default the L&B server indexes data according to JobId only. Because +the querying capabilities of L&B release 2 were considerably extended, +the server refuses to process a query which would not utilize any index. +This prevent overloading the underlying database engine. Consequently, even a trivial query 'give me all my jobs' results in an error in the default setup because under certain conditions processing such query may require handling gigabytes of data. + +.PP +The server administrator can create and modify the set of indices and control the set of supported queries. The description of columns to be indexed is given to +.B glite-lb-bkindex +utility via +.I FILE +that follows this syntax (subset of ClassAd syntax): + +.PP +.IP +.I FILE +::= +.BI [\ JobIndices\ =\ {\ index-description\ \fR[\fP ,\ index-description\fR]*\fP +.B } ] +.br +.I index-description +::= +.I column-description \fR|\fP list-of-columns +.br +.I list-of-columns +::= +.BI {\ column-description\ \fR+\fP\ } +.br +.I column-description +::= +.BI [\ column-type ;\ column-name ;\ prefix-len\ \fR?\fP\ ] +.br +.I column-type +::= +.B type = """user""" \fR|\fP type = """system""" \fR|\fP type = """time""" +.br +.I column-name +::= +.B name = """\fIactual_column_name\fP""" +.br +.I prefix-len +::= +.BI prefixlen\ =\ integer + +.PP +The only top-level attribute +.B JobIndices +is a list (possibly empty) of index descriptions. Each index description is either a single column or a list of columns (where the order is important). The column is described by mandatory attributes +.B type +and +.B name\fR,\fP +and an optional attribute +.B prefixlen\fR.\fP +Possible values of type are +.B """system""" +(L&B internal attributes) +.B """time""" +("state entered" timestamps), and +.B """user""" +(user tags). +Currently supported system column names are +.I owner\fR,\fP destination \fRand\fP location\fR.\fP +and valid timestamps are the job states, i.e. +.I submitted, waiting, ready, scheduled, running, done, cleared, cancelled, aborted\fR. +Names of user tags are arbitrary as long as their length is less than 60 characters and they contain only ASCII printable characters excluding backtick (`). +The +.B prefixlen +value may be used to restrict indexing of columns, which may grow rather long, to a fixed size. This becomes necessary with compound indices as MySQL limits the total size of index to 250 bytes only. + +.PP +In the following example the index file contains two indices, the first +one on a single system attribute -- , the second one composed +from system attribute (job) and user tag called : + +.IP +[ +.br +\ \ JobIndices = { +.br +\ \ \ \ [ type = "system"; name = "owner" ], +.br +\ \ \ \ { +.br +\ \ \ \ \ \ [ type = "system"; name = "destination"; +.br +\ \ \ \ \ \ \ \ prefixlen = 100 ], +.br +\ \ \ \ \ \ [ type = "user"; name = "experiment number"; +.br +\ \ \ \ \ \ \ \ prefixlen = 100 ] +.br +\ \ \ \ } +.br +\ \ } +.br +] + +.SH OPTIONS +.TP +.B "-d\fR,\fP --dump" +Print current setup on +.I stdout\fR.\fP + +.TP +.BI \-m " STRING" "\fR,\fP --mysql " STRING +.I STRING +is the database connect string in the form + +USER_NAME/PASSWORD@DB_HOSTNAME:DB_NAME + +It defaults to lbserver/@localhost:lbserver20 (common with +.B glite-lb-bkserverd\fR). + +.TP +.B "-r, --really" +Really perform reindexing. + +.TP +.B "-R, --remove" +Remove all indexes from server. + +.TP +.B "-v, --verbose" +Increase verbosity. + + +.\".SH USAGE +.\" Add any additional description here + +.PP + +.SH REPORTING BUGS +Please, report all bugs to EU DataGrid Bug Tracking System located at http://marianne.in2p3.fr/datagrid/bugzilla + +.SH SEE ALSO +.B glite-lb-purge\fR(8),\fP glite-lb-bkserverd\fR(8) + +.SH AUTHOR +EU DataGrid Work Package 1, CESNET group. diff --git a/org.glite.lb.server/doc/glite-lb-bkserverd.8 b/org.glite.lb.server/doc/glite-lb-bkserverd.8 new file mode 100644 index 0000000..d326ac3 --- /dev/null +++ b/org.glite.lb.server/doc/glite-lb-bkserverd.8 @@ -0,0 +1,300 @@ +.TH EDG-WL-BKSERVERD 8 "May 2003" "EU DataGrid Project" "Logging&Bookkeeping" + +.SH NAME +glite-lb-bkserverd - bookkeeping server daemon + +.SH SYNOPSIS +.B glite-lb-bkserverd +.RI [ options ] +.br + +.SH DESCRIPTION +.B glite-lb-bkserverd +accepts L&B events from interlogger, manages the data storage, +and serves user queries (L&B consumer clients). + +Currently, the data are stored in MySQL database. + +.PP +The main process spawns several +.I slave +processes. Then it listens on +.I port +(default value is 9000) for consumer queries and +.I port+1 +for interlogger daemon requests to store events to database. +Requests are redirected to the slaves +(and picked up by the first free slave process). +All the connection handling and data processing (including SSL handshake) +is performed by the slaves. + +The slaves voluntarily die and are resurrected periodically to workaround +potential resource leakage. + + +.SH OPTIONS +.TP +.BI \-a " HOST:PORT" "\fR,\fP --address " HOST:PORT +Make the server pretend (wrt. data handling) it listens on +.I HOST:PORT +instead of the real listening address. +This is used for debugging and various administrative purposes +(e.g. computing usage statistics on data dumped from another server). + +.TP +.BI \-c " FILE" "\fR,\fP --cert " FILE +Get certificate from +.I FILE\fR.\fP +This option overrides the +.B \fR$\fPX509_USER_CERT +environment variable. + +.TP +.BI \-b " SWITCH" "\fR,\fP --transactions " SWITCH +Set transaction on/off. +.I SWITCH +is true where its value is not 0. + +.TP +.BI \-C " DIR" "\fR,\fP --CAdir " DIR +Trusted CA's certificates directory. +.I DIR\fR.\fP +This option overrides the +.B \fR$\fPX509_CERT_DIR +environment variable. + +.TP +.BI \-V " DIR" "\fR,\fP --VOMSdir " DIR +Trusted VOMS server's certificates directory. +.I DIR\fR.\fP +This option overrides the +.B \fR$\fPX509_VOMS_DIR +environment variable. + +.TP +.B "-d\fR,\fP --debug" +Don't run as daemon, print additional diagnostics. + +.TP +.B "-r\fR,\fP --rgmaexport" +Write state info to RGMA interface. + +.TP +.BI \-i " FILE" "\fR,\fP --pidfile " FILE +Store master pid into +.I FILE +instead of the default pidfile location. +Unless unavoidable it is not recommended to change the pidfile location +as it is also used for locking to prevent starting up multiple server instances +and to generate the semaphore set unique key (see option -l). + + +.TP +.BI \-k " FILE" "\fR,\fP --key " FILE +Get private key from +.I FILE\fR.\fP +This option overrides the +.B \fR$\fPX509_USER_KEY +environment variable. + +.TP +.BI -l " NUM" "\fR,\fP --semaphores " NUM +Use +.I NUM +semaphores (job locks). + +To prevent race conditions on simultaneous event storage and job-state updates +.B glite-lb-bkserverd +uses a set of semaphores to lock jobs. +Jobids are hashed to the semaphores and only one of the set of jobs that map +to a particular semaphore may be updated at any given time. +Therefore +.I NUM +is a limit on potential processing parallelism. + +.I NUM +defaults to the number of slaves (see -s). + + +.TP +.BI -L " NUM:NUM:NUM" "\fR,\fP --limits " NUM:NUM:NUM +Limit the maximum number of returned jobs/events that match user query to +.I NUM:NUM:NUM\fP in format events_limit:jobs_limit:size_limit.\ + +.TP +.BI -N " NUM" "\fR,\fP --notif-dur " NUM +Maximal duration of notification registrations in hours +.I NUM\fR.\fP + +.TP +.BI -S " PREFIX" "\fR,\fP --purge-prefix " PREFIX +Purge files full-path prefix +.I PREFIX\fR.\fP + +.TP +.BI -D " PREFIX" "\fR,\fP --dump-prefix " PREFIX +Dump files full-path prefix +.I PREFIX\fR.\fP + +.TP +.BI -J " DIR" "\fR,\fP --jpreg-dir " DIR +JP registration temporary files prefix (implies '-j'). +.I DIR\fR.\fP + +.TP +.BI "-j \fR,\fP --enable-jpreg-export" +Enable JP registration export (disabled by default). + +.TP +.BI \-m " STRING" "\fR,\fP --mysql " STRING +.I STRING +is a database connect string in the form + +USER_NAME/PASSWORD@DB_HOSTNAME:DB_NAME + +It defaults to lbserver/@localhost:lbserver20. + +.TP +.B "-n, --noauth" +Turn off any authorization. + +.TP +.BI \-p " NUM" "\fR,\fP --port " NUM +Listen on the port +.I NUM\fR.\fP + +.TP +.BI \-w " NUM" "\fR,\fP --wsport " NUM +Set port to serve the web services requests to +.I NUM\fR. + +.TP +.B "-r\fR,\fP --rgmaexport" +Write state info into the RGMA interface. Experimental. + +.TP +.BI -s " NUM" "\fR,\fP --slaves " NUM +Fork +.I NUM +slaves. + +.TP +.BI --super-user " USER" "\fR,\fP --super-user-file " FILE +Bypass query authorization (i.e. return results anyway) when +USER (or users listed in FILE) connect. + +.TP +.BI --no-index " NUM" +Disable checking the indices for superusers (NUM=1) or entirely (NUM=2). + +.TP +.BI --strict-locking +.I (not recommended). +Use a more strict locking strategy -- lock jobs sooner, when events +are stored, not just for status update. +This prevents a non-probable but still possible race +condition with parallel purge but may slow down processing on +heavy loaded servers. Anyway, the only consequence of the race condition +is a little garbage left in the database which is cleaned +form time to time anyway. + +.TP +.BI --notif-il-sock " SOCK +Socket +.I SOCK +to send notifications. + +.TP +.BI --notif-il-sock " PREFIX +File +.I PREFIX +for notifications. + +.TP +.BI --count-statistics " NUM +Count certain statistics on jobs for superusers (NUM=1) or for all (NUM=2). + +.TP +.BI -t " NUM" "\fR,\fP --request-timeout " NUM +Request timeout +.I NUM +for one client. + +.TP +.BI --silent +Do not print diagnostic, even if -d is on. + +.TP +.BI -g \fP --greyjobs +Allow delayed registration (grey jobs), implies --strict-locking. + + + +.\".SH USAGE +.\" Add any additional description here + +.PP + +.SH FILES +.TP +.I /var/run/edg-bkserverd.pid +if running as root (not recommended). + +.TP +.I $HOME/edg-bkserverd.pid +otherwise. + +.TP +File to store pid and to generate semaphores key. + +.TP +No configuration files needed. + +.SH ENVIRONMENT +.TP +.B X509_USER_KEY +If +.B \fR$\fPX509_USER_KEY +is set, it is used to locate private key file. + +.TP +.B X509_USER_CERT +If +.B \fR$\fPX509_USER_CERT +is set, it is used to locate certificate file. + +.TP +.B X509_CERT_DIR +If +.B \fR$\fPX509_CERT_DIR +is set, it is used to locate the trusted CA's certificates and ca-signing-policy files. + +.TP +.B X509_USER_PROXY +If +.B \fR$\fPX509USER_PROXY +is set, it is used to locate proxy certificate file. + +.TP +.B EDG_WL_RGMA_FILE +If +.B \fR$\fPEDG_WL_RGMA_FILE +is set, it is used as name of file for sharing data with RGMA services. + +.TP +.B EDG_WL_RGMA_SOCK +If +.B \fR$\fPEDG_WL_RGMA_SOCK +is set, it is used as name of socket for communication with RGMA. + +.SH BUGS +In a case of a sudden crash, alive slave processes may survive or some resources (IPC locks, open ports) may stay allocated. Please, deal with such problems via standard commands +.B kill\fR and\fP ipcrm\fR.\fP + +Please, report all bugs to EU DataGrid Bug Tracking System located at http://marianne.in2p3.fr/datagrid/bugzilla + +.SH SEE ALSO +.B glite-lb-bkindex\fR(8),\fP glite-lb-purge\fR(8),\fP glite-lb-interlogd\fR(8),\fP glite-lb-logd\fR(8),\fP glite-lb-logevent\fR(1),\fP + +.SH AUTHOR +EU DataGrid Work Package 1, CESNET group. diff --git a/org.glite.lb.server/project/ChangeLog b/org.glite.lb.server/project/ChangeLog new file mode 100644 index 0000000..1d7e3c8 --- /dev/null +++ b/org.glite.lb.server/project/ChangeLog @@ -0,0 +1,20 @@ +1.8.0-1 (lb-1.7.0-1) +- add FQAN-based specification of superusers +- avoid giving no answer to client on some errors during purge (bug #30256) + +1.8.1-1 (lb-1.7.1-1) +- don't propagate errors from sent notifications to IL +- added man pages (not perfect yet) +- allow setting debug flags in startup script +- added run-time dependence on mysql-server +- fixed several memleaks +- ignore shallow branch seqence code for events coming from UI and NS + (fixes impossibility to clear jobs with ReallyRunning events) + +1.8.2-1 (lb-1.7.2-1) +- fix debug output redirection without tscat in startup script +- support notification expiration in interlogger +- implement expired notification purging on server +- rewrite internal event number increment code to avoid race & inifinit loop + with read-consistent transactional database (should fix bug #27555) +- revert enforced "strict locking" with transcactions (not needed anymore) diff --git a/org.glite.lb.server/project/version.properties b/org.glite.lb.server/project/version.properties index 338ed3f..f9ec6c9 100644 --- a/org.glite.lb.server/project/version.properties +++ b/org.glite.lb.server/project/version.properties @@ -1,3 +1,3 @@ # $Header$ -module.version=1.8.0 +module.version=1.8.2 module.age=1 diff --git a/org.glite.lb.server/src/bkserverd.c b/org.glite.lb.server/src/bkserverd.c index 8af2ed8..789f365 100644 --- a/org.glite.lb.server/src/bkserverd.c +++ b/org.glite.lb.server/src/bkserverd.c @@ -727,7 +727,6 @@ int main(int argc, char *argv[]) use_dbcaps = ctx->dbcaps; if (count_statistics) edg_wll_InitStatistics(ctx); - if ((ctx->dbcaps & GLITE_LBU_DB_CAP_TRANSACTIONS)) strict_locking = 1; edg_wll_FreeContext(ctx); if ( !debug ) { diff --git a/org.glite.lb.server/src/il_lbproxy.h b/org.glite.lb.server/src/il_lbproxy.h index 53aa931..7399567 100644 --- a/org.glite.lb.server/src/il_lbproxy.h +++ b/org.glite.lb.server/src/il_lbproxy.h @@ -10,7 +10,7 @@ extern char *lbproxy_ilog_socket_path; extern char *lbproxy_ilog_file_prefix; -int edg_wll_EventSendProxy(edg_wll_Context ctx, const edg_wlc_JobId jobid, const char *event); +int edg_wll_EventSendProxy(edg_wll_Context ctx, glite_jobid_const_t jobid, const char *event); #ifdef __cplusplus } diff --git a/org.glite.lb.server/src/il_notification.c b/org.glite.lb.server/src/il_notification.c index 61d65bf..48fcb8c 100644 --- a/org.glite.lb.server/src/il_notification.c +++ b/org.glite.lb.server/src/il_notification.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "glite/lbu/escape.h" #include "glite/lb/context-int.h" @@ -29,6 +30,7 @@ #define FCNTL_TIMEOUT 1 #define FILE_PREFIX "/tmp/notif_events" #define DEFAULT_SOCKET "/tmp/notif_interlogger.sock" +#define NOTIF_TIMEOUT 1 char *notif_ilog_socket_path = DEFAULT_SOCKET; char *notif_ilog_file_prefix = FILE_PREFIX; @@ -42,6 +44,7 @@ notif_create_ulm( const char *host, const uint16_t port, const char *owner, + int expires, const char *notif_data, char **ulm_data, char **reg_id_s) @@ -65,6 +68,8 @@ notif_create_ulm( event->notification.dest_port = port; if (notif_data) event->notification.jobstat = strdup(notif_data); + event->notification.expires = expires; + if ((*ulm_data = edg_wll_UnparseNotifEvent(context,event)) == NULL) { edg_wll_SetError(context, ret = ENOMEM, "edg_wll_UnparseNotifEvent()"); goto out; @@ -93,9 +98,10 @@ edg_wll_NotifSend(edg_wll_Context context, const char *host, int port, const char *owner, + int expires, const char *notif_data) { - struct timeval timeout; + struct timeval timeout = {NOTIF_TIMEOUT, 0}; int ret; long filepos; char *ulm_data, @@ -107,6 +113,7 @@ edg_wll_NotifSend(edg_wll_Context context, host, port, owner, + expires, notif_data, &ulm_data, ®_id_s))) { @@ -147,6 +154,7 @@ edg_wll_NotifJobStatus(edg_wll_Context context, const char *host, int port, const char *owner, + int expires, const edg_wll_JobStat notif_job_stat) { int ret=0; @@ -160,23 +168,33 @@ edg_wll_NotifJobStatus(edg_wll_Context context, goto out; } - ret=edg_wll_NotifSend(context, reg_id, host, port, owner, xml_esc_data); + if ((ret=edg_wll_NotifSend(context, reg_id, host, port, owner, expires, xml_esc_data))) { + char *ed = NULL, *et = NULL; + + if(ret) edg_wll_UpdateError(context, ret, "edg_wll_NotifJobStatus()"); + edg_wll_Error(context,&et,&ed); + fprintf(stderr,"%s - %s\n", ed, et); + syslog(LOG_INFO,"%s - %s\n", ed, et); + edg_wll_ResetError(context); + free(et); + free(ed); + } out: if(xml_data) free(xml_data); if(xml_esc_data) free(xml_esc_data); - if(ret) edg_wll_UpdateError(context, ret, "edg_wll_NotifJobStatus()"); - return(ret); + return(edg_wll_Error(context,NULL,NULL)); } int -edg_wll_NotifChangeDestination(edg_wll_Context context, +edg_wll_NotifChangeIL(edg_wll_Context context, edg_wll_NotifId reg_id, const char *host, - int port) + int port, + int expires) { - return(edg_wll_NotifSend(context, reg_id, host, port, "", "")); + return(edg_wll_NotifSend(context, reg_id, host, port, "", expires, "")); } @@ -184,6 +202,6 @@ int edg_wll_NotifCancelRegId(edg_wll_Context context, edg_wll_NotifId reg_id) { - return(edg_wll_NotifSend(context, reg_id, NULL, 0, "", "")); + return(edg_wll_NotifSend(context, reg_id, NULL, 0, "", 0, "")); } diff --git a/org.glite.lb.server/src/il_notification.h b/org.glite.lb.server/src/il_notification.h index b008007..78754f0 100644 --- a/org.glite.lb.server/src/il_notification.h +++ b/org.glite.lb.server/src/il_notification.h @@ -46,6 +46,7 @@ edg_wll_NotifSend(edg_wll_Context context, const char *host, int port, const char *owner, + int expires, const char *notif_data); @@ -66,6 +67,7 @@ edg_wll_NotifJobStatus(edg_wll_Context context, const char *host, int port, const char *owner, + int expires, const edg_wll_JobStat notif_job_stat); @@ -77,10 +79,11 @@ edg_wll_NotifJobStatus(edg_wll_Context context, * \see edg_wll_NotifSend() */ int -edg_wll_NotifChangeDestination(edg_wll_Context context, +edg_wll_NotifChangeIL(edg_wll_Context context, edg_wll_NotifId reg_id, const char *host, - int port); + int port, + int expires); /** Cancel registration. * Creates ULM string and uses edg_wll_NotifSend() to pass it to diff --git a/org.glite.lb.server/src/jobstat.c b/org.glite.lb.server/src/jobstat.c index a717fdc..b67a91f 100644 --- a/org.glite.lb.server/src/jobstat.c +++ b/org.glite.lb.server/src/jobstat.c @@ -111,6 +111,7 @@ int edg_wll_JobStatus( if (intErr) { free(md5_jobid); free(string_jobid); + free(jobstat.pub.owner); return edg_wll_Error(ctx,NULL,NULL); } @@ -130,6 +131,7 @@ int edg_wll_JobStatus( } } } + free(jobstat.pub.owner); intErr = edg_wll_LoadIntState(ctx, job, -1 /*all events*/, &ijsp); if (!intErr) { @@ -368,6 +370,7 @@ int edg_wll_intJobStatus( if (edg_wll_QueryEventsServer(ctx,1, (const edg_wll_QueryRec **)jqra, NULL, &events)) { free(string_jobid); free(jqra); + free(intstat->pub.owner); return edg_wll_Error(ctx, NULL, NULL); } free(jqra); @@ -377,6 +380,7 @@ int edg_wll_intJobStatus( if (num_events == 0) { free(string_jobid); + free(intstat->pub.owner); return edg_wll_SetError(ctx,ENOENT,NULL); } diff --git a/org.glite.lb.server/src/jobstat.h b/org.glite.lb.server/src/jobstat.h index 7ce8c07..c2a70d8 100644 --- a/org.glite.lb.server/src/jobstat.h +++ b/org.glite.lb.server/src/jobstat.h @@ -42,6 +42,7 @@ typedef struct _branch_state { char *destination; char *ce_node; char *jdl; + /*!! if adding new field, modify also free_branch_state() */ } branch_state; @@ -57,6 +58,8 @@ typedef struct _intJobStat { struct timeval last_pbs_event_timestamp; int pbs_reruning; // true if rerun event arrived + + /*!! if adding new field, modify also destroy_intJobStat_extension() */ } intJobStat; typedef enum _edg_wll_PBSEventSource { diff --git a/org.glite.lb.server/src/jobstat_supp.c b/org.glite.lb.server/src/jobstat_supp.c index 0ad1adb..199f2a7 100644 --- a/org.glite.lb.server/src/jobstat_supp.c +++ b/org.glite.lb.server/src/jobstat_supp.c @@ -218,15 +218,18 @@ static char **dec_strlist(char *in, char **rest) return NULL; } + /* count number of fields only */ len = 0; tmp_in = in = strchr(in, ' ') + 1 ; do { tmp_ret = dec_string(tmp_in, &tmp_in); + free(tmp_ret); len++; } while (tmp_ret != NULL); out = (char**) malloc(len*sizeof(char*)); + /* get them */ if (out) { len = 0; tmp_in = in; diff --git a/org.glite.lb.server/src/lb_xml_parse.c.T b/org.glite.lb.server/src/lb_xml_parse.c.T index e0efdcb..74e4548 100644 --- a/org.glite.lb.server/src/lb_xml_parse.c.T +++ b/org.glite.lb.server/src/lb_xml_parse.c.T @@ -555,8 +555,7 @@ static void endJobQueryRec(void *data, const char *el UNUSED_VAR) break; } } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } @@ -583,8 +582,7 @@ static void endQueryJobsRequest(void *data, const char *el UNUSED_VAR) &XMLCtx->job_conditions); } } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } @@ -731,8 +729,7 @@ static void endQueryEventsRequest(void *data, const char *el UNUSED_VAR) } } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); } XMLCtx->level--; } @@ -773,8 +770,7 @@ static void endPurgeRequest(void *data, const char *el UNUSED_VAR) } } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } @@ -801,8 +797,7 @@ static void endDumpRequest(void *data, const char *el UNUSED_VAR) free(s); } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } @@ -817,8 +812,7 @@ static void endLoadRequest(void *data, const char *el UNUSED_VAR) XMLCtx->loadRequestGlobal.server_file = edg_wll_from_string_to_string(XMLCtx); } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } @@ -852,8 +846,7 @@ static void endNotifRequest(void *data, const char *el UNUSED_VAR) } } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } @@ -872,8 +865,7 @@ static void endQuerySequenceCodeRequest(void *data, const char *el UNUSED_VAR) } } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } @@ -907,8 +899,7 @@ static void endStatsRequest(void *data, const char *el UNUSED_VAR) } } - XMLCtx->char_buf = NULL; - XMLCtx->char_buf_len = 0; + edg_wll_freeBuf(XMLCtx); XMLCtx->level--; } diff --git a/org.glite.lb.server/src/notif_match.c b/org.glite.lb.server/src/notif_match.c index a50a9db..ef9310d 100644 --- a/org.glite.lb.server/src/notif_match.c +++ b/org.glite.lb.server/src/notif_match.c @@ -29,7 +29,7 @@ int edg_wll_NotifMatch(edg_wll_Context ctx, const edg_wll_JobStat *stat) edg_wll_NotifId nid = NULL; char *jobq,*ju = NULL,*jobc[5]; glite_lbu_Statement jobs = NULL; - int ret,i; + int ret,i,expires; time_t now = time(NULL); char *cond_where = NULL; @@ -75,7 +75,7 @@ int edg_wll_NotifMatch(edg_wll_Context ctx, const edg_wll_JobStat *stat) if (edg_wll_ExecSQL(ctx,jobq,&jobs) < 0) goto err; while ((ret = edg_wll_FetchRow(ctx,jobs,sizeof(jobc)/sizeof(jobc[0]),NULL,jobc)) > 0) { - if (now > glite_lbu_DBToTime(jobc[2])) + if (now > (expires = glite_lbu_DBToTime(jobc[2]))) edg_wll_NotifExpired(ctx,jobc[0]); else if (notif_match_conditions(ctx,stat,jobc[4]) && notif_check_acl(ctx,stat,jobc[3])) @@ -107,7 +107,7 @@ int edg_wll_NotifMatch(edg_wll_Context ctx, const edg_wll_JobStat *stat) /* XXX: only temporary hack!!! */ ctx->p_instance = strdup(""); - if ( edg_wll_NotifJobStatus(ctx, nid, dest, port, jobc[3], *stat) ) + if ( edg_wll_NotifJobStatus(ctx, nid, dest, port, jobc[3], expires, *stat) ) { free(dest); for (i=0; i - component_seqcode(js,EDG_WLL_SOURCE_NETWORK_SERVER); + return ((component_seqcode(es,EDG_WLL_SOURCE_NETWORK_SERVER) > + component_seqcode(js,EDG_WLL_SOURCE_NETWORK_SERVER)) + || + (component_seqcode(es,EDG_WLL_SOURCE_USER_INTERFACE) > + component_seqcode(js,EDG_WLL_SOURCE_USER_INTERFACE))); } @@ -279,7 +283,7 @@ static int processEvent_glite(intJobStat *js, edg_wll_Event *e, int ev_seq, int res = RET_LATE; } -/* new event coming from NS => forget about any resubmission loops */ +/* new event coming from NS or UI => forget about any resubmission loops */ if (e->type != EDG_WLL_EVENT_CANCEL && js->last_seqcode && after_enter_wm(e->any.seqcode,js->last_seqcode)) @@ -971,9 +975,13 @@ int add_stringlist(char ***lptr, const char *new_item) void destroy_intJobStat_extension(intJobStat *p) { - free(p->last_seqcode); p->last_seqcode = NULL; - free(p->last_cancel_seqcode); p->last_cancel_seqcode = NULL; - p->resubmit_type = EDG_WLL_RESUBMISSION_UNDEFINED; + if (p->last_seqcode) free(p->last_seqcode); + if (p->last_cancel_seqcode) free(p->last_cancel_seqcode); + if (p->branch_tag_seqcode) free(p->branch_tag_seqcode); + if (p->last_branch_seqcode) free(p->last_branch_seqcode); + if (p->deep_resubmit_seqcode) free(p->deep_resubmit_seqcode); + free_branch_state(&p->branch_states); + memset(p,0,sizeof(*p)); } void destroy_intJobStat(intJobStat *p) diff --git a/org.glite.lb.server/src/query.c b/org.glite.lb.server/src/query.c index aabe4e6..ce9bd38 100644 --- a/org.glite.lb.server/src/query.c +++ b/org.glite.lb.server/src/query.c @@ -1260,6 +1260,7 @@ int convert_event_head(edg_wll_Context ctx,char **f,edg_wll_Event *e) edg_wll_SetError(ctx,-ret,"edg_wlc_JobIdParse()"); goto err; } + free(f[0]); e->type = atoi(f[1]); free(f[1]); f[1] = NULL; diff --git a/org.glite.lb.server/src/srv_purge.c b/org.glite.lb.server/src/srv_purge.c index 91c05ac..ae7a4aa 100644 --- a/org.glite.lb.server/src/srv_purge.c +++ b/org.glite.lb.server/src/srv_purge.c @@ -297,7 +297,10 @@ int edg_wll_PurgeServer(edg_wll_Context ctx,const edg_wll_PurgeRequest *request) } memset(&stat,0,sizeof stat); - if (edg_wll_JobStatus(ctx,job,0,&stat)) goto abort; /* FIXME: replace by intJobStatus + fix memory leak */ + if (edg_wll_JobStatus(ctx,job,0,&stat)) { /* FIXME: replace by intJobStatus ?? */ + edg_wll_FreeStatus(&stat); + goto abort; + } switch (stat.state) { case EDG_WLL_JOB_CLEARED: @@ -311,8 +314,10 @@ int edg_wll_PurgeServer(edg_wll_Context ctx,const edg_wll_PurgeRequest *request) if (now-stat.lastUpdateTime.tv_sec > timeout[i] && !check_strict_jobid(ctx,job)) { - if (purge_one(ctx,job,dumpfile,request->flags&EDG_WLL_PURGE_REALLY_PURGE)) + if (purge_one(ctx,job,dumpfile,request->flags&EDG_WLL_PURGE_REALLY_PURGE)) { + edg_wll_FreeStatus(&stat); goto abort; + } /* XXX: change with the streaming interface */ if (request->flags & EDG_WLL_PURGE_LIST_JOBS) { @@ -397,6 +402,8 @@ abort: asprintf(&response, "HTTP/1.1 %d %s", ret, edg_wll_HTTPErrorMessage(ret)); edg_wll_http_send(ctx, response, resp_headers, message,ctx->connections->serverConnection); + if (response) free(response); + if (message) free(message); return edg_wll_Error(ctx,NULL,NULL); } diff --git a/org.glite.lb.server/src/store.c.T b/org.glite.lb.server/src/store.c.T index fdddc18..ae5fec6 100644 --- a/org.glite.lb.server/src/store.c.T +++ b/org.glite.lb.server/src/store.c.T @@ -111,8 +111,8 @@ int edg_wll_StoreEvent(edg_wll_Context ctx,edg_wll_Event *e,int *seq) ssrc = edg_wll_SourceToString(e->any.source); /* try to insert (someone else may be doing the same) */ - while (1) { - char *max = NULL, *stamp; + { + char *max = NULL; if (edg_wll_ExecSQL(ctx,select_max,&sh) < 0 || edg_wll_FetchRow(ctx,sh,1,NULL,&max) < 0) @@ -123,7 +123,10 @@ int edg_wll_StoreEvent(edg_wll_Context ctx,edg_wll_Event *e,int *seq) glite_lbu_FreeStmt(&sh); next = max && *max ? atoi(max)+1 : 0; + free(max); + } + while (1) { /* * 1) when using transactions: * Store the whole event right now. @@ -132,6 +135,8 @@ int edg_wll_StoreEvent(edg_wll_Context ctx,edg_wll_Event *e,int *seq) * Store an UNDEF event first in order to prevent race condition * with readers and update event code later. */ + char *stamp = NULL; + glite_lbu_TimeToDB(e->any.timestamp.tv_sec, &stamp); trio_asprintf(&stmt, "insert into events(jobid,event,code,prog,host,time_stamp,usec,arrived,level,userid) " @@ -148,8 +153,8 @@ int edg_wll_StoreEvent(edg_wll_Context ctx,edg_wll_Event *e,int *seq) } else break; /* successful insert */ /* we were late -- try once again */ + next++; free(stmt); - free(max); } free(stmt); stmt = NULL; diff --git a/org.glite.lb.server/test/test_query_events.cpp b/org.glite.lb.server/test/test_query_events.cpp index 3c06fa0..e4e39e8 100644 --- a/org.glite.lb.server/test/test_query_events.cpp +++ b/org.glite.lb.server/test/test_query_events.cpp @@ -55,7 +55,7 @@ void QueryEventsTest::oneJob() job[0].attr = EDG_WLL_QUERY_ATTR_JOBID; job[0].op = EDG_WLL_QUERY_OP_EQUAL ; edg_wlc_JobIdParse("https://lhun.ics.muni.cz:4850/WrCEKje9QTXFiSOZuPMLtw", - &job[0].value.j); + (glite_jobid_t *) &job[0].value.j); job[1].attr = EDG_WLL_QUERY_ATTR_UNDEF; string file(test_dir); @@ -83,7 +83,7 @@ void QueryEventsTest::oneJob() qry_file.close(); CPPUNIT_ASSERT(!edg_wll_QueryEventsServer(ctx,1,jobs,NULL,&events)); - edg_wlc_JobIdFree(job[0].value.j); + edg_wlc_JobIdFree((glite_jobid_t) job[0].value.j); for (i = 0; events[i].type; i++) edg_wll_FreeEvent(&events[i]); free(events); } diff --git a/org.glite.lb.types/events.T b/org.glite.lb.types/events.T index 70ead14..8dc293d 100644 --- a/org.glite.lb.types/events.T +++ b/org.glite.lb.types/events.T @@ -197,6 +197,7 @@ string owner Identification of the job owner (certificate subject). string dest_host Hostname the notification is sent to. port dest_port Port number the notification is sent to. + int expires When the notification expires string jobstat Status of the job (the notification content). diff --git a/org.glite.lb.ws-interface/project/ChangeLog b/org.glite.lb.ws-interface/project/ChangeLog new file mode 100644 index 0000000..b273bc7 --- /dev/null +++ b/org.glite.lb.ws-interface/project/ChangeLog @@ -0,0 +1,2 @@ +2.4.0-2 (lb-1.7.1-1) +- follow changes in lb.build (1.6.0-1 -> 1.6.1-1) diff --git a/org.glite.lb.ws-interface/project/version.properties b/org.glite.lb.ws-interface/project/version.properties index 89b439a..5d0033a 100755 --- a/org.glite.lb.ws-interface/project/version.properties +++ b/org.glite.lb.ws-interface/project/version.properties @@ -1,4 +1,3 @@ -#Fri Sep 02 14:16:49 CEST 2005 -# glite-lb-ws-interface_branch_3_0_0_RC15 tak is taken! -module.version=2.3.0 -module.age=2 +# $Header$ +module.version=2.4.0 +module.age=3 diff --git a/org.glite.lb/project/dependencies.properties b/org.glite.lb/project/dependencies.properties deleted file mode 100644 index 7e279a4..0000000 --- a/org.glite.lb/project/dependencies.properties +++ /dev/null @@ -1,20 +0,0 @@ - -################################################################### -# System dependencies -################################################################### - -org.glite.version = HEAD -org.glite.lb.version = HEAD - -# Component dependencies tag = do not remove this line = -org.glite.lb.client-interface.version = HEAD -org.glite.lb.ws-interface.version = HEAD -org.glite.lb.common.version = HEAD -org.glite.lb.client.version = HEAD -org.glite.lb.server.version = HEAD -org.glite.lb.proxy.version = HEAD -org.glite.lb.server-bones.version = HEAD -org.glite.lb.logger.version = HEAD -org.glite.lb.utils.version = HEAD - -ext.gsoap.version = 2.7.6b diff --git a/org.glite.lb/project/version.properties b/org.glite.lb/project/version.properties index 1822b41..5518786 100644 --- a/org.glite.lb/project/version.properties +++ b/org.glite.lb/project/version.properties @@ -1,4 +1,3 @@ -# $Id$ -# $Name$ -module.version=1.4.5 +# $Header$ +module.version=1.7.2 module.age=2 diff --git a/org.glite.security.gsoap-plugin/Makefile b/org.glite.security.gsoap-plugin/Makefile index 7663601..34590c3 100644 --- a/org.glite.security.gsoap-plugin/Makefile +++ b/org.glite.security.gsoap-plugin/Makefile @@ -68,10 +68,10 @@ CFLAGS:= ${DEBUG} \ LDFLAGS:=${COVERAGE_FLAGS} COMPILE:=libtool --mode=compile ${CC} ${CFLAGS} +COMPILEXX:=libtool --mode=compile ${CXX} ${CFLAGS} LINK:=libtool --mode=link ${CC} -rpath ${stagedir}/lib ${LDFLAGS} -LINKXX:=libtool --mode=link ${CXX} ${LDFLAGS} +LINKXX:=PATH=${top_srcdir}/project/libtoolhack:${PATH} libtool --mode=link ${CXX} -rpath ${stagedir}/lib ${LDFLAGS} INSTALL:=libtool --mode=install install -LINKXX:=libtool --mode=link ${CXX} -rpath ${stagedir}/lib ${LDFLAGS} GSOAP_LIBS:= -L${gsoap_prefix}/lib -lgsoap @@ -110,10 +110,10 @@ ${THRSTATICLIB}: ${THROBJS} ranlib $@ ${LTLIB}: ${OBJS} - ${LINK} ${version_info} -o $@ ${LOBJS} ${EX_LIBS} + PATH=${top_builddir}:${PATH} ${LINKXX} ${version_info} -o $@ ${LOBJS} ${EX_LIBS} ${THRLTLIB}: ${THROBJS} - ${LINK} ${version_info} -o $@ ${THRLOBJS} ${EX_THRLIBS} + PATH=${top_builddir}:${PATH} ${LINKXX} ${version_info} -o $@ ${THRLOBJS} ${EX_THRLIBS} ${STATICLIB_S}: ${OBJS_S} ar crv $@ ${OBJS_S} @@ -124,10 +124,10 @@ ${THRSTATICLIB_S}: ${THROBJS_S} ranlib $@ ${LTLIB_S}: ${OBJS_S} - ${LINK} ${version_info} -o $@ ${LOBJS_S} ${EX_LIBS} + PATH=${top_builddir}:${PATH} ${LINKXX} ${version_info} -o $@ ${LOBJS_S} ${EX_LIBS} ${THRLTLIB_S}: ${THROBJS_S} - ${LINK} ${version_info} -o $@ ${THRLOBJS_S} ${EX_THRLIBS} + PATH=${top_builddir}:${PATH} ${LINKXX} ${version_info} -o $@ ${THRLOBJS_S} ${EX_THRLIBS} all compile: \ ${STATICLIB} ${LTLIB} ${THRSTATICLIB} ${THRLTLIB} \ @@ -162,6 +162,9 @@ link-gsoap: ln -sf ${ext_repository}/${gsoap_name}/${gsoap_version}/src/stdsoap2.[ch] .; \ elif [ -f ${ext_repository}/${gsoap_name}/${gsoap_version}/${gsoap_platform}/stdsoap2.c ]; then \ ln -sf ${ext_repository}/${gsoap_name}/${gsoap_version}/${gsoap_platform}/stdsoap2.[ch] .; \ + elif [ -f ${ext_repository}/${gsoap_name}/${gsoap_version}/${gsoap_platform}/src/stdsoap2.c ]; then \ + ln -sf ${ext_repository}/${gsoap_name}/${gsoap_version}/${gsoap_platform}/src/stdsoap2.c .; \ + ln -sf ${ext_repository}/${gsoap_name}/${gsoap_version}/${gsoap_platform}/include/stdsoap2.h .; \ elif [ -f ${ext_repository}/${gsoap_name}/${gsoap_version}/src/stdsoap2.c ]; then \ ln -sf ${ext_repository}/${gsoap_name}/${gsoap_version}/src/stdsoap2.[ch] .; \ elif [ -f ${ext_repository}/${gsoap_name}/${gsoap_version}/src/${gsoap_srcname}/soapcpp2/stdsoap2.c ]; then \ @@ -212,7 +215,7 @@ install: ${INSTALL} -m 644 ${top_srcdir}/LICENSE ${PREFIX}/share/doc/${package}-${version} cd ${top_srcdir}/interface && ${INSTALL} -m 644 ${HDRS} ${PREFIX}/include/glite/security/ for v in ${gsoap_versions}; do \ - (cd `echo $$v | tr -d .` && ${MAKE} -f ../Makefile install-soaplib gsoap_version=$$v PREFIX=${PREFIX} ); \ + (cd `echo $$v | tr -d .` && ${MAKE} -f ../Makefile install-soaplib gsoap_version=$$v PREFIX=${PREFIX} top_srcdir=${top_srcdir}/..); \ done install-soaplib: @@ -223,6 +226,9 @@ clean: %.o: %.c ${COMPILE} -o $@ -c $< +stdsoap2.o: stdsoap2.c + ${COMPILEXX} ${GLOBUS_INC} -o $@ -c $< + %.thr.o: %.c ${COMPILE} -o $@ -c $< diff --git a/org.glite.security.gsoap-plugin/project/libtoolhack/gcc b/org.glite.security.gsoap-plugin/project/libtoolhack/gcc new file mode 100644 index 0000000..32933cd --- /dev/null +++ b/org.glite.security.gsoap-plugin/project/libtoolhack/gcc @@ -0,0 +1 @@ +g++ "$@" diff --git a/org.glite.security.gsoap-plugin/project/version.properties b/org.glite.security.gsoap-plugin/project/version.properties index 4dae1ef..4bf42a5 100644 --- a/org.glite.security.gsoap-plugin/project/version.properties +++ b/org.glite.security.gsoap-plugin/project/version.properties @@ -1,2 +1,3 @@ -module.version=1.4.2 -module.age=3 +# $Header$ +module.version=1.5.2 +module.age=1 -- 1.8.2.3