* performance testing options to IL

diff --git a/org.glite.lb.logger/Makefile b/org.glite.lb.logger/Makefile
index 95aecb3..5577918 100644 (file)
--- a/org.glite.lb.logger/Makefile
+++ b/org.glite.lb.logger/Makefile
@@ -95,8 +95,8 @@ INTERLOG_OBJS:=il_error.o input_queue_socket.o \
        queue_mgr.o server_msg.o queue_thread.o
 
 INTERLOG_NOBJS:=${INTERLOG_OBJS:.o=.no}
-INTERLOG_PERF_OBJS:=${INTERLOG_OBJS:.o=.po}
-INTERLOG_EMPTY_OBJS:=${INTERLOG_OBJS:.o=.eo}
+INTERLOG_PERF_OBJS:=${INTERLOG_OBJS:.o=.perf.o}
+INTERLOG_EMPTY_OBJS:=${INTERLOG_OBJS:.o=.empty.o}
 #INTERLOG_INLINE_EMPTY_OBJS:=${INTERLOG_OBJS:.o=.io}
 
 INTERLOG_TEST_OBJS:= \
@@ -138,8 +138,8 @@ glite-lb-interlogd: ${INTERLOG_OBJS}
 glite-lb-notif-interlogd: ${INTERLOG_NOBJS}
        ${LINK} -o $@ ${INTERLOG_NOBJS} ${COMMON_LIB}_${thrflavour}
 
-glite-lb-interlogd-perf: ${INTERLOG_OBJS}
-       ${LINK} -o $@ ${INTERLOG_OBJS} ${COMMON_LIB}_${thrflavour}
+glite-lb-interlogd-perf: ${INTERLOG_PERF_OBJS}
+       ${LINK} -o $@ ${INTERLOG_PERF_OBJS} ${COMMON_LIB}_${thrflavour}
 
 glite-lb-interlogd-perf-empty: ${INTERLOG_EMPTY_OBJS}
        ${LINK} -o $@ ${INTERLOG_EMPTY_OBJS} ${COMMON_LIB}_${thrflavour}
@@ -187,6 +187,9 @@ install:
        if [ x${DOSTAGE} = xyes ]; then \
                ${INSTALL} -m 755 ${NOTIF_INTERLOGD} ${PREFIX}/bin; \
        fi
+ifdef LB_PERF
+       ${INSTALL} -m 755 ${top_srcdir}/src/perftest_il.sh ${PREFIX}/sbin
+endif
        ${INSTALL} -m 755 ${top_srcdir}/config/startup ${PREFIX}/etc/init.d/glite-lb-locallogger
        ${INSTALL} -m 644 ${top_srcdir}/LICENSE ${PREFIX}/share/doc/${package}-${version}
 
@@ -196,17 +199,17 @@ ${INTERLOG_NOBJS}: %.no: %.c
 ${INTERLOG_OBJS}: %.o: %.c
        ${CC} ${CFLAGS} ${GLOBUSTHRINC} -c $< -o $@
 
-${INTERLOG_EMPTY_OBJS}: %.eo: %.c
+${INTERLOG_EMPTY_OBJS}: %.empty.o: %.c
        ${CC} ${CFLAGS} ${GLOBUSTHRINC} -DPERF_EMPTY -c $< -o $@
 
-${INTERLOG_PERF_OBJS}: %.po: %.c
+${INTERLOG_PERF_OBJS}: %.perf.o: %.c
        ${CC} ${CFLAGS} ${GLOBUSTHRINC} -c $< -o $@
 
 #${INTERLOG_INLINE_EMPTY_OBJS}: %.io: %.c
 #      ${CC} ${CFLAGS} ${GLOBUSTHRINC} -DLB_PERF -DPERF_EMPTY -DPERF_EVENTS_INLINE -c $< -o $@
 
 ${LOGD_NOBJS}: %.no: %.c
-       ${CC} ${CFLAGS} ${GLOBUSINC} -DLB_PERF -DLOGD_NOFILE -c $< -o $@
+       ${CC} ${CFLAGS} ${GLOBUSINC} -DLOGD_NOFILE -c $< -o $@
 
 ${LOGD_OBJS}: %.o: %.c
        ${CC} ${CFLAGS} ${GLOBUSINC} -c $< -o $@

diff --git a/org.glite.lb.logger/src/il_master.c b/org.glite.lb.logger/src/il_master.c
index c09499d..0b17c01 100644 (file)
--- a/org.glite.lb.logger/src/il_master.c
+++ b/org.glite.lb.logger/src/il_master.c
@@ -327,12 +327,24 @@ handle_msg(il_octet_string_t *event, long offset)
                return(-1);
        msg->es = es;
 
-       ret = event_store_sync(es, offset);
+#ifdef LB_PERF
+       if(nosync) 
+               ret = 1;
+       else 
+               ret = event_store_sync(es, offset);
+#endif
        il_log(LOG_DEBUG, "  syncing event store at %d with event at %d, result %d\n", es->offset, offset, ret);
        if(ret < 0) {
                il_log(LOG_ERR, "    handle_msg: error syncing event store:\n      %s\n", error_get_msg());
-               event_store_release(es);
-               return(0);
+               /* XXX should error during event store recovery cause us to drop the message? */
+               /* Probably no, because the attempt to recover means we have missed some events,
+                  and delivery of this one will not move offset ahead. So try our best and deliver it
+                  even if it may cause duplicates on server. */
+               /* COMMENTED OUT:
+                  server_msg_free(msg);
+                  event_store_release(es);
+                  return(0);
+               */
        } else if(ret == 0) {
                /* we have seen this event already */
                server_msg_free(msg);

diff --git a/org.glite.lb.logger/src/input_queue_socket.c b/org.glite.lb.logger/src/input_queue_socket.c
index 19c81e3..dec64aa 100644 (file)
--- a/org.glite.lb.logger/src/input_queue_socket.c
+++ b/org.glite.lb.logger/src/input_queue_socket.c
@@ -241,7 +241,7 @@ read_event(int sock, long *offset, il_octet_string_t *msg)
  * Returns: -1 on error, 0 if no message available, message length otherwise
  *
  */
-#ifdef PERF_EVENTS_INLINE
+#if 0 && defined(PERF_EVENTS_INLINE)
 int
 input_queue_get(il_octet_string *buffer, long *offset, int timeout)
 {

diff --git a/org.glite.lb.logger/src/interlogd.c b/org.glite.lb.logger/src/interlogd.c
index aff7510..ce5d522 100644 (file)
--- a/org.glite.lb.logger/src/interlogd.c
+++ b/org.glite.lb.logger/src/interlogd.c
@@ -58,6 +58,9 @@ static void usage (int status)
               "  -s, --socket <path>        non-default path of local socket\n"
 #ifdef LB_PERF
               "  -n, --nosend               PERFTEST: consume events instead of sending\n"
+              "  -S, --nosync               PERFTEST: do not check logd files for lost events\n"
+              "  -R, --norecover            PERFTEST: do not start recovery thread\n"
+              "  -P, --noparse              PERFTEST: do not parse messages, use built-in server address\n"
 #ifdef PERF_EVENTS_INLINE
               "  -e, --event_file <file>    PERFTEST: file to read test events from\n"
               "  -j, --njobs <n>            PERFTEST: number of jobs to send\n"
@@ -74,7 +77,7 @@ static int verbose = 0;
 char *file_prefix = DEFAULT_PREFIX;
 int bs_only = 0;
 #ifdef LB_PERF
-int nosend = 0;
+int nosend = 0, norecover=0, nosync=0, noparse=0;
 char *event_source = NULL;
 int njobs = 0;
 #endif
@@ -100,6 +103,9 @@ static struct option const long_options[] =
   {"socket", required_argument, 0, 's'},
 #ifdef LB_PERF
   {"nosend", no_argument, 0, 'n'},
+  {"nosync", no_argument, 0, 'S'},
+  {"norecover", no_argument, 0, 'R'},
+  {"noparse", no_argument, 0, 'P'},
 #ifdef PERF_EVENTS_INLINE
   {"event_file", required_argument, 0, 'e'},
   {"njobs", required_argument, NULL, 'j'},
@@ -132,6 +138,9 @@ decode_switches (int argc, char **argv)
                           "d" /* debug */
 #ifdef LB_PERF
                           "n" /* nosend */
+                          "S" /* nosync */
+                          "R" /* norecover */
+                          "P" /* noparse */
 #ifdef PERF_EVENTS_INLINE
                           "e:" /* event file */
                           "j:" /* num jobs */
@@ -190,6 +199,18 @@ decode_switches (int argc, char **argv)
                nosend = 1;
                break;
 
+       case 'R':
+               norecover = 1;
+               break;
+
+       case 'S':
+               nosync = 1;
+               break;
+
+       case 'P':
+               noparse = 1;
+               break;
+
 #ifdef PERF_EVENTS_INLINE
        case 'e':
                event_source = strdup(optarg);
@@ -306,6 +327,14 @@ main (int argc, char **argv)
 
 #ifndef PERF_EMPTY
   /* find all unsent events waiting in files */
+#ifdef LB_PERF
+  if(norecover) {
+         if(event_store_init(file_prefix) < 0) {
+                 il_log(LOG_CRIT, "Failed to init event stores: %s\n", error_get_msg());
+                 exit(EXIT_FAILURE);
+         }
+  } else
+#endif
   { 
          pthread_t rid;
 

diff --git a/org.glite.lb.logger/src/interlogd.h b/org.glite.lb.logger/src/interlogd.h
index 846d9ea..656e18d 100644 (file)
--- a/org.glite.lb.logger/src/interlogd.h
+++ b/org.glite.lb.logger/src/interlogd.h
@@ -71,7 +71,7 @@ extern char *CAcert_dir;
 extern int bs_only;
 extern int killflg;
 #ifdef LB_PERF
-extern int nosend;
+extern int nosend, nosync, norecover, noparse;
 #ifdef PERF_EVENTS_INLINE
 extern char *event_source;
 #endif

diff --git a/org.glite.lb.logger/src/perftest_il.sh b/org.glite.lb.logger/src/perftest_il.sh
index e9562f9..32dbaf6 100644 (file)
--- a/org.glite.lb.logger/src/perftest_il.sh
+++ b/org.glite.lb.logger/src/perftest_il.sh
@@ -14,59 +14,90 @@ DEBUG=${DEBUG:-0}
 
 check_test_files || exit 1
 
+
+echo "-------------------------------------------"
+echo "Logging test:"
+echo "  - events sent through IPC and/or files"
+echo "  - events discarded by IL immediately"
+echo "-------------------------------------------"
+
+echo -e "\tsmall_job \t big_job \t small_dag \t big_dag"
+
+
 PERFTEST_CONSUMER=$STAGEDIR/bin/glite-lb-interlogd-perf-empty
-CONSUMER_ARGS="-d -v"
+CONSUMER_ARGS="-d"
+
+LOGJOBS_ARGS="--nofile"
+echo "Only IPC"
+run_test il $numjobs
+print_result
+LOGJOBS_ARGS=""
+echo "IPC & files"
+run_test il $numjobs
+print_result
+rm -f /tmp/dglogd.log.*
+
+echo "--------------------------------"
+echo "Interlogger test:"
+echo "  - events sent through IPC only"
+echo "  - events discarded in IL"
+echo "--------------------------------"
+echo -e "\tsmall_job \t big_job \t small_dag \t big_dag"
+
+PERFTEST_CONSUMER=$STAGEDIR/bin/glite-lb-interlogd-perf
+LOGJOBS_ARGS="--nofile"
+
+CONSUMER_ARGS="-d --nosend --noparse"
+echo "No event parsing"
+run_test il $numjobs
+print_result
+
+CONSUMER_ARGS="-d --nosend --nosync"
+echo "No checking of event files"
+run_test il $numjobs
+print_result
+
+CONSUMER_ARGS="-d --nosend --norecover"
+echo "No recovery thread"
+run_test il $numjobs
+print_result
 
+CONSUMER_ARGS="-d --nosend"
+echo "Normal operation:"
+run_test il $numjobs
+print_result
+
+echo "-----------------------------------"
+echo "Interlogger test:"
+echo "  - events sent through IPC & files"
+echo "  - events discarded in IL"
+echo "-----------------------------------"
 echo -e "\tsmall_job \t big_job \t small_dag \t big_dag"
+
+PERFTEST_CONSUMER=$STAGEDIR/bin/glite-lb-interlogd-perf
+LOGJOBS_ARGS=""
+
+CONSUMER_ARGS="-d --nosend --noparse"
+echo "No event parsing"
 run_test il $numjobs
-j=0
-while [[ $j -lt 4 ]]
-do
-    echo -e -n "\t ${PERFTEST_THROUGHPUT[$j]}" 
-    j=$((j+1))
-done
-echo ""
-#j=0
-#while [[ $j -lt 4 ]]
-#do
-#    echo -e -n "\t (${PERFTEST_EV_THROUGHPUT[$j]})"   
-#    j=$((j+1))
-#done
-#echo ""
-
-
-#
-#    dst=il
-#
-## i)1)
-#
-#    glite_lb_interlogd_perf_noparse --nosend
-#    run_test()
-#
-#    glite_lb_interlogd_perf_nosync --nosend
-#    run_test()
-#
-#    glite_lb_interlogd_perf_norecover --nosend
-#    run_test()
-#
-#    glite_lb_interlogd_perf --nosend
-#    run_test()
-#
-## ii)1)
-#
-#glite_lb_bkserverd_perf_empty
-#
-#    glite_lb_interlogd_perf_noparse
-#    run_test()
-#
-#    glite_lb_interlogd_perf_nosync
-#    run_test()
-#    
-#    glite_lb_interlogd_perf_norecover
-#    run_test()
-#
-#    glite_lb_interlogd_perf_lazy
-#    run_test()
-#
-#    glite_lb_interlogd_perf
-#    run_test()
+print_result
+rm -f /tmp/dglogd.log.*
+
+CONSUMER_ARGS="-d --nosend --nosync"
+echo "No checking of event files"
+run_test il $numjobs
+print_result
+rm -f /tmp/dglogd.log.*
+
+CONSUMER_ARGS="-d --nosend --norecover"
+echo "No recovery thread"
+run_test il $numjobs
+print_result
+rm -f /tmp/dglogd.log.*
+
+CONSUMER_ARGS="-d --nosend"
+echo "Normal operation:"
+run_test il $numjobs
+print_result
+rm -f /tmp/dglogd.log.*
+

diff --git a/org.glite.lb.logger/src/send_event.c b/org.glite.lb.logger/src/send_event.c
index dc33757..cc2a55f 100644 (file)
--- a/org.glite.lb.logger/src/send_event.c
+++ b/org.glite.lb.logger/src/send_event.c
@@ -279,6 +279,7 @@ event_queue_send(struct event_queue *eq)
     } else {
            glite_wll_perftest_consumeEventIlMsg(msg->msg+17);
            code = LB_OK;
+           rep = strdup("OK");
     }
 #endif
     

diff --git a/org.glite.lb.logger/src/server_msg.c b/org.glite.lb.logger/src/server_msg.c
index b7a005b..51c8911 100644 (file)
--- a/org.glite.lb.logger/src/server_msg.c
+++ b/org.glite.lb.logger/src/server_msg.c
@@ -166,7 +166,12 @@ server_msg_init(struct server_msg *msg, il_octet_string_t *event)
        if(msg->len < 0) {
                return(-1);
        }
-       msg->job_id_s = edg_wll_GetJobId(event->data);
+#ifdef LB_PERF
+       if(noparse) {
+               msg->job_id_s = strdup("https://localhost:9001/not_so_unique_string");
+       } else 
+#endif
+               msg->job_id_s = edg_wll_GetJobId(event->data);
 #endif
        /* remember to add event separator to the length */
        msg->ev_len = event->len + 1;