From: Jiří Filipovič Date: Thu, 16 Feb 2012 10:34:25 +0000 (+0000) Subject: implementation of first server statistics X-Git-Tag: gridsite-core_R_1_7_17~93 X-Git-Url: http://scientific.zcu.cz/git/?a=commitdiff_plain;h=aa5d9bab869dad3efd7c300f371ba8d40a53b1de;p=jra1mw.git implementation of first server statistics --- diff --git a/org.glite.lb.server/Makefile b/org.glite.lb.server/Makefile index 334a05b..764151d 100644 --- a/org.glite.lb.server/Makefile +++ b/org.glite.lb.server/Makefile @@ -169,7 +169,7 @@ BKSERVER_BASE_OBJS:= \ lb_xml_parse_V21.o lb_text.o \ openserver.o query.o userjobs.o db_store.o request.o store.o \ stored_master.o srv_purge.o server_state.o dump.o lb_authz.o load.o \ - notification.o il_notification.o notif_match.o stats.o db_calls.o db_supp.o lb_rss.o pretty_print_wrapper.o \ + notification.o il_notification.o notif_match.o stats.o server_stats.o db_calls.o db_supp.o lb_rss.o pretty_print_wrapper.o \ policy_gram.o policy_lex.o authz_policy.o crypto.o gsoap_version ?= ${gsoap_default_version} @@ -213,7 +213,7 @@ endif INDEX_OBJS:= index.o index_parse.o jobstat_supp.o openserver.o \ jobstat.o query.o get_events.o write2rgma.o index_lex.o \ - lb_authz.o store.o bkindex.o stats.o\ + lb_authz.o store.o bkindex.o stats.o server_stats.o\ request.o db_store.o srv_purge.o notif_match.o il_lbproxy.o dump.o lb_xml_parse.o il_notification.o lb_proto.o lb_text.o server_state.o lb_xml_parse_V21.o lb_html.o cond_dump.o notification.o seqcode.o userjobs.o load.o db_calls.o db_supp.o lb_rss.o pretty_print_wrapper.o authz_policy.o crypto.o INDEX_LIBS:= ${SRVBONES_LIB} ${COMMON_LIBS} ${LB_MACHINE_LIB} ${EXT_LIBS} ${LB_UTILS_DB_LIB} @@ -233,7 +233,7 @@ LIB_OBJS_BK:= \ lb_xml_parse_V21.o lb_text.o \ openserver.o query.o userjobs.o db_store.o request.o store.o \ stored_master.o srv_purge.o server_state.o dump.o lb_authz.o load.o \ - notification.o il_notification.o notif_match.o stats.o write2rgma.o \ + notification.o il_notification.o notif_match.o stats.o server_stats.o write2rgma.o \ db_calls.o db_supp.o lb_rss.o pretty_print_wrapper.o authz_policy.o \ crypto.o diff --git a/org.glite.lb.server/src/bkserverd.c b/org.glite.lb.server/src/bkserverd.c index c6b9055..8740840 100644 --- a/org.glite.lb.server/src/bkserverd.c +++ b/org.glite.lb.server/src/bkserverd.c @@ -72,6 +72,7 @@ enum lb_srv_perf_sink sink_mode; #include "lb_authz.h" #include "il_notification.h" #include "stats.h" +#include "server_stats.h" #include "db_calls.h" #include "db_supp.h" #include "openserver.h" @@ -142,8 +143,10 @@ int rgma_export = 0; static int noAuth = 0; static int noIndex = 0; static int strict_locking = 0; -static int greyjobs = 0; -static int count_statistics = 1; +static int greyjobs = 0; +static int count_statistics = 1; +static int count_server_stats = 1; +static char *server_stats_file = NULL; static int hardJobsLimit = 0; static int hardEventsLimit = 0; static int hardRespSizeLimit = 0; @@ -215,6 +218,8 @@ static struct option opts[] = { {"notif-il-sock", 1, NULL, 'X'}, {"notif-il-fprefix", 1, NULL, 'Y'}, {"count-statistics", 1, NULL, 'T'}, + {"count-server-stats", 1, NULL, 'e'}, + {"server-stats-file", 1, NULL, 'f'}, {"request-timeout", 1, NULL, 't'}, #ifdef LB_PERF {"perf-sink", 1, NULL, 'K'}, @@ -234,7 +239,7 @@ static struct option opts[] = { {NULL,0,NULL,0} }; -static const char *get_opt_string = "Ac:k:C:V:p:a:drm:ns:i:S:D:J:jR:F:xOL:N:X:Y:T:t:zb:gPBo:q:W:Z:GI:l:E" +static const char *get_opt_string = "Ac:k:C:V:p:a:drm:ns:i:S:D:J:jR:F:xOL:N:X:Y:T:t:e:f:zb:gPBo:q:W:Z:GI:l:E" #ifdef GLITE_LB_SERVER_WITH_WS "w:" #endif @@ -277,6 +282,10 @@ static void usage(char *me) "\t--notif-il-fprefix\t file prefix for notifications\n" "\t--count-statistics=1\t count certain statistics on jobs\n" "\t =2\t ... and allow anonymous access\n" + "\t--count-server-stats=0\t do not count server statistics\n" + "\t =1\t count server statistics (default)\n" + "\t =2\t count server statistics, restrict access only to superusers\n" + "\t--server-stats-file path to file where server statistics are persistently stored\n" "\t-t, --request-timeout\t request timeout for one client\n" #ifdef LB_PERF "\t-K, --perf-sink\t where to sink events\n" @@ -477,6 +486,10 @@ int main(int argc, char *argv[]) break; case 'T': count_statistics = atoi(optarg); break; + case 'e': count_server_stats = atoi(optarg); + break; + case 'f': server_stats_file = strdup(optarg); + break; case 't': request_timeout = atoi(optarg); break; #ifdef LB_PERF @@ -795,6 +808,9 @@ int main(int argc, char *argv[]) use_dbcaps = ctx->dbcaps; if (count_statistics) edg_wll_InitStatistics(ctx); + + if (count_server_stats) edg_wll_InitServerStatistics(ctx, server_stats_file); + edg_wll_FreeContext(ctx); if ( !debug ) { @@ -1623,6 +1639,8 @@ int bk_accept_ws(int conn, struct timeval *timeout, void *cdata) ctx->processRequest_cb = NULL; } + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_WS_QUERIES); + return ENOTCONN; } #endif /* GLITE_LB_SERVER_WITH_WS */ diff --git a/org.glite.lb.server/src/jobstat.c b/org.glite.lb.server/src/jobstat.c index 0a04d1b..b7de13e 100644 --- a/org.glite.lb.server/src/jobstat.c +++ b/org.glite.lb.server/src/jobstat.c @@ -42,6 +42,7 @@ limitations under the License. #include "db_supp.h" #include "db_calls.h" #include "authz_policy.h" +#include "server_stats.h" #define DAG_ENABLE 1 @@ -503,6 +504,34 @@ int edg_wll_intJobStatus( if ((!maxts.tv_sec && !maxts.tv_usec) || (ts.tv_sec > maxts.tv_sec) || (ts.tv_sec == maxts.tv_sec && ts.tv_usec > maxts.tv_usec)) maxts = ts; + // event has been processed by state machine, refresh statistics + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_JOB_EVENTS); + // check if this is job registration (counted only here) + if (events[i].type == EDG_WLL_EVENT_REGJOB){ + switch (intstat->pub.jobtype) { + case EDG_WLL_STAT_SIMPLE: + case EDG_WLL_STAT_DAG: + case EDG_WLL_STAT_COLLECTION: + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_GLITEJOB_REGS); + break; + case EDG_WLL_STAT_PBS: + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_PBSJOB_REGS); + break; + case EDG_WLL_STAT_CONDOR: + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_CONDOR_REGS); + break; + case EDG_WLL_STAT_CREAM: + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_CREAM_REGS); + break; + case EDG_WLL_STAT_FILE_TRANSFER: + case EDG_WLL_STAT_FILE_TRANSFER_COLLECTION: + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_SANDBOX_REGS); + break; + default: + glite_common_log(LOG_CATEGORY_LB_SERVER_REQUEST, LOG_PRIORITY_DEBUG, "Unknown job type, registration will not be counted in statistics."); + break; + } + } } /* no events or status computation error */ if (intJobStat_to_JobStat(intstat)->state == EDG_WLL_JOB_UNDEF) { @@ -1238,6 +1267,9 @@ edg_wll_ErrorCode edg_wll_StepIntStateParent(edg_wll_Context ctx, edg_wll_UpdateStatistics(ctx, oldstat, e, intJobStat_to_JobStat(ijsp)); + // event has been processed by state machine, refresh statistics + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_JOB_EVENTS); + if (ctx->rgma_export) write2rgma_chgstatus(ijsp, oldstat_rgmaline); if (stat_out) { @@ -1287,6 +1319,9 @@ edg_wll_ErrorCode edg_wll_StepIntState(edg_wll_Context ctx, edg_wll_UpdateStatistics(ctx, oldstat, e, intJobStat_to_JobStat(ijsp)); + // event has been processed by state machine, refresh statistics + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_JOB_EVENTS); + /* check whether subjob state change does not change parent state */ if ((intJobStat_to_JobStat(ijsp)->parent_job) && (oldstat->state != intJobStat_to_JobStat(ijsp)->state)) { diff --git a/org.glite.lb.server/src/lb_html.c b/org.glite.lb.server/src/lb_html.c index c48e14d..9f90c25 100644 --- a/org.glite.lb.server/src/lb_html.c +++ b/org.glite.lb.server/src/lb_html.c @@ -21,6 +21,8 @@ limitations under the License. #include "lb_proto.h" #include "cond_dump.h" #include "pretty_print_wrapper.h" +#include "server_stats.h" +#include "authz_policy.h" #include "glite/lb/context-int.h" #include "glite/lb/xml_conversions.h" @@ -504,6 +506,87 @@ int edg_wll_WSDLOutput(edg_wll_Context ctx UNUSED_VAR, char **message, char *fil return 0; } +int edg_wll_StatisticsToHTML(edg_wll_Context ctx, char **message) { + char *out; + + struct _edg_wll_GssPrincipal_data princ; + memset(&princ, 0, sizeof princ); + princ.name = ctx->peerName; + princ.fqans = ctx->fqans; + if (ctx->count_server_stats == 2 && !ctx->noAuth && !check_authz_policy(&ctx->authz_policy, &princ, ADMIN_ACCESS)) + { + asprintf(&out,"

LB Server Usage Statistics

\n" + "Only superusers can view server usage statistics on this particular server.\n"); + } + else + { + char* times[SERVER_STATISTICS_COUNT]; + int i; + for (i = 0; i < SERVER_STATISTICS_COUNT; i++) + if (edg_wll_ServerStatisticsGetStart(ctx, i)) + times[i] = strdup((const char*)ctime(edg_wll_ServerStatisticsGetStart(ctx, i))); + else + times[i] = 0; + + asprintf(&out, + "

LB Server Usage Statistics

\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "
VariableValueMeasured from
gLite job regs%i%s
PBS job regs%i%s
Condor job regs%i%s
CREAM job regs%i%s
Sandbox regs%i%s
Notification regs (legacy interface)%i%s
Notification regs (msg interface)%i%s
Job events%i%s
Notifications sent (legacy)%i%s
Notifications sent (msg)%i%s
L&B protocol accesses%i%s
WS queries%i%s
HTML accesses%i%s
Plain text accesses%i%s
RSS accesses%i%s
\n", + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_GLITEJOB_REGS), + times[SERVER_STATS_GLITEJOB_REGS], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_PBSJOB_REGS), + times[SERVER_STATS_PBSJOB_REGS], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_CONDOR_REGS), + times[SERVER_STATS_CONDOR_REGS], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_CREAM_REGS), + times[SERVER_STATS_CREAM_REGS], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_SANDBOX_REGS), + times[SERVER_STATS_SANDBOX_REGS], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_NOTIF_LEGACY_REGS), + times[SERVER_STATS_NOTIF_LEGACY_REGS], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_NOTIF_MSG_REGS), + times[SERVER_STATS_NOTIF_MSG_REGS], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_JOB_EVENTS), + times[SERVER_STATS_JOB_EVENTS], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_NOTIF_LEGACY_SENT), + times[SERVER_STATS_NOTIF_LEGACY_SENT], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_NOTIF_MSG_SENT), + times[SERVER_STATS_NOTIF_MSG_SENT], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_LBPROTO), + times[SERVER_STATS_LBPROTO], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_WS_QUERIES), + times[SERVER_STATS_WS_QUERIES], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_HTML_VIEWS), + times[SERVER_STATS_HTML_VIEWS], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_TEXT_VIEWS), + times[SERVER_STATS_TEXT_VIEWS], + edg_wll_ServerStatisticsGetValue(ctx, SERVER_STATS_RSS_VIEWS), + times[SERVER_STATS_RSS_VIEWS] + ); + + for (i = 0; i < SERVER_STATISTICS_COUNT; i++) + free(times[i]); + } + + *message = out; +} + char *edg_wll_ErrorToHTML(edg_wll_Context ctx,int code) { char *out,*et,*ed; diff --git a/org.glite.lb.server/src/lb_html.h b/org.glite.lb.server/src/lb_html.h index faea17d..a1bb1f7 100644 --- a/org.glite.lb.server/src/lb_html.h +++ b/org.glite.lb.server/src/lb_html.h @@ -33,5 +33,6 @@ int edg_wll_UserNotifsToHTML(edg_wll_Context ctx, char **notifids, char **messag int edg_wll_NotificationToHTML(edg_wll_Context ctx, notifInfo *ni, char **message); char *edg_wll_ErrorToHTML(edg_wll_Context,int); int edg_wll_FileTransferStatusToHTML(edg_wll_Context ctx, edg_wll_JobStat stat, char **message); +int edg_wll_StatisticsToHTML(edg_wll_Context ctx, char **message); #endif /* GLITE_LB_HTML_H */ diff --git a/org.glite.lb.server/src/lb_proto.c b/org.glite.lb.server/src/lb_proto.c index 3460791..96fb612 100644 --- a/org.glite.lb.server/src/lb_proto.c +++ b/org.glite.lb.server/src/lb_proto.c @@ -41,6 +41,7 @@ limitations under the License. #include "lb_html.h" #include "lb_rss.h" #include "stats.h" +#include "server_stats.h" #include "jobstat.h" #include "get_events.h" #include "purge.h" @@ -679,10 +680,14 @@ edg_wll_ErrorCode edg_wll_Proto(edg_wll_Context ctx, flags = (requestPTR[1]=='?') ? edg_wll_string_to_stat_flags(requestPTR + 2) : 0; switch (edg_wll_UserJobsServer(ctx, EDG_WLL_STAT_CHILDREN, &jobsOut, &statesOut)) { - case 0: if (text) + case 0: if (text){ edg_wll_UserInfoToText(ctx, jobsOut, &message); - else if (html) + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_TEXT_VIEWS); + } + else if (html){ edg_wll_UserInfoToHTML(ctx, jobsOut, statesOut, &message); + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_HTML_VIEWS); + } else ret = HTTP_OK; break; case ENOENT: ret = HTTP_NOTFOUND; break; @@ -732,9 +737,12 @@ edg_wll_ErrorCode edg_wll_Proto(edg_wll_Context ctx, ret = HTTP_BADREQ; } else switch (edg_wll_JobStatusServer(ctx,jobId,EDG_WLL_STAT_CLASSADS | EDG_WLL_STAT_CHILDREN,&stat)) { - case 0: if (text) + case 0: if (text) { edg_wll_JobStatusToText(ctx,stat,&message); - else if (html) + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_TEXT_VIEWS); + } + else if (html) { + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_HTML_VIEWS); switch(stat.jobtype){ case EDG_WLL_STAT_CREAM: edg_wll_CreamJobStatusToHTML(ctx,stat,&message); @@ -748,6 +756,7 @@ edg_wll_ErrorCode edg_wll_Proto(edg_wll_Context ctx, edg_wll_GeneralJobStatusToHTML(ctx,stat,&message); break; } + } else ret = HTTP_OK; break; @@ -772,10 +781,14 @@ edg_wll_ErrorCode edg_wll_Proto(edg_wll_Context ctx, char *userid = strmd5(can_peername, NULL); getUserNotifications(ctx, userid, ¬ifids); free(can_peername); - if (text) + if (text) { edg_wll_UserNotifsToText(ctx, notifids, &message); - else if (html) + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_TEXT_VIEWS); + } + else if (html) { edg_wll_UserNotifsToHTML(ctx, notifids, &message); + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_HTML_VIEWS); + } else ret = HTTP_OK; /*GET /NOTIF:[notifId]: Notification info*/ @@ -792,10 +805,14 @@ edg_wll_ErrorCode edg_wll_Proto(edg_wll_Context ctx, } free(pomCopy); - if (text) + if (text) { edg_wll_NotificationToText(ctx, &ni, &message); - else + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_TEXT_VIEWS); + } + else { edg_wll_NotificationToHTML(ctx, &ni, &message); + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_HTML_VIEWS); + } freeNotifInfo(&ni); @@ -823,13 +840,16 @@ edg_wll_ErrorCode edg_wll_Proto(edg_wll_Context ctx, ret = HTTP_NOTFOUND; edg_wll_SetError(ctx, ENOENT, "current index configuration does not support RSS feeds"); } - else + else { edg_wll_RSSFeed(ctx, states, requestPTR, &message); + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_RSS_VIEWS); + } /*GET /?wsdl */ #define WSDL_LB "LB.wsdl" #define WSDL_LBTYPES "LBTypes.wsdl" #define WSDL_LB4AGU "lb4agu.wsdl" } else if (strncmp(requestPTR, "/?wsdl", strlen("/?wsdl")) == 0) { + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_TEXT_VIEWS); char *filename; asprintf(&filename, "%s/" WSDL_PATH "/%s", glite_location(), WSDL_LB); if (edg_wll_WSDLOutput(ctx, &message, filename)) @@ -837,6 +857,7 @@ edg_wll_ErrorCode edg_wll_Proto(edg_wll_Context ctx, free(filename); /* GET /?types */ } else if (strncmp(requestPTR, "/?types", strlen("/?types")) == 0) { + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_TEXT_VIEWS); char *filename; asprintf(&filename, "%s/" WSDL_PATH "/%s", glite_location(), WSDL_LBTYPES); if (edg_wll_WSDLOutput(ctx, &message, filename)) @@ -844,6 +865,7 @@ edg_wll_ErrorCode edg_wll_Proto(edg_wll_Context ctx, free(filename); /* GET /?agu */ } else if (strncmp(requestPTR, "/?agu", strlen("/?agu")) == 0) { + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_TEXT_VIEWS); char *filename; asprintf(&filename, "%s/" WSDL_PATH "/%s", glite_location(), WSDL_LB4AGU); if (edg_wll_WSDLOutput(ctx, &message, filename)) @@ -851,11 +873,17 @@ edg_wll_ErrorCode edg_wll_Proto(edg_wll_Context ctx, free(filename); /* GET /?version */ } else if (strncmp(requestPTR, "/?version", strlen("/?version")) == 0) { + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_TEXT_VIEWS); asprintf(&message, "%s", VERSION); /* GET /?configuration*/ } else if (strncmp(requestPTR, "/?configuration", strlen("/?configuration")) == 0) { // also browser-readable HTML version here? edg_wll_ConfigurationToText(ctx, &message); + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_TEXT_VIEWS); + /* GET /?stats*/ + } else if (strncmp(requestPTR, "/?stats", strlen("/?stats")) == 0) { + edg_wll_StatisticsToHTML(ctx, &message); + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_HTML_VIEWS); /* GET [something else]: not understood */ } else ret = HTTP_BADREQ; free(requestPTR); requestPTR = NULL; @@ -1344,6 +1372,9 @@ edg_wll_ErrorCode edg_wll_Proto(edg_wll_Context ctx, /* POST [something else]: not understood */ else ret = HTTP_BADREQ; + if (ret != HTTP_BADREQ) + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_LBPROTO); + free(requestPTR); requestPTR = NULL; /* other HTTP methods */ diff --git a/org.glite.lb.server/src/notif_match.c b/org.glite.lb.server/src/notif_match.c index 2554332..c663e83 100644 --- a/org.glite.lb.server/src/notif_match.c +++ b/org.glite.lb.server/src/notif_match.c @@ -38,6 +38,7 @@ limitations under the License. #include "index.h" #include "authz_policy.h" #include "get_events.h" +#include "server_stats.h" static int notif_match_conditions(edg_wll_Context,const edg_wll_JobStat *,const edg_wll_JobStat *,const char *, int flags); static edg_wll_Event* fetch_history(edg_wll_Context ctx, edg_wll_JobStat *stat); @@ -197,6 +198,11 @@ int edg_wll_NotifMatch(edg_wll_Context ctx, const edg_wll_JobStat *oldstat, cons goto err; } } + + if (! strncmp(jobc[1], "x-msg", 5)) + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_NOTIF_MSG_SENT); + else + edg_wll_ServerStatisticsIncrement(ctx, SERVER_STATS_NOTIF_LEGACY_SENT); for (i=0; i