From 0b07fa0efc915e227d9340c11a58dd1dcd28b9da Mon Sep 17 00:00:00 2001 From: =?utf8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= Date: Thu, 16 Nov 2006 20:56:25 +0000 Subject: [PATCH] update docs and tests various fixes query jobid feature --- org.glite.jp.index/Makefile | 2 +- org.glite.jp.index/config/glite-jpis-config.xml | 1 + org.glite.jp.index/config/startup | 2 +- org.glite.jp.index/doc/glite-jp-indexd.sgml | 17 ++++-- org.glite.jp.index/examples/jpis-client.c | 5 ++ org.glite.jp.index/examples/jpis-test.c | 6 +- .../examples/query-tests/complex_query.out | 2 +- org.glite.jp.index/examples/query-tests/dump1.sql | 4 +- .../examples/query-tests/jobid_query.in | 19 +++++++ .../examples/query-tests/jobid_query.out | 13 +++++ .../examples/query-tests/run-test.sh | 18 ++++-- .../examples/query-tests/simple_query.out | 2 +- org.glite.jp.index/src/context.c | 1 + org.glite.jp.index/src/db_ops.c | 4 +- org.glite.jp.index/src/soap_ops.c | 66 ++++++++++++---------- 15 files changed, 112 insertions(+), 50 deletions(-) create mode 100644 org.glite.jp.index/examples/query-tests/jobid_query.in create mode 100644 org.glite.jp.index/examples/query-tests/jobid_query.out diff --git a/org.glite.jp.index/Makefile b/org.glite.jp.index/Makefile index 5128c9d..7a53618 100644 --- a/org.glite.jp.index/Makefile +++ b/org.glite.jp.index/Makefile @@ -49,7 +49,7 @@ INSTALL:=libtool --mode=install install daemon:=glite-jp-indexd examples:=glite-jpis-test glite-jpis-client test:=run-test.sh -test_files:=dump1.sql simple_query.in simple_query.out complex_query.in complex_query.out authz.out +test_files:=dump1.sql simple_query.in simple_query.out complex_query.in complex_query.out authz.out jobid_query.in jobid_query.out MANS1:=glite-jpis-client.1 MANS8:=glite-jp-indexd.8 MANS:=${MANS1} ${MANS8} diff --git a/org.glite.jp.index/config/glite-jpis-config.xml b/org.glite.jp.index/config/glite-jpis-config.xml index 176a470..9a2bdea 100644 --- a/org.glite.jp.index/config/glite-jpis-config.xml +++ b/org.glite.jp.index/config/glite-jpis-config.xml @@ -27,6 +27,7 @@ http://egee.cesnet.cz/en/Schema/JP/System:jobId http://egee.cesnet.cz/en/Schema/LB/Attributes:user http://egee.cesnet.cz/en/Schema/LB/Attributes:CE + http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus diff --git a/org.glite.jp.index/config/startup b/org.glite.jp.index/config/startup index a907f0d..569ba5c 100755 --- a/org.glite.jp.index/config/startup +++ b/org.glite.jp.index/config/startup @@ -75,7 +75,7 @@ start() echo -n Starting glite-jp-indexd ... #su " $GLITE_LOCATION/bin/glite-jp-indexd \ su - $GLITE_USER -c " $GLITE_LOCATION/bin/glite-jp-indexd \ - -s $GLITE_JPIS_PS $GLITE_JPIS_DEBUG \ + $GLITE_JPIS_DEBUG \ -q $GLITE_JPIS_QT $creds $GLITE_JPIS_AUTH \ -m $GLITE_JPIS_DB -p $GLITE_JPIS_PORT \ -i $GLITE_JPIS_PIDFILE -o $GLITE_JPIS_LOGFILE \ diff --git a/org.glite.jp.index/doc/glite-jp-indexd.sgml b/org.glite.jp.index/doc/glite-jp-indexd.sgml index 1007bf4..a0f2d83 100644 --- a/org.glite.jp.index/doc/glite-jp-indexd.sgml +++ b/org.glite.jp.index/doc/glite-jp-indexd.sgml @@ -22,11 +22,6 @@ --help - - -s - --ps-server - HOST:PORT - -d @@ -63,6 +58,11 @@ --logfile FILE.LOG + + -x + --config + CONFIG.XML + @@ -152,6 +152,13 @@ File to store master pid (default: /var/run/glite-jp-indexd.pid/var/run/glite-jp-indexd.log or $HOME/glite-jp-indexd.log). + + + | + +Configuration file containing information about feeds, primary storages, ... Structure is defined in server_conf.xsd. + + diff --git a/org.glite.jp.index/examples/jpis-client.c b/org.glite.jp.index/examples/jpis-client.c index fb551ce..aeb5395 100644 --- a/org.glite.jp.index/examples/jpis-client.c +++ b/org.glite.jp.index/examples/jpis-client.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -23,6 +24,7 @@ #define soap_serialize__jpisclient__QueryJobsResponse soap_serialize__jpelem__QueryJobsResponse #define DEFAULT_JPIS "http://localhost:8902" +//#define USE_GMT 1 /* namespaces[] not used here but needed to prevent linker to complain... */ @@ -315,6 +317,9 @@ static void queryresult_print(FILE *out, const struct _jpelem__QueryJobsRespons struct jptype__attrValue *attr; int i, j, k; +#if USE_GMT + setenv("TZ","UTC",1); tzset(); +#endif fprintf(out, "Result %d jobs:\n", in->__sizejobs); for (j=0; j__sizejobs; j++) { fprintf(out, "\tjobid = %s, owner = %s\n", in->jobs[j]->jobid, in->jobs[j]->owner); diff --git a/org.glite.jp.index/examples/jpis-test.c b/org.glite.jp.index/examples/jpis-test.c index ef926dd..54395d6 100644 --- a/org.glite.jp.index/examples/jpis-test.c +++ b/org.glite.jp.index/examples/jpis-test.c @@ -42,12 +42,16 @@ int main(int argc,char *argv[]) { glite_jp_db_stmt_t stmt; glite_jp_context_t ctx; - glite_jpis_context_t isctx; + glite_jpis_context_t isctx = NULL; glite_jp_is_conf *conf; glite_jp_init_context(&ctx); glite_jp_get_conf(argc, argv, NULL, &conf); + if (!conf) { + fprintf(stderr, "Can't gather configuration\n"); + goto end; + } if (default_server) strcpy(server, default_server); else snprintf(server, sizeof(server), "http://localhost:%s", conf->port ? conf->port : GLITE_JPIS_DEFAULT_PORT_STR); printf("JP index server: %s\n", server); diff --git a/org.glite.jp.index/examples/query-tests/complex_query.out b/org.glite.jp.index/examples/query-tests/complex_query.out index bc11975..1acd133 100644 --- a/org.glite.jp.index/examples/query-tests/complex_query.out +++ b/org.glite.jp.index/examples/query-tests/complex_query.out @@ -13,4 +13,4 @@ Attributes: http://egee.cesnet.cz/en/Schema/LB/Attributes:user OK -https://localhost:7846/pokus1/O=CESNET/O=Masaryk University/CN=Milos Mulachttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusDone1969-12-31T23:00:00ZSYSTEMhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1969-12-31T23:00:00ZSYSTEMhttp://localhost:8901https://localhost:7846/pokus2OwnerNamehttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusReady1969-12-31T23:00:00ZSYSTEMhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1969-12-31T23:00:00ZSYSTEMhttp://localhost:8901 \ No newline at end of file +https://localhost:7846/pokus1/O=CESNET/O=Masaryk University/CN=Milos Mulachttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusDone1970-01-01T02:00:01ZFILEhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1970-01-01T02:00:01ZFILEhttp://localhost:8901https://localhost:7846/pokus2OwnerNamehttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusReady1970-01-01T02:00:01ZSYSTEMhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1970-01-01T02:00:01ZSYSTEMhttp://localhost:8901 diff --git a/org.glite.jp.index/examples/query-tests/dump1.sql b/org.glite.jp.index/examples/query-tests/dump1.sql index c0a5708..5551181 100644 --- a/org.glite.jp.index/examples/query-tests/dump1.sql +++ b/org.glite.jp.index/examples/query-tests/dump1.sql @@ -303,7 +303,7 @@ CREATE TABLE `attr_9892f81a8175c09bd00afcb152f510ad` ( /*!40000 ALTER TABLE `attr_9892f81a8175c09bd00afcb152f510ad` DISABLE KEYS */; LOCK TABLES `attr_9892f81a8175c09bd00afcb152f510ad` WRITE; -INSERT INTO `attr_9892f81a8175c09bd00afcb152f510ad` VALUES ('593e62a063231f8c623b74406b3e12b0','S:CertSubj','S:CertSubj',1),('9276789a0093ad44457655ef03ade36a','S:CertSubj','S:CertSubj',2); +INSERT INTO `attr_9892f81a8175c09bd00afcb152f510ad` VALUES ('593e62a063231f8c623b74406b3e12b0','CertSubj','S:3601:F::CertSubj',1),('9276789a0093ad44457655ef03ade36a','CertSubj','S:3601:S::CertSubj',2); UNLOCK TABLES; /*!40000 ALTER TABLE `attr_9892f81a8175c09bd00afcb152f510ad` ENABLE KEYS */; @@ -352,7 +352,7 @@ CREATE TABLE `attr_a1e9e0a1b7943cc041fefb5da65868f9` ( /*!40000 ALTER TABLE `attr_a1e9e0a1b7943cc041fefb5da65868f9` DISABLE KEYS */; LOCK TABLES `attr_a1e9e0a1b7943cc041fefb5da65868f9` WRITE; -INSERT INTO `attr_a1e9e0a1b7943cc041fefb5da65868f9` VALUES ('593e62a063231f8c623b74406b3e12b0','S:Done','S:Done',1),('9276789a0093ad44457655ef03ade36a','S:Ready','S:Ready',1); +INSERT INTO `attr_a1e9e0a1b7943cc041fefb5da65868f9` VALUES ('593e62a063231f8c623b74406b3e12b0','Done','S:3601:F::Done',1),('9276789a0093ad44457655ef03ade36a','Ready','S:3601:S::Ready',1); UNLOCK TABLES; /*!40000 ALTER TABLE `attr_a1e9e0a1b7943cc041fefb5da65868f9` ENABLE KEYS */; diff --git a/org.glite.jp.index/examples/query-tests/jobid_query.in b/org.glite.jp.index/examples/query-tests/jobid_query.in new file mode 100644 index 0000000..d8a6b35 --- /dev/null +++ b/org.glite.jp.index/examples/query-tests/jobid_query.in @@ -0,0 +1,19 @@ + + + + + http://egee.cesnet.cz/en/Schema/JP/System:jobId + + EQUAL + + https://localhost:7846/pokus1 + + + + + http://egee.cesnet.cz/en/Schema/JP/System:owner + http://egee.cesnet.cz/en/Schema/JP/System:jobId + http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus + http://egee.cesnet.cz/en/Schema/LB/Attributes:user + + diff --git a/org.glite.jp.index/examples/query-tests/jobid_query.out b/org.glite.jp.index/examples/query-tests/jobid_query.out new file mode 100644 index 0000000..36e4164 --- /dev/null +++ b/org.glite.jp.index/examples/query-tests/jobid_query.out @@ -0,0 +1,13 @@ +query: using JPIS http://localhost:12002 + +Conditions: + http://egee.cesnet.cz/en/Schema/JP/System:jobId + == https://localhost:7846/pokus1 +Attributes: + http://egee.cesnet.cz/en/Schema/JP/System:owner + http://egee.cesnet.cz/en/Schema/JP/System:jobId + http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus + http://egee.cesnet.cz/en/Schema/LB/Attributes:user + +OK +https://localhost:7846/pokus1/O=CESNET/O=Masaryk University/CN=Milos Mulachttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusDone1970-01-01T02:00:01ZFILEhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1970-01-01T02:00:01ZFILEhttp://localhost:8901 diff --git a/org.glite.jp.index/examples/query-tests/run-test.sh b/org.glite.jp.index/examples/query-tests/run-test.sh index 2e27c08..cfa335f 100755 --- a/org.glite.jp.index/examples/query-tests/run-test.sh +++ b/org.glite.jp.index/examples/query-tests/run-test.sh @@ -63,6 +63,7 @@ init() { GLITE_JPIS_TEST_PORT=${GLITE_JPIS_TEST_PORT:-"10000"} GLITE_JPIS_TEST_PIDFILE=${GLITE_JPIS_TEST_PIDFILE:-"/tmp/glite-jp-indexd.pid"} GLITE_JPIS_TEST_LOGFILE=${GLITE_JPIS_TEST_LOGFILE:-"/tmp/glite-jp-indexd.log"} + GLITE_JPIS_TEST_CONFIG=${GLITE_JPIS_TEST_CONFIG:-"$GLITE_LOCATION/etc/glite-jpis-config.xml"} if [ -z "$GLITE_JPIS_TEST_DB" ]; then GLITE_JPIS_TEST_DB="jpis/@localhost:jpis1test" @@ -107,7 +108,8 @@ run_is() { # run index server X509_USER_KEY=${X509_USER_KEY} X509_USER_CERT=${X509_USER_CERT} \ $GLITE_LOCATION/bin/glite-jp-indexd -m $GLITE_JPIS_TEST_DB -p $GLITE_JPIS_TEST_PORT \ - -i ${GLITE_JPIS_TEST_PIDFILE} -o ${GLITE_JPIS_TEST_LOGFILE} $1\ + -i ${GLITE_JPIS_TEST_PIDFILE} -o ${GLITE_JPIS_TEST_LOGFILE} \ + -x ${GLITE_JPIS_TEST_CONFIG} $1\ 2>/dev/null @@ -144,7 +146,7 @@ run_test_query() { X509_USER_KEY=${X509_USER_KEY} X509_USER_CERT=${X509_USER_CERT} \ $GLITE_LOCATION/examples/glite-jpis-client -f xml -q $1 \ -i http://localhost:$GLITE_JPIS_TEST_PORT &>/tmp/result - DIFF=`diff --ignore-matching-lines="query: using JPIS" $2 /tmp/result` + DIFF=`diff -b --ignore-matching-lines="query: using JPIS" $2 /tmp/result` if [ -z "$DIFF" -a "$?" -eq "0" ] ; then echo "OK." rm /tmp/result @@ -171,7 +173,8 @@ run_test_query() { run_test_feed() { # run the example numok=`X509_USER_KEY=${X509_USER_KEY} X509_USER_CERT=${X509_USER_CERT}\ - $GLITE_LOCATION/examples/glite-jpis-test -p $GLITE_JPIS_TEST_PORT -m $GLITE_JPIS_TEST_DB ` + $GLITE_LOCATION/examples/glite-jpis-test -p $GLITE_JPIS_TEST_PORT \ + -m $GLITE_JPIS_TEST_DB -x $GLITE_JPIS_TEST_CONFIG | grep -c OK` if [ "$numok" -eq "2" ]; then echo OK. else @@ -216,7 +219,12 @@ create_db; run_is; import_db $GLITE_LOCATION/examples/query-tests/dump1.sql; run_test_query $GLITE_LOCATION/examples/query-tests/simple_query.in $GLITE_LOCATION/examples/query-tests/authz.out; -drop_db; kill_is; - +echo -n "Query jobId test........... " +create_db; +run_is "-n"; +import_db $GLITE_LOCATION/examples/query-tests/dump1.sql; +run_test_query $GLITE_LOCATION/examples/query-tests/jobid_query.in $GLITE_LOCATION/examples/query-tests/jobid_query.out; +drop_db; +kill_is; diff --git a/org.glite.jp.index/examples/query-tests/simple_query.out b/org.glite.jp.index/examples/query-tests/simple_query.out index c3edb73..4972f0f 100644 --- a/org.glite.jp.index/examples/query-tests/simple_query.out +++ b/org.glite.jp.index/examples/query-tests/simple_query.out @@ -10,4 +10,4 @@ Attributes: http://egee.cesnet.cz/en/Schema/LB/Attributes:user OK -https://localhost:7846/pokus2OwnerNamehttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusReady1969-12-31T23:00:00ZSYSTEMhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1969-12-31T23:00:00ZSYSTEMhttp://localhost:8901 \ No newline at end of file +https://localhost:7846/pokus2OwnerNamehttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusReady1970-01-01T02:00:01ZSYSTEMhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1970-01-01T02:00:01ZSYSTEMhttp://localhost:8901 diff --git a/org.glite.jp.index/src/context.c b/org.glite.jp.index/src/context.c index 092fe28..0f09b40 100644 --- a/org.glite.jp.index/src/context.c +++ b/org.glite.jp.index/src/context.c @@ -22,6 +22,7 @@ int glite_jpis_init_context(glite_jpis_context_t *isctx, glite_jp_context_t jpct void glite_jpis_free_context(glite_jpis_context_t ctx) { + if (!ctx) return; free(ctx->hname); free(ctx); } diff --git a/org.glite.jp.index/src/db_ops.c b/org.glite.jp.index/src/db_ops.c index b7b5482..71f1291 100644 --- a/org.glite.jp.index/src/db_ops.c +++ b/org.glite.jp.index/src/db_ops.c @@ -289,7 +289,7 @@ int glite_jpis_initDatabase(glite_jpis_context_t ctx) { // attrs table and attrid_* tables attrs = ctx->conf->attrs; i = 0; - while (attrs[i]) { + if (attrs) while (attrs[i]) { type_full = glite_jp_attrval_db_type_full(jpctx, attrs[i]); type_index = glite_jp_attrval_db_type_index(jpctx, attrs[i], INDEX_LENGTH); @@ -322,7 +322,7 @@ int glite_jpis_initDatabase(glite_jpis_context_t ctx) { if (glite_jp_db_prepare(jpctx, "INSERT INTO feeds (state, locked, source, condition) VALUES (?, ?, ?, ?)", &stmt, param, NULL) != 0) goto fail; feeds = ctx->conf->feeds; i = 0; - while (feeds[i]) { + if (feeds) while (feeds[i]) { state = (feeds[i]->history ? GLITE_JP_IS_STATE_HIST : 0) | (feeds[i]->continuous ? GLITE_JP_IS_STATE_CONT : 0); locked = 0; diff --git a/org.glite.jp.index/src/soap_ops.c b/org.glite.jp.index/src/soap_ops.c index 123fc2e..68bf149 100644 --- a/org.glite.jp.index/src/soap_ops.c +++ b/org.glite.jp.index/src/soap_ops.c @@ -6,6 +6,7 @@ #include "glite/jp/context.h" #include "glite/jp/strmd5.h" #include "glite/jp/attr.h" +#include "glite/jp/known_attr.h" #include "glite/lb/trio.h" #include "jpis_H.h" @@ -182,7 +183,8 @@ static int checkIndexedConditions(glite_jpis_context_t ctx, struct _jpelem__Quer for (k=0; k < in->__sizeconditions; k++) { for (j=0; j < i; j++) { - if (!strcmp(in->conditions[k]->attr, indexed_attrs[j])) { + char *attr = in->conditions[k]->attr; + if (!strcmp(attr, GLITE_JP_ATTR_JOBID) || !strcmp(attr, indexed_attrs[j])) { ret = 0; goto end; } @@ -271,39 +273,43 @@ static int get_jobids(struct soap *soap, glite_jpis_context_t ctx, struct _jpele else */ { - attr_md5 = str2md5(in->conditions[i]->attr); - trio_asprintf(&qa,"%s jobs.jobid = attr_%|Ss.jobid AND (", - (i ? "AND" : ""), attr_md5); - + if (strcmp(in->conditions[i]->attr, GLITE_JP_ATTR_JOBID) == 0) { + /* no subset from attr_ table, used jobs table instead */ + attr_md5 = NULL; + qa = strdup("("); + } else { + /* select given records in attr_ table */ + attr_md5 = str2md5(in->conditions[i]->attr); + trio_asprintf(&qa,"%s jobs.jobid = attr_%|Ss.jobid AND (", + (i ? "AND" : ""), attr_md5); + } + + /* inside part of the condition - ORs */ for (j=0; j < in->conditions[i]->__sizerecord; j++) { if (get_op(in->conditions[i]->record[j]->op, &qop)) goto err; - add_attr_table(attr_md5, &attr_tables); + if (attr_md5) add_attr_table(attr_md5, &attr_tables); + attr.name = in->conditions[i]->attr; if (in->conditions[i]->record[j]->value->string) { - attr.name = in->conditions[i]->attr; attr.value = in->conditions[i]->record[j]->value->string; attr.binary = 0; - glite_jpis_SoapToAttrOrig(soap, - in->conditions[i]->origin, &(attr.origin)); - trio_asprintf(&qb,"%s%sattr_%|Ss.value %s \"%|Ss\"", - qa, (j ? " OR " : ""), attr_md5, qop, - glite_jp_attrval_to_db_index(ctx->jpctx, &attr, 255)); - free(qop); - free(qa); qa = qb; qb = NULL; - } - else { - attr.name = in->conditions[i]->attr; + } else { attr.value = in->conditions[i]->record[j]->value->blob->__ptr; - attr.binary = 1; attr.size = in->conditions[i]->record[j]->value->blob->__size; - glite_jpis_SoapToAttrOrig(soap, - in->conditions[i]->origin, &(attr.origin)); - trio_asprintf(&qb,"%s %s attr_%|Ss.value %s \"%|Ss\"", - qa, (j ? "OR" : ""), attr_md5, qop, + attr.binary = 1; + } + glite_jpis_SoapToAttrOrig(soap, + in->conditions[i]->origin, &(attr.origin)); + if (strcmp(in->conditions[i]->attr, GLITE_JP_ATTR_JOBID) == 0) { + trio_asprintf(&qb,"%s%sjobs.dg_jobid %s \"%|Ss\"", + qa, (j ? " OR " : ""), qop, attr.value); + } else { + trio_asprintf(&qb,"%s%sattr_%|Ss.value %s \"%|Ss\"", + qa, (j ? " OR " : ""), attr_md5, qop, glite_jp_attrval_to_db_index(ctx->jpctx, &attr, 255)); - free(qop); - free(qa); qa = qb; qb = NULL; } + free(qop); + free(qa); qa = qb; qb = NULL; } trio_asprintf(&qb,"%s %s)", qwhere, qa); free(qa); qwhere = qb; qb = NULL; qa = NULL; @@ -390,7 +396,7 @@ static int get_attr(struct soap *soap, glite_jpis_context_t ctx, char *jobid, ch { glite_jp_attrval_t jav; struct jptype__attrValue **av = NULL;; - //enum jptype__attrOrig *origin; + enum jptype__attrOrig *origin; char *query, *fv, *jobid_md5, *attr_md5; int i, ret; glite_jp_db_stmt_t stmt; @@ -432,12 +438,10 @@ static int get_attr(struct soap *soap, glite_jpis_context_t ctx, char *jobid, ch else { av[i]->value->string = soap_strdup(soap, jav.value); } -// XXX: load timestamp and origin from DB -// need to add columns to DB -// av[i]->timestamp = jav.timestamp; -// glite_jpis_AttrOrigToSoap(soap, jav.origin, &origin); -// av[i]->origin = *origin; free(origin); -// av[i]->originDetail = soap_strdup(soap, jav.origin_detail); + av[i]->timestamp = jav.timestamp; + glite_jpis_AttrOrigToSoap(soap, jav.origin, &origin); + av[i]->origin = *origin; soap_dealloc(soap, origin); + av[i]->originDetail = soap_strdup(soap, jav.origin_detail); i++; freeAttval_t(jav); -- 1.8.2.3