From: František Dvořák Date: Fri, 30 Mar 2007 19:59:05 +0000 (+0000) Subject: added EXISTS operation and test (with some cleanup due to SQL strings) X-Git-Tag: glite-jp_R_1_4_0_1~14 X-Git-Url: http://scientific.zcu.cz/git/?a=commitdiff_plain;h=65939e84fa26fc4b6001553f3af86e204de16b10;p=jra1mw.git added EXISTS operation and test (with some cleanup due to SQL strings) simplified tests (using fixed human readable outputs) LILO-style output in tests --- diff --git a/org.glite.jp.index/Makefile b/org.glite.jp.index/Makefile index dda38cb..b7c028c 100644 --- a/org.glite.jp.index/Makefile +++ b/org.glite.jp.index/Makefile @@ -49,7 +49,7 @@ INSTALL:=libtool --mode=install install daemon:=glite-jp-indexd examples:=glite-jpis-test glite-jpis-client test:=run-test.sh -test_files:=dump1.sql simple_query.in simple_query.out complex_query.in complex_query.out authz.out jobid_query.in jobid_query.out origin_query.in origin_query.out +test_files:=dump1.sql simple_query.in simple_query.out complex_query.in complex_query.out authz.out jobid_query.in jobid_query.out origin_query.in origin_query.out exists_query.in exists_query.out MANS1:=glite-jpis-client.1 MANS8:=glite-jp-indexd.8 diff --git a/org.glite.jp.index/examples/query-tests/authz.out b/org.glite.jp.index/examples/query-tests/authz.out index 4d93551..53ff341 100644 --- a/org.glite.jp.index/examples/query-tests/authz.out +++ b/org.glite.jp.index/examples/query-tests/authz.out @@ -11,4 +11,4 @@ Attributes: http://egee.cesnet.cz/en/Schema/LB/Attributes:user OK - +Result 0 jobs: diff --git a/org.glite.jp.index/examples/query-tests/complex_query.out b/org.glite.jp.index/examples/query-tests/complex_query.out index 98775c5..69971c1 100644 --- a/org.glite.jp.index/examples/query-tests/complex_query.out +++ b/org.glite.jp.index/examples/query-tests/complex_query.out @@ -15,4 +15,22 @@ Attributes: http://egee.cesnet.cz/en/Schema/LB/Attributes:user OK -https://localhost:7846/pokus1/O=CESNET/O=Masaryk University/CN=Milos Mulachttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusDone1970-01-01T02:00:01ZFILEhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1970-01-01T02:00:01ZFILEhttp://localhost:8901https://localhost:7846/pokus2OwnerNamehttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusReady1970-01-01T02:00:01ZSYSTEMhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1970-01-01T02:00:01ZSYSTEMhttp://localhost:8901 +Result 2 jobs: + jobid = https://localhost:7846/pokus1, owner = /O=CESNET/O=Masaryk University/CN=Milos Mulac + http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus + value = Done + origin = FILE (no detail) + time = Thu Jan 1 02:00:01 1970 + http://egee.cesnet.cz/en/Schema/LB/Attributes:user + value = CertSubj + origin = FILE (no detail) + time = Thu Jan 1 02:00:01 1970 + jobid = https://localhost:7846/pokus2, owner = OwnerName + http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus + value = Ready + origin = SYSTEM (no detail) + time = Thu Jan 1 02:00:01 1970 + http://egee.cesnet.cz/en/Schema/LB/Attributes:user + value = CertSubj + origin = SYSTEM (no detail) + time = Thu Jan 1 02:00:01 1970 diff --git a/org.glite.jp.index/examples/query-tests/dump1.sql b/org.glite.jp.index/examples/query-tests/dump1.sql index 814b5e4..1953b66 100644 --- a/org.glite.jp.index/examples/query-tests/dump1.sql +++ b/org.glite.jp.index/examples/query-tests/dump1.sql @@ -401,6 +401,7 @@ CREATE TABLE `attr_c47f78255056386d2b3da6d506d1f244` ( /*!40000 ALTER TABLE `attr_c47f78255056386d2b3da6d506d1f244` DISABLE KEYS */; LOCK TABLES `attr_c47f78255056386d2b3da6d506d1f244` WRITE; +INSERT INTO `attr_c47f78255056386d2b3da6d506d1f244` VALUES ('593e62a063231f8c623b74406b3e12b0','VOCE','S:7201:F::VOCE',3); UNLOCK TABLES; /*!40000 ALTER TABLE `attr_c47f78255056386d2b3da6d506d1f244` ENABLE KEYS */; diff --git a/org.glite.jp.index/examples/query-tests/exists_query.in b/org.glite.jp.index/examples/query-tests/exists_query.in new file mode 100644 index 0000000..0af4672 --- /dev/null +++ b/org.glite.jp.index/examples/query-tests/exists_query.in @@ -0,0 +1,17 @@ + + + + + http://egee.cesnet.cz/en/Schema/LB/Attributes:CE + + EXISTS + + + + http://egee.cesnet.cz/en/Schema/JP/System:owner + http://egee.cesnet.cz/en/Schema/JP/System:jobId + http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus + http://egee.cesnet.cz/en/Schema/LB/Attributes:user + http://egee.cesnet.cz/en/Schema/LB/Attributes:CE + + diff --git a/org.glite.jp.index/examples/query-tests/exists_query.out b/org.glite.jp.index/examples/query-tests/exists_query.out new file mode 100644 index 0000000..04f2c48 --- /dev/null +++ b/org.glite.jp.index/examples/query-tests/exists_query.out @@ -0,0 +1,28 @@ +query: using JPIS http://localhost:10000 + +Conditions: + http://egee.cesnet.cz/en/Schema/LB/Attributes:CE + origin IS ANY + value exists +Attributes: + http://egee.cesnet.cz/en/Schema/JP/System:owner + http://egee.cesnet.cz/en/Schema/JP/System:jobId + http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus + http://egee.cesnet.cz/en/Schema/LB/Attributes:user + http://egee.cesnet.cz/en/Schema/LB/Attributes:CE + +OK +Result 1 jobs: + jobid = https://localhost:7846/pokus1, owner = /O=CESNET/O=Masaryk University/CN=Milos Mulac + http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus + value = Done + origin = FILE (no detail) + time = Thu Jan 1 02:00:01 1970 + http://egee.cesnet.cz/en/Schema/LB/Attributes:user + value = CertSubj + origin = FILE (no detail) + time = Thu Jan 1 02:00:01 1970 + http://egee.cesnet.cz/en/Schema/LB/Attributes:CE + value = VOCE + origin = FILE (no detail) + time = Thu Jan 1 02:00:01 1970 diff --git a/org.glite.jp.index/examples/query-tests/jobid_query.out b/org.glite.jp.index/examples/query-tests/jobid_query.out index dc31dd9..e00d456 100644 --- a/org.glite.jp.index/examples/query-tests/jobid_query.out +++ b/org.glite.jp.index/examples/query-tests/jobid_query.out @@ -1,4 +1,4 @@ -query: using JPIS http://localhost:12002 +query: using JPIS http://localhost:10000 Conditions: http://egee.cesnet.cz/en/Schema/JP/System:jobId @@ -11,4 +11,13 @@ Attributes: http://egee.cesnet.cz/en/Schema/LB/Attributes:user OK -https://localhost:7846/pokus1/O=CESNET/O=Masaryk University/CN=Milos Mulachttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusDone1970-01-01T02:00:01ZFILEhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1970-01-01T02:00:01ZFILEhttp://localhost:8901 +Result 1 jobs: + jobid = https://localhost:7846/pokus1, owner = /O=CESNET/O=Masaryk University/CN=Milos Mulac + http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus + value = Done + origin = FILE (no detail) + time = Thu Jan 1 02:00:01 1970 + http://egee.cesnet.cz/en/Schema/LB/Attributes:user + value = CertSubj + origin = FILE (no detail) + time = Thu Jan 1 02:00:01 1970 diff --git a/org.glite.jp.index/examples/query-tests/origin_query.out b/org.glite.jp.index/examples/query-tests/origin_query.out index a88401a..573f5d7 100644 --- a/org.glite.jp.index/examples/query-tests/origin_query.out +++ b/org.glite.jp.index/examples/query-tests/origin_query.out @@ -1,4 +1,4 @@ -query: using JPIS http://localhost:12002 +query: using JPIS http://localhost:10000 Conditions: http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus @@ -11,4 +11,13 @@ Attributes: http://egee.cesnet.cz/en/Schema/LB/Attributes:user OK -https://localhost:7846/pokus1/O=CESNET/O=Masaryk University/CN=Milos Mulachttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusDone1970-01-01T02:00:01ZFILEhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1970-01-01T02:00:01ZFILEhttp://localhost:8901 +Result 1 jobs: + jobid = https://localhost:7846/pokus1, owner = /O=CESNET/O=Masaryk University/CN=Milos Mulac + http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus + value = Done + origin = FILE (no detail) + time = Thu Jan 1 02:00:01 1970 + http://egee.cesnet.cz/en/Schema/LB/Attributes:user + value = CertSubj + origin = FILE (no detail) + time = Thu Jan 1 02:00:01 1970 diff --git a/org.glite.jp.index/examples/query-tests/run-test.sh b/org.glite.jp.index/examples/query-tests/run-test.sh index 89f99d0..ad2e381 100755 --- a/org.glite.jp.index/examples/query-tests/run-test.sh +++ b/org.glite.jp.index/examples/query-tests/run-test.sh @@ -89,6 +89,9 @@ create_db() { import_db() { # import database + echo -n "D" + # delay (if JP IS downloads something...) + sleep 1 cat $1 | sed "s/jpis1test/$DB_NAME/" | mysql -u $DB_USER -h $DB_HOST if [ x"$?" != x"0" ]; then echo "FAILED to import database." @@ -96,6 +99,7 @@ import_db() { drop_db; exit 1 fi + echo -n "B " } drop_db() { @@ -113,6 +117,7 @@ run_is() { exit 1 fi + echo -n "I" # run index server X509_USER_KEY=${X509_USER_KEY} X509_USER_CERT=${X509_USER_CERT} \ $GLITE_LOCATION/bin/glite-jp-indexd -m $GLITE_JPIS_TEST_DB -p $GLITE_JPIS_TEST_PORT \ @@ -143,6 +148,7 @@ run_is() { i=$(($i+1)) LC_ALL=C sleep 0.1 done + echo -n "S " } kill_is() { @@ -154,9 +160,11 @@ kill_is() { } run_test_query() { + echo -n "Q" X509_USER_KEY=${X509_USER_KEY} X509_USER_CERT=${X509_USER_CERT} \ - $GLITE_LOCATION/examples/glite-jpis-client -f strippedxml -q $1 \ - -i http://localhost:$GLITE_JPIS_TEST_PORT 2>&1 | sed -e 's,,,' -e 's, SOAP-ENV:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/",,' > /tmp/result + $GLITE_LOCATION/examples/glite-jpis-client -f hr -q $1 \ + -i http://localhost:$GLITE_JPIS_TEST_PORT > /tmp/result 2>&1 + echo -n "R " DIFF=`diff -b -B --ignore-matching-lines="query: using JPIS" $2 /tmp/result` if [ -z "$DIFF" -a "$?" -eq "0" ] ; then echo "OK." @@ -259,3 +267,11 @@ import_db $GLITE_LOCATION/examples/query-tests/dump1.sql; run_test_query $GLITE_LOCATION/examples/query-tests/origin_query.in $GLITE_LOCATION/examples/query-tests/origin_query.out; drop_db; kill_is; + +echo -n "EXISTS operation test........... " +create_db; +run_is "-n"; +import_db $GLITE_LOCATION/examples/query-tests/dump1.sql; +run_test_query $GLITE_LOCATION/examples/query-tests/exists_query.in $GLITE_LOCATION/examples/query-tests/exists_query.out; +drop_db; +kill_is; diff --git a/org.glite.jp.index/examples/query-tests/simple_query.out b/org.glite.jp.index/examples/query-tests/simple_query.out index 06c027b..3426131 100644 --- a/org.glite.jp.index/examples/query-tests/simple_query.out +++ b/org.glite.jp.index/examples/query-tests/simple_query.out @@ -11,4 +11,13 @@ Attributes: http://egee.cesnet.cz/en/Schema/LB/Attributes:user OK -https://localhost:7846/pokus2OwnerNamehttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusReady1970-01-01T02:00:01ZSYSTEMhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1970-01-01T02:00:01ZSYSTEMhttp://localhost:8901 +Result 1 jobs: + jobid = https://localhost:7846/pokus2, owner = OwnerName + http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus + value = Ready + origin = SYSTEM (no detail) + time = Thu Jan 1 02:00:01 1970 + http://egee.cesnet.cz/en/Schema/LB/Attributes:user + value = CertSubj + origin = SYSTEM (no detail) + time = Thu Jan 1 02:00:01 1970 diff --git a/org.glite.jp.index/src/db_ops.c b/org.glite.jp.index/src/db_ops.c index f1c0eb6..28915fa 100644 --- a/org.glite.jp.index/src/db_ops.c +++ b/org.glite.jp.index/src/db_ops.c @@ -311,7 +311,16 @@ int glite_jpis_initDatabase(glite_jpis_context_t ctx) { sql[sizeof(sql) - 1] = '\0'; snprintf(sql, sizeof(sql) - 1, SQLCMD_CREATE_DATA_TABLE, attrid, type_index, type_full); llprintf(LOG_SQL, "creating table: '%s'\n", sql); - if ((glite_jp_db_execstmt(jpctx, sql, NULL)) == -1) goto fail; + if ((glite_jp_db_execstmt(jpctx, sql, NULL)) == -1) { + glite_jp_error_t err; + + memset(&err,0,sizeof err); + err.code = EAGAIN; + err.source = __FUNCTION__; + err.desc = "If the atribute table already exists, restart may help."; + glite_jp_stack_error(ctx, &err); + goto fail; + } i++; } diff --git a/org.glite.jp.index/src/soap_ops.c b/org.glite.jp.index/src/soap_ops.c index 834c035..354d26b 100644 --- a/org.glite.jp.index/src/soap_ops.c +++ b/org.glite.jp.index/src/soap_ops.c @@ -199,15 +199,58 @@ static int get_op(const enum jptype__queryOp in, char **out) return(0); } + +static char *get_sql_or(struct soap *soap, glite_jpis_context_t ctx, struct jptype__indexQuery *condition, const char *attr_md5) { + struct jptype__indexQueryRecord *record; + glite_jp_attrval_t attr; + char *qop, *sql, *s; + int j; + + sql = strdup(""); + for (j=0; j < condition->__sizerecord; j++) { + record = GLITE_SECURITY_GSOAP_LIST_GET(condition->record, j); + if (record->op == jptype__queryOp__EXISTS) { + /* no additional conditions needed when existing is enough */ + } else { + if (get_op(record->op, &qop)) return NULL; + + memset(&attr, 0, sizeof(attr)); + attr.name = condition->attr; + if (GSOAP_ISSTRING(record->value)) { + attr.value = GSOAP_STRING(record->value); + attr.binary = 0; + } else { + attr.value = GSOAP_BLOB(record->value)->__ptr; + attr.size = GSOAP_BLOB(record->value)->__size; + attr.binary = 1; + } + glite_jpis_SoapToAttrOrig(soap, + condition->origin, &(attr.origin)); + if (strcmp(condition->attr, GLITE_JP_ATTR_JOBID) == 0) { + trio_asprintf(&s,"%s%sjobs.dg_jobid %s \"%|Ss\"", + sql, (sql[0] ? " OR " : ""), qop, attr.value); + } else { + trio_asprintf(&s,"%s%sattr_%|Ss.value %s \"%|Ss\"", + sql, (sql[0] ? " OR " : ""), attr_md5, qop, + glite_jp_attrval_to_db_index(ctx->jpctx, &attr, 255)); + } + free(qop); + free(sql); sql = s; + } + } + + return sql; +} + + /* get all jobids matching the query conditions */ static int get_jobids(struct soap *soap, glite_jpis_context_t ctx, struct _jpelem__QueryJobs *in, char ***jobids, char *** ps_list) { - char *qa = NULL, *qb = NULL, *qop = NULL, *attr_md5, + char *qa = NULL, *qb = NULL, *qor, *attr_md5, *qwhere = NULL, *query = NULL, *res[2], **jids = NULL, **pss = NULL, **attr_tables = NULL; - int i, j, ret; + int i, ret; glite_jp_db_stmt_t stmt; - glite_jp_attrval_t attr; glite_jp_attr_orig_t orig; @@ -221,9 +264,10 @@ static int get_jobids(struct soap *soap, glite_jpis_context_t ctx, struct _jpele if (strcmp(condition->attr, GLITE_JP_ATTR_JOBID) == 0) { /* no subset from attr_ table, used jobs table instead */ attr_md5 = NULL; - qa = strdup("("); + qa = strdup(""); } else { attr_md5 = str2md5(condition->attr); + add_attr_table(attr_md5, &attr_tables); /* origin */ if (condition->origin) { @@ -233,43 +277,22 @@ static int get_jobids(struct soap *soap, glite_jpis_context_t ctx, struct _jpele trio_asprintf(&qb, ""); /* select given records in attr_ table */ - trio_asprintf(&qa,"%s%s jobs.jobid = attr_%|Ss.jobid AND (", + trio_asprintf(&qa,"%s%s jobs.jobid = attr_%|Ss.jobid", (i ? "AND" : ""), qb, attr_md5); free(qb); } /* inside part of the condition: record list (ORs) */ - for (j=0; j < condition->__sizerecord; j++) { - struct jptype__indexQueryRecord *record; - - record = GLITE_SECURITY_GSOAP_LIST_GET(condition->record, j); - if (get_op(record->op, &qop)) goto err; - if (attr_md5) add_attr_table(attr_md5, &attr_tables); - - attr.name = condition->attr; - if (GSOAP_ISSTRING(record->value)) { - attr.value = GSOAP_STRING(record->value); - attr.binary = 0; - } else { - attr.value = GSOAP_BLOB(record->value)->__ptr; - attr.size = GSOAP_BLOB(record->value)->__size; - attr.binary = 1; - } - glite_jpis_SoapToAttrOrig(soap, - condition->origin, &(attr.origin)); - if (strcmp(condition->attr, GLITE_JP_ATTR_JOBID) == 0) { - trio_asprintf(&qb,"%s%sjobs.dg_jobid %s \"%|Ss\"", - qa, (j ? " OR " : ""), qop, attr.value); - } else { - trio_asprintf(&qb,"%s%sattr_%|Ss.value %s \"%|Ss\"", - qa, (j ? " OR " : ""), attr_md5, qop, - glite_jp_attrval_to_db_index(ctx->jpctx, &attr, 255)); - } - free(qop); - free(qa); qa = qb; qb = NULL; + if ((qor = get_sql_or(soap, ctx, condition, attr_md5)) == NULL) goto err; + if (qor[0]) { + asprintf(&qb, "%s%s(%s)", qa, qa[0] ? " AND " : "", qor); + free(qa); + qa = qb; } - trio_asprintf(&qb,"%s %s)", qwhere, qa); + free(qor); + + trio_asprintf(&qb,"%s %s", qwhere, qa); free(qa); qwhere = qb; qb = NULL; qa = NULL; free(attr_md5); } @@ -282,7 +305,7 @@ static int get_jobids(struct soap *soap, glite_jpis_context_t ctx, struct _jpele } if (ctx->conf->no_auth) { - trio_asprintf(&query, "SELECT DISTINCT dg_jobid,ps FROM jobs%s WHERE %s", qa, qwhere); + trio_asprintf(&query, "SELECT DISTINCT dg_jobid,ps FROM jobs%s WHERE%s", qa, qwhere); } else { trio_asprintf(&query, "SELECT DISTINCT dg_jobid,ps FROM jobs,users%s WHERE (jobs.ownerid = users.userid AND users.cert_subj='%s') AND %s", qa, ctx->jpctx->peer, qwhere);