From 65e2c9023ae5923c4ce19d8fb7067e101007e573 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= Date: Mon, 12 May 2008 15:36:20 +0000 Subject: [PATCH] jp.common: - helper function for accessing JPIS DB - enabled more warnings and their fixes jp.client: - creating whole subtree via mkdir for jp importer - build fix for enabled statistics jp.index: - switched to the new code in jp.common - plans for the new schema --- org.glite.jp.client/config/startup | 4 +- org.glite.jp.client/src/jpimporter.c | 3 +- org.glite.jp.common/Makefile | 10 +-- org.glite.jp.common/interface/indexdb.h | 59 ++++++++++++++++++ org.glite.jp.common/src/attr.c | 3 +- org.glite.jp.common/src/context.c | 5 +- org.glite.jp.common/src/indexdb.c | 71 ++++++++++++++++++++++ .../config/glite-jp-index-dbsetup.sql | 33 +++++++++- org.glite.jp.index/src/db_ops.c | 34 +++-------- org.glite.jp.index/src/soap_ops.c | 5 +- 10 files changed, 186 insertions(+), 41 deletions(-) create mode 100644 org.glite.jp.common/interface/indexdb.h create mode 100644 org.glite.jp.common/src/indexdb.c diff --git a/org.glite.jp.client/config/startup b/org.glite.jp.client/config/startup index 7b843bb..d4b3405 100755 --- a/org.glite.jp.client/config/startup +++ b/org.glite.jp.client/config/startup @@ -38,10 +38,10 @@ start() [ -z "$GLITE_LB_EXPORT_JPREG_MAILDIR" ] && GLITE_LB_EXPORT_JPREG_MAILDIR=$GLITE_LOCATION_VAR/jpreg jpreg_maildir="--reg-mdir $GLITE_LB_EXPORT_JPREG_MAILDIR " - [ -d "$GLITE_LB_EXPORT_JPREG_MAILDIR" ] || mkdir "$GLITE_LB_EXPORT_JPREG_MAILDIR" && chown $GLITE_USER:$GLITE_GROUP "$GLITE_LB_EXPORT_JPREG_MAILDIR" + [ -d "$GLITE_LB_EXPORT_JPREG_MAILDIR" ] || mkdir -p "$GLITE_LB_EXPORT_JPREG_MAILDIR" && chown $GLITE_USER:$GLITE_GROUP "$GLITE_LB_EXPORT_JPREG_MAILDIR" [ -z "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" ] && GLITE_LB_EXPORT_JPDUMP_MAILDIR=$GLITE_LOCATION_VAR/jpdump jpdump_maildir="--dump-mdir $GLITE_LB_EXPORT_JPDUMP_MAILDIR " - [ -d "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" ] || mkdir "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" && chown $GLITE_USER:$GLITE_GROUP "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" + [ -d "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" ] || mkdir -p "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" && chown $GLITE_USER:$GLITE_GROUP "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" [ -n "$GLITE_LB_EXPORT_JPPS" ] && jpps="--jpps $GLITE_LB_EXPORT_JPPS " [ -n "$GLITE_LB_EXPORT_SANDBOX_MAILDIR" ] && sandbox_maildir="--sandbox-mdir $GLITE_LB_EXPORT_SANDBOX_MAILDIR " diff --git a/org.glite.jp.client/src/jpimporter.c b/org.glite.jp.client/src/jpimporter.c index e5a20fb..4195963 100644 --- a/org.glite.jp.client/src/jpimporter.c +++ b/org.glite.jp.client/src/jpimporter.c @@ -538,6 +538,8 @@ static int dump_importer(void) dprintf("[%s] dump JP import request received\n", name); if ( !debug ) syslog(LOG_INFO, "dump JP import request received"); + soap_begin(soap); + ret = 0; if ( parse_msg(msg, tab) < 0 ) { dprintf("[%s] Wrong format of message!\n", name); @@ -569,7 +571,6 @@ static int dump_importer(void) } if (perf.name && !perf.limit) stats_get_limit(&perf, name); } - soap_begin(soap) if (!(sink & 2)) { #endif retry_upload = 2; diff --git a/org.glite.jp.common/Makefile b/org.glite.jp.common/Makefile index 445bd0e..e111a36 100644 --- a/org.glite.jp.common/Makefile +++ b/org.glite.jp.common/Makefile @@ -25,8 +25,8 @@ VPATH=${top_srcdir}/src:${top_srcdir}/test:${top_srcdir}/project:${jpproject} GLOBUSINC:= -I${globus_prefix}/include/${nothrflavour} GLOBUSINCTHR:= -I${globus_prefix}/include/${thrflavour} -DEBUG:=-g -O0 -CFLAGS:=${DEBUG} -I. -I${top_srcdir}/interface -I${stagedir}/include +DEBUG:=-g -O0 -W -Wno-sign-compare +CFLAGS:=${DEBUG} -D_GNU_SOURCE -I. -I${top_srcdir}/interface -I${stagedir}/include offset=0 version_info:=-version-info ${shell \ @@ -38,12 +38,12 @@ LINKXX:=libtool --mode=link ${CXX} ${LDFLAGS} INSTALL:=libtool --mode=install install COMPILE:=libtool --mode=compile ${CC} ${CFLAGS} -HDRS:=types.h context.h attr.h known_attr.h backend.h builtin_plugins.h file_plugin.h +HDRS:=types.h context.h attr.h known_attr.h backend.h builtin_plugins.h file_plugin.h indexdb.h -SRCS:=context.c attr.c utils.c +SRCS:=context.c attr.c utils.c indexdb.c OBJS:=${SRCS:.c=.lo} THROBJS:=${OBJS:.o=.thr.lo} -LIBS:=-L${globus_prefix}/lib -lcrypto_${nothrflavour} -L${stagedir}/lib -lglite_jobid +LIBS:=-L${globus_prefix}/lib -lcrypto_${nothrflavour} -L${stagedir}/lib -lglite_jobid -lglite_lbu_db THRLIBS:=-L${globus_prefix}/lib -lcrypto_${thrflavour} commonlib:= libglite_jp_common_${nothrflavour}.la diff --git a/org.glite.jp.common/interface/indexdb.h b/org.glite.jp.common/interface/indexdb.h new file mode 100644 index 0000000..7130d73 --- /dev/null +++ b/org.glite.jp.common/interface/indexdb.h @@ -0,0 +1,59 @@ +#ifndef GLITE_JP_INDEXDB_H +#define GLITE_JP_INDEXDB_H + +/** + * \file indexdb.h + * \brief Helper functions for accessing Job Provenance Index Server database. + */ + +#include "glite/lbu/db.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifdef GLITE_JP_INDEX_COMPILE + +/** + * For generating table names. + */ +#define GLITE_JP_INDEXDB_TABLE_ATTR_PREFIX "attr_" + +#endif + + +/** + * Returns internal id from attribute name. + * + * The attribute id is used in some places in the database schema. Because of the future possible changes in the schema, you should rather use other functions to get SQL commands (table names and where conditions). + * + * \param[in] name attribute name + * \return attribute id string + */ +char *glite_jp_indexdb_attr2id(const char *name); + +/** + * Get parts of the SQL SELECT command to the given attribute: + * + * SELECT column, full_value_column FROM table WHERE where; + * + * This is quick version requiring additional information about indexing of the attribute. + */ +int glite_jp_indexdb_attr2select_from_index(const char *name, int indexed, char **column, char **full_value_column, char **table, char **where); + +/** + * Get parts of the SQL SELECT command to the given attribute: + * + * SELECT column, full_value_column FROM table WHERE where; + * + * This is the most portable way - it will peep to the DB for information about indexing. + */ +int glite_jp_indexdb_attr2select_from_db(const char *name, glite_lbu_DBContext dbctx, char **column, char **full_value_column, char **table, char **where); + +#ifdef __cplusplus +}; +#endif + + +#endif /* GLITE_JP_INDEXDB_H */ diff --git a/org.glite.jp.common/src/attr.c b/org.glite.jp.common/src/attr.c index 8905d09..505d58e 100644 --- a/org.glite.jp.common/src/attr.c +++ b/org.glite.jp.common/src/attr.c @@ -8,6 +8,7 @@ #include "types.h" #include "attr.h" #include "type_plugin.h" +#include "context.h" void glite_jp_attrval_free(glite_jp_attrval_t *a,int f) { @@ -26,7 +27,7 @@ void glite_jp_attrval_copy(glite_jp_attrval_t *dst,const glite_jp_attrval_t *src dst->timestamp = src->timestamp; dst->origin_detail = src->origin_detail ? strdup(src->origin_detail) : NULL; - if (dst->binary = src->binary) { + if ((dst->binary = src->binary)) { dst->value = malloc(src->size); memcpy(dst->value,src->value,src->size); } diff --git a/org.glite.jp.common/src/context.c b/org.glite.jp.common/src/context.c index a6608aa..62bd3c7 100644 --- a/org.glite.jp.common/src/context.c +++ b/org.glite.jp.common/src/context.c @@ -1,5 +1,6 @@ #include #include +#include #include "types.h" #include "context.h" @@ -37,7 +38,7 @@ char *glite_jp_error_chain(glite_jp_context_t ctx) ret = realloc(ret,len + add + 1); strncpy(ret + len,buf,add); ret[len += add] = 0; strcat(indent," "); - } while (ep = ep->reason); + } while ((ep = ep->reason)); return ret; } @@ -97,7 +98,7 @@ int glite_jp_run_deferred(glite_jp_context_t ctx) glite_jp_clear_error(ctx); for (cnt=0;ctx->deferred_func[cnt];cnt++); for (i=0; ideferred_func)(ctx,*ctx->deferred_arg)) { + if ((ret = (*ctx->deferred_func)(ctx,*ctx->deferred_arg))) { glite_jp_error_t err; char desc[100]; diff --git a/org.glite.jp.common/src/indexdb.c b/org.glite.jp.common/src/indexdb.c new file mode 100644 index 0000000..cc1fc77 --- /dev/null +++ b/org.glite.jp.common/src/indexdb.c @@ -0,0 +1,71 @@ +#ident "$Header:" + +#include +#include +#include +#include +#include + +#include +#include +#include + +#define GLITE_JP_INDEX_COMPILE 1 + +#include "indexdb.h" + +char *glite_jp_indexdb_attr2id(const char *name) { + size_t i, len; + char *lname, *id; + + len = strlen(name); + lname = malloc(len + 1); + for (i = 0; i < len + 1; i++) lname[i] = tolower(name[i]); + id = str2md5(lname); + free(lname); + + return id; +} + + +int glite_jp_indexdb_attr2select_from_index(const char *name, int indexed __attribute__((unused)), char **column, char **full_value_column, char **table, char **where) { + char *id; + + if (column) *column = strdup("value"); + if (full_value_column) *full_value_column = strdup("full_value"); + if (table) { + id = glite_jp_indexdb_attr2id(name); + asprintf(table, GLITE_JP_INDEXDB_TABLE_ATTR_PREFIX "%s", id); + free(id); + } + if (where) *where = strdup(""); + + return 0; +} + + +int glite_jp_indexdb_attr2select_from_db(const char *name, glite_lbu_DBContext dbctx, char **column, char **full_value_column, char **table, char **where) { + char *sql, *id; + glite_lbu_Statement stmt; + int ret; + + if (table) { + trio_asprintf(&sql, "SELECT attrid FROM attrs WHERE name='%|Ss'", name); + ret = glite_lbu_ExecSQL(dbctx, sql, &stmt); + free(sql); + switch (ret) { + case -1: return glite_lbu_DBError(dbctx, NULL, NULL); + case 1: break; + default: return EINVAL; + } + if (glite_lbu_FetchRow(stmt, 1, NULL, &id) < 0) return glite_lbu_DBError(dbctx, NULL, NULL); + asprintf(table, GLITE_JP_INDEXDB_TABLE_ATTR_PREFIX "%s", id); + free(id); + glite_lbu_FreeStmt(&stmt); + } + if (column) *column = strdup("value"); + if (full_value_column) *full_value_column = strdup("full_value"); + if (where) *where = strdup(""); + + return 0; +} diff --git a/org.glite.jp.index/config/glite-jp-index-dbsetup.sql b/org.glite.jp.index/config/glite-jp-index-dbsetup.sql index 615b11d..0985239 100644 --- a/org.glite.jp.index/config/glite-jp-index-dbsetup.sql +++ b/org.glite.jp.index/config/glite-jp-index-dbsetup.sql @@ -53,9 +53,8 @@ create table users ( unique (cert_subj) ) engine=innodb; - -# data tables - created one for each configured attribute, index on -# value is created only for attributes configured to be indexed +# data tables - created one for each configured and indexed attribute, +# in future values of the non-indexed attributes will be stored in attr_values # #create table attr_ ( # `jobid` char(32) binary not null, @@ -67,3 +66,31 @@ create table users ( # index (value) #) engine=innodb; + +# ---- future schema improvements ---- + +#create table attr_values ( +# `jobid` char(32), +# `attrid` char(32) binary not null, +# `value` varchar(255) binary not null, +# `full_value` mediumblob not null, +# `origin` int not null, +## `is_multival` int, +# +## primary key (jobid, attrid) +# index (jobid), +# index (attrid), +# index (value) +#) engine=innodb; + +#create table attr_multivalues ( +# `jobid` char(32), +# `attrid` char(32) binary not null, +# `value` varchar(255) binary not null, +# `full_value` mediumblob not null, +# `origin` int not null, +# +# index (jobid), +# index (attrid) +# index (value) +#) engine=innodb; diff --git a/org.glite.jp.index/src/db_ops.c b/org.glite.jp.index/src/db_ops.c index dac59f5..c642097 100644 --- a/org.glite.jp.index/src/db_ops.c +++ b/org.glite.jp.index/src/db_ops.c @@ -10,6 +10,8 @@ #include #include +#define GLITE_JP_INDEX_COMPILE 1 + #include #include #include @@ -18,6 +20,7 @@ #include #include #include "glite/jp/known_attr.h" +#include "glite/jp/indexdb.h" #include "conf.h" #include "context.h" @@ -29,9 +32,8 @@ #define LOG_SQL 1 #endif -#define TABLE_PREFIX_DATA "attr_" -#define SQLCMD_DROP_DATA_TABLE "DROP TABLE " TABLE_PREFIX_DATA "%s" -#define SQLCMD_CREATE_DATA_TABLE "CREATE TABLE " TABLE_PREFIX_DATA "%s (\n\ +#define SQLCMD_DROP_DATA_TABLE "DROP TABLE " GLITE_JP_INDEXDB_TABLE_ATTR_PREFIX "%s" +#define SQLCMD_CREATE_DATA_TABLE "CREATE TABLE " GLITE_JP_INDEXDB_TABLE_ATTR_PREFIX "%s (\n\ `jobid` CHAR(32) BINARY NOT NULL,\n\ `value` %s BINARY NOT NULL,\n\ `full_value` %s NOT NULL,\n\ @@ -40,7 +42,7 @@ INDEX (jobid),\n\ INDEX (value)\n\ ) ENGINE=innodb;" -#define SQLCMD_INSERT_ATTRVAL "INSERT INTO " TABLE_PREFIX_DATA "%|Ss (jobid, value, full_value, origin) VALUES (\n\ +#define SQLCMD_INSERT_ATTRVAL "INSERT INTO " GLITE_JP_INDEXDB_TABLE_ATTR_PREFIX "%|Ss (jobid, value, full_value, origin) VALUES (\n\ '%|Ss',\n\ '%|Ss',\n\ '%|Ss',\n\ @@ -260,24 +262,6 @@ fail: } -/** - * Convert attribute name to attribute id. - */ -char *glite_jpis_attr_name2id(const char *name) { - size_t i, len; - char *lname, *id; - - len = strlen(name); - lname = malloc(len + 1); - for (i = 0; i < len + 1; i++) lname[i] = tolower(name[i]); - id = str2md5(lname); - free(lname); - - return id; -// return str2md5(name); -} - - /* Init the database. * * \retval 0 OK @@ -328,7 +312,7 @@ int glite_jpis_initDatabase(glite_jpis_context_t ctx) { type_full = glite_jp_attrval_db_type_full(jpctx, attrs[i]); type_index = glite_jp_attrval_db_type_index(jpctx, attrs[i], INDEX_LENGTH); - attrid = glite_jpis_attr_name2id(attrs[i]); + attrid = glite_jp_indexdb_attr2id(attrs[i]); indexed = is_indexed(ctx->conf, attrs[i]); if (glite_jp_db_ExecPreparedStmt(jpctx, stmt, 4, GLITE_LBU_DB_TYPE_VARCHAR, attrid, @@ -415,7 +399,7 @@ int glite_jpis_dropDatabase(glite_jpis_context_t ctx) { while ((ret = glite_jp_db_FetchRow(jpctx, stmt_tabs, 1, &len, &attrid)) > 0) { snprintf(sql, sizeof(sql), SQLCMD_DROP_DATA_TABLE, attrid); llprintf(LOG_SQL, "dropping '%s' ==> '%s'\n", attrid, sql); - if (glite_jp_db_ExecSQL(jpctx, sql, NULL) == -1) printf("warning: can't drop table '" TABLE_PREFIX_DATA "%s': %s (%s)\n", attrid, jpctx->error->desc, jpctx->error->source); + if (glite_jp_db_ExecSQL(jpctx, sql, NULL) == -1) printf("warning: can't drop table '" GLITE_JP_INDEXDB_TABLE_ATTR_PREFIX "%s': %s (%s)\n", attrid, jpctx->error->desc, jpctx->error->source); } if (ret != 0) goto fail; glite_jp_db_FreeStmt(&stmt_tabs); @@ -644,7 +628,7 @@ int glite_jpis_insertAttrVal(glite_jpis_context_t ctx, const char *jobid, glite_ char *sql, *table, *value, *full_value, *md5_jobid; long int origin; - table = glite_jpis_attr_name2id(av->name); + table = glite_jp_indexdb_attr2id(av->name); value = glite_jp_attrval_to_db_index(ctx->jpctx, av, INDEX_LENGTH); full_value = glite_jp_attrval_to_db_full(ctx->jpctx, av); md5_jobid = str2md5(jobid); diff --git a/org.glite.jp.index/src/soap_ops.c b/org.glite.jp.index/src/soap_ops.c index 6c98b36..85f935b 100644 --- a/org.glite.jp.index/src/soap_ops.c +++ b/org.glite.jp.index/src/soap_ops.c @@ -8,6 +8,7 @@ #include "glite/jp/context.h" #include "glite/jp/attr.h" #include "glite/jp/known_attr.h" +#include "glite/jp/indexdb.h" #include "jp_H.h" #include "soap_version.h" @@ -361,7 +362,7 @@ static int get_jobids(glite_jpis_context_t ctx, struct _jpelem__QueryJobs *in, c attr_md5 = NULL; qa = strdup(""); } else { - attr_md5 = glite_jpis_attr_name2id(condition->attr); + attr_md5 = glite_jp_indexdb_attr2id(condition->attr); add_attr_table(attr_md5, &attr_tables); /* origin */ @@ -475,7 +476,7 @@ static int get_attr(struct soap *soap, glite_jpis_context_t ctx, char *jobid, ch memset(&jav,0,sizeof(jav)); jobid_md5 = str2md5(jobid); - attr_md5 = glite_jpis_attr_name2id(attr_name); + attr_md5 = glite_jp_indexdb_attr2id(attr_name); trio_asprintf(&query,"SELECT full_value FROM attr_%|Ss WHERE jobid = \"%s\"", attr_md5, jobid_md5); free(attr_md5); -- 1.8.2.3