From ea350d49bb1063c5b707667e029746bff2582cc1 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Ale=C5=A1=20K=C5=99enek?= Date: Tue, 10 Oct 2006 17:36:32 +0000 Subject: [PATCH] merge from 3.1 --- org.glite.jp.client/config/startup | 12 +- org.glite.jp.client/examples/glite-jp-importer.sh | 48 +- org.glite.jp.client/project/version.properties | 2 +- org.glite.jp.client/src/jpimporter.c | 47 +- org.glite.jp.common/interface/known_attr.h | 5 + org.glite.jp.common/interface/types.h | 1 + org.glite.jp.common/project/version.properties | 2 +- org.glite.jp.index/Makefile | 18 +- org.glite.jp.index/examples/jpis-test.c | 2 +- org.glite.jp.index/examples/query-tests/authz.out | 1 - .../examples/query-tests/complex_query.out | 21 +- .../examples/query-tests/run-test.sh | 30 +- .../examples/query-tests/simple_query.out | 12 +- org.glite.jp.index/project/version.properties | 2 +- org.glite.jp.index/src/conf.c | 84 + org.glite.jp.index/src/soap_ops.c | 79 +- org.glite.jp.index/src/ws_is_typeref.c | 2 +- org.glite.jp.primary/Makefile | 22 +- org.glite.jp.primary/examples/dag-deps.c | 215 +++ .../project/configure.properties.xml | 8 + org.glite.jp.primary/project/version.properties | 2 +- org.glite.jp.primary/src/attrs.c | 13 + org.glite.jp.primary/src/authz.c | 9 +- org.glite.jp.primary/src/bones_server.c | 3 +- org.glite.jp.primary/src/ftp_backend.c | 1744 -------------------- org.glite.jp.primary/src/new_ftp_backend.c | 5 +- org.glite.jp.primary/src/soap_ops.c | 51 +- 27 files changed, 585 insertions(+), 1855 deletions(-) create mode 100644 org.glite.jp.primary/examples/dag-deps.c delete mode 100644 org.glite.jp.primary/src/ftp_backend.c diff --git a/org.glite.jp.client/config/startup b/org.glite.jp.client/config/startup index 6e498fd..5d5173e 100755 --- a/org.glite.jp.client/config/startup +++ b/org.glite.jp.client/config/startup @@ -41,11 +41,21 @@ start() jpdump_maildir="--dump-mdir $GLITE_LB_EXPORT_JPDUMP_MAILDIR " [ -d "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" ] || mkdir "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" && chown $GLITE_USER:$GLITE_GROUP "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" [ -n "$GLITE_LB_EXPORT_JPPS" ] && jpps="--jpps $GLITE_LB_EXPORT_JPPS " + [ -n "$GLITE_LB_EXPORT_SANDBOX_MAILDIR" ] && sandbox_maildir="--sandbox-mdir $GLITE_LB_EXPORT_SANDBOX_MAILDIR " + if [ -n "$GLITE_LB_EXPORT_JOBSDIR_KEEP" ]; then + keep_jobs="--store ${GLITE_LB_EXPORT_JOBSDIR_KEEP} " + [ -d $GLITE_LB_EXPORT_JOBSDIR_KEEP ] || mkdir -p $GLITE_LB_EXPORT_JOBSDIR_KEEP + fi + echo -n Starting glite-jp-importer ... + +# XXX: HEAD +# -i $pidfile $jpreg_maildir $jpdump_maildir $jpps $sandbox_maildir $creds" \ + su - $GLITE_USER -c "$GLITE_LOCATION/bin/glite-jp-importer \ - -i $pidfile $jpreg_maildir $jpdump_maildir $jpps $sandbox_maildir $creds" \ + -i $pidfile $jpreg_maildir $jpdump_maildir $jpps $sandbox_maildir $keep_jobs $creds" \ && echo " done" || echo " FAILED" } diff --git a/org.glite.jp.client/examples/glite-jp-importer.sh b/org.glite.jp.client/examples/glite-jp-importer.sh index ef94018..edc5497 100644 --- a/org.glite.jp.client/examples/glite-jp-importer.sh +++ b/org.glite.jp.client/examples/glite-jp-importer.sh @@ -21,11 +21,9 @@ if [ -z "$GLITE_LB_EXPORT_JPPS" ]; then echo "Please specify the Job Provanance Primary Storage server." exit 1 fi -# bookkeeping server -if [ -z "$GLITE_LB_EXPORT_BKSERVER" ]; then - echo "Please specify the Book Keeping server." - exit 1 -fi +# book keeping server +GLITE_LB_SERVER_PORT=${GLITE_LB_SERVER_PORT:-9000} +GLITE_LB_EXPORT_BKSERVER=${GLITE_LB_EXPORT_BKSERVER:-localhost:$GLITE_LB_SERVER_PORT} # certificates if [ -z "$X509_USER_CERT" -o -z "$X509_USER_KEY" ]; then echo "Please set X509_USER_CERT and X509_USER_KEY." @@ -41,27 +39,49 @@ if [ -z "$GLITE_LB_EXPORT_JPREG_MAILDIR" ]; then GLITE_LB_EXPORT_JPREG_MAILDIR=/tmp/jpreg echo "GLITE_LB_EXPORT_JPREG_MAILDIR not specified (-J arguent of the bkserver), used $GLITE_LB_EXPORT_JPREG_MAILDIR" fi +# pidfile +[ -n "$GLITE_JP_IMPORTER_PIDFILE" ] && pidfile="-i $GLITE_JP_IMPORTER_PIDFILE " CERT_ARGS="-c $X509_USER_CERT -k $X509_USER_KEY" GLITE_LB_EXPORT_JPDUMP_MAILDIR=${GLITE_LB_EXPORT_JPDUMP_MAILDIR:-/tmp/jpdump} -GLITE_LB_EXPORT_DUMPDIR_OLD=${GLITE_LB_EXPORT_DUMPDIR_OLD:-$GLITE_LB_EXPORT_DUMPDIR.old} -GLITE_LB_EXPORT_EXPORTDIR=${GLITE_LB_EXPORT_EXPORTDIR:-/tmp/lbexport} + +# XXX: head +#GLITE_LB_EXPORT_DUMPDIR_OLD=${GLITE_LB_EXPORT_DUMPDIR_OLD:-$GLITE_LB_EXPORT_DUMPDIR.old} +#GLITE_LB_EXPORT_EXPORTDIR=${GLITE_LB_EXPORT_EXPORTDIR:-/tmp/lbexport} + GLITE_LB_EXPORT_SANDBOX_MAILDIR=${GLITE_LB_EXPORT_SANDBOX_MAILDIR:-/tmp/jpsandbox} + +# 3.1 +GLITE_LB_EXPORT_JOBSDIR=${GLITE_LB_EXPORT_JOBSDIR:-/tmp/lbexport} + PREFIX=${PREFIX:-`dirname $0`/..} LOGDIR=$GLITE_LOCATION_VAR -GLITE_LB_EXPORT_PURGE_ARGS=${GLITE_LB_EXPORT_PURGE_ARGS:--a 1h -c 1h -n 1h -o 1d} +GLITE_LB_EXPORT_PURGE_ARGS=${GLITE_LB_EXPORT_PURGE_ARGS:---cleared 2d --aborted 2w --cancelled 2w --other 2m} [ -d $GLITE_LB_EXPORT_JPDUMP_MAILDIR ] || mkdir -p $GLITE_LB_EXPORT_JPDUMP_MAILDIR [ -d $GLITE_LB_EXPORT_DUMPDIR ] || mkdir -p $GLITE_LB_EXPORT_DUMPDIR -[ -d $GLITE_LB_EXPORT_DUMPDIR_OLD ] || mkdir -p $GLITE_LB_EXPORT_DUMPDIR_OLD -[ -d $GLITE_LB_EXPORT_EXPORTDIR ] || mkdir -p $GLITE_LB_EXPORT_EXPORTDIR + +# XXX: head +#[ -d $GLITE_LB_EXPORT_DUMPDIR_OLD ] || mkdir -p $GLITE_LB_EXPORT_DUMPDIR_OLD +#[ -d $GLITE_LB_EXPORT_EXPORTDIR ] || mkdir -p $GLITE_LB_EXPORT_EXPORTDIR + [ -d $GLITE_LB_EXPORT_SANDBOX_MAILDIR ] || mkdir -p $GLITE_LB_EXPORT_SANDBOX_MAILDIR + +# 3.1 +[ -n "$GLITE_LB_EXPORT_DUMPDIR_KEEP" -a ! -d $GLITE_LB_EXPORT_DUMPDIR_KEEP ] && mkdir -p $GLITE_LB_EXPORT_DUMPDIR_KEEP +[ -d $GLITE_LB_EXPORT_JOBSDIR ] || mkdir -p $GLITE_LB_EXPORT_JOBSDIR +if [ -n "$GLITE_LB_EXPORT_JOBSDIR_KEEP" ]; then + keep_jobs="--store ${GLITE_LB_EXPORT_JOBSDIR_KEEP} " + [ -d $GLITE_LB_EXPORT_JOBSDIR_KEEP ] || mkdir -p $GLITE_LB_EXPORT_JOBSDIR_KEEP +fi + [ -d $LOGDIR ] || mkdir -p $LOGDIR echo "Using cert args $CERT_ARGS" $PREFIX/bin/glite-jp-importer --reg-mdir $GLITE_LB_EXPORT_JPREG_MAILDIR --dump-mdir $GLITE_LB_EXPORT_JPDUMP_MAILDIR $CERT_ARGS --sandbox-mdir $GLITE_LB_EXPORT_SANDBOX_MAILDIR -g --jpps $GLITE_LB_EXPORT_JPPS $pidfile$keep_jobs> $LOGDIR/jp-importer.log 2>&1 & + JP_PID=$! trap "kill $JP_PID; exit 0" SIGINT @@ -70,8 +90,12 @@ while [ 1 ]; do for file in $GLITE_LB_EXPORT_DUMPDIR/*; do if [ -s $file ]; then - $PREFIX/sbin/glite-lb-lb_dump_exporter -d $file -s $GLITE_LB_EXPORT_EXPORTDIR -m $GLITE_LB_EXPORT_JPDUMP_MAILDIR - mv $file $GLITE_LB_EXPORT_DUMPDIR_OLD + $PREFIX/sbin/glite-lb-lb_dump_exporter -d $file -s $GLITE_LB_EXPORT_JOBSDIR -m $GLITE_LB_EXPORT_JPDUMP_MAILDIR + if [ -n "$GLITE_LB_EXPORT_DUMPDIR_KEEP" ]; then + mv $file $GLITE_LB_EXPORT_DUMPDIR_KEEP + else + rm $file + fi else rm $file fi diff --git a/org.glite.jp.client/project/version.properties b/org.glite.jp.client/project/version.properties index 08e9de3..c31be1e 100644 --- a/org.glite.jp.client/project/version.properties +++ b/org.glite.jp.client/project/version.properties @@ -1,2 +1,2 @@ -module.version=1.1.0 +module.version=1.1.2 module.age=0 diff --git a/org.glite.jp.client/src/jpimporter.c b/org.glite.jp.client/src/jpimporter.c index 71173e5..bd71d11 100644 --- a/org.glite.jp.client/src/jpimporter.c +++ b/org.glite.jp.client/src/jpimporter.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "glite/lb/lb_maildir.h" #include "glite/security/glite_gsplugin.h" @@ -71,6 +72,7 @@ static char *jpps = GLITE_JPPS; static char reg_mdir[PATH_MAX] = GLITE_REG_IMPORTER_MDIR; static char dump_mdir[PATH_MAX] = GLITE_DUMP_IMPORTER_MDIR; static char sandbox_mdir[PATH_MAX] = GLITE_SANDBOX_IMPORTER_MDIR; +static char *store = NULL; static struct soap *soap; static time_t cert_mtime; @@ -93,6 +95,7 @@ static struct option opts[] = { { "sandbox-mdir",1, NULL, 's'}, { "pidfile", 1, NULL, 'i'}, { "poll", 1, NULL, 't'}, + { "store", 1, NULL, 's'}, { NULL, 0, NULL, 0} }; @@ -112,6 +115,7 @@ static void usage(char *me) "\t-s, --sandbox-mdir path to the 'LB maildir' subtree for input/output sandboxes\n" "\t-i, --pidfile file to store master pid\n" "\t-t, --poll maildir polling interval (in seconds)\n", + "\t-s, --store keep uploaded jobs in this directory\n", me); } @@ -164,6 +168,7 @@ int main(int argc, char *argv[]) case 'C': cadir = optarg; break; case 'p': jpps = optarg; break; case 't': poll = atoi(optarg); break; + case 's': store = optarg; break; case 'r': strcpy(reg_mdir, optarg); break; case 'd': strcpy(dump_mdir, optarg); break; case 's': strcpy(sandbox_mdir, optarg); break; @@ -196,6 +201,12 @@ int main(int argc, char *argv[]) edg_wll_MaildirInit(reg_mdir); edg_wll_MaildirInit(dump_mdir); edg_wll_MaildirInit(sandbox_mdir); + if (store && *store) { + if (mkdir(store, 0750) != 0 && errno != EEXIST) { + fprintf(stderr, "Can't create directory %s: %s\n", store, strerror(errno)); + store = NULL; + } + } if ( !debug ) { if ( daemon(1,0) == -1 ) { perror("deamon()"); exit(1); } @@ -424,7 +435,9 @@ static int dump_importer(void) static int readnew = 1; char *msg = NULL, *fname = NULL, - *aux; + *aux, + *bname; + char fspec[PATH_MAX]; int ret; int fhnd; msg_pattern_t tab[] = { @@ -440,13 +453,13 @@ static int dump_importer(void) if ( readnew ) ret = edg_wll_MaildirTransStart(dump_mdir, &msg, &fname); - else ret = edg_wll_MaildirRetryTransStart(dump_mdir, (time_t)60, &msg, &fname); + else ret = edg_wll_MaildirRetryTransStart(dump_mdir, (time_t)60, (time_t)600, &msg, &fname); if ( !ret ) { - readnew = ~readnew; + readnew = !readnew; if ( readnew ) ret = edg_wll_MaildirTransStart(dump_mdir, &msg, &fname); - else ret = edg_wll_MaildirRetryTransStart(dump_mdir, (time_t)60, &msg, &fname); + else ret = edg_wll_MaildirRetryTransStart(dump_mdir, (time_t)60, (time_t)600, &msg, &fname); if ( !ret ) { - readnew = ~readnew; + readnew = !readnew; return 0; } } @@ -474,10 +487,13 @@ static int dump_importer(void) dprintf(("[%s] Importing LB dump file '%s'\n", name, tab[_file].val)); if ( !debug ) syslog(LOG_INFO, "Importing LB dump file '%s'\n", msg); ret = soap_call___jpsrv__StartUpload(soap, tab[_jpps].val?:jpps, "", &su_in, &su_out); - ret = check_soap_fault(soap, ret); - /* XXX: grrrrrrr! test it!!!*/ -// if ( (ret = check_soap_fault(soap, ret)) ) break; + if ( (ret = check_soap_fault(soap, ret)) ) break; dprintf(("[%s] Destination: %s\n\tCommit before: %s\n", name, su_out.destination, ctime(&su_out.commitBefore))); + if (su_out.destination == NULL) { + dprintf(("[%s] StartUpload returned NULL destination\n", name)); + if ( !debug ) syslog(LOG_ERR, "StartUpload returned NULL destination"); + break; + } if ( (fhnd = open(tab[_file].val, O_RDONLY)) < 0 ) { dprintf(("[%s] Can't open dump file: %s\n", name, tab[_file].val)); @@ -492,6 +508,20 @@ static int dump_importer(void) ret = soap_call___jpsrv__CommitUpload(soap, tab[_jpps].val?:jpps, "", &cu_in, &empty); if ( (ret = check_soap_fault(soap, ret)) ) break; dprintf(("[%s] Dump upload succesfull\n", name)); + if (store && *store) { + bname = strdup(tab[_file].val); + snprintf(fspec, sizeof fspec, "%s/%s", store, basename(bname)); + free(bname); + if (rename(tab[_file].val, fspec) != 0) + fprintf(stderr, "moving %s to %s failed: %s\n", tab[_file].val, fspec, strerror(errno)); + else + dprintf(("[%s] moving %s to %s OK\n", name, tab[_file].val, fspec)); + } else { + if (unlink(tab[_file].val) != 0) + fprintf(stderr, "removing %s failed: %s\n", tab[_file].val, strerror(errno)); + else + dprintf(("[%s] %s removed\n", name, tab[_file].val)); + } } while (0); edg_wll_MaildirTransEnd(dump_mdir, fname, ret? LBMD_TRANS_FAILED_RETRY: LBMD_TRANS_OK); @@ -764,6 +794,7 @@ static int gftp_put_file(const char *url, int fhnd) globus_cond_init(&gCond, GLOBUS_NULL); gDone = GLOBUS_FALSE; + gError = GLOBUS_FALSE; /* do the op */ if ( globus_ftp_client_put( diff --git a/org.glite.jp.common/interface/known_attr.h b/org.glite.jp.common/interface/known_attr.h index d53c792..fafeb82 100644 --- a/org.glite.jp.common/interface/known_attr.h +++ b/org.glite.jp.common/interface/known_attr.h @@ -3,6 +3,7 @@ /** Namespace of JP system attributes */ #define GLITE_JP_SYSTEM_NS "http://egee.cesnet.cz/en/Schema/JP/System" +#define GLITE_JP_WORKFLOW_NS "http://egee.cesnet.cz/en/Schema/JP/Workflow" /** Job owner, as specified with RegisterJob JPPS operation */ #define GLITE_JP_ATTR_OWNER GLITE_JP_SYSTEM_NS ":owner" @@ -14,6 +15,10 @@ * Should be almost the same time as registration with LB. */ #define GLITE_JP_ATTR_REGTIME GLITE_JP_SYSTEM_NS ":regtime" +/** Workflow node relationships. */ +#define GLITE_JP_ATTR_WF_ANCESTOR GLITE_JP_WORKFLOW_NS ":ancestor" +#define GLITE_JP_ATTR_WF_SUCCESSOR GLITE_JP_WORKFLOW_NS ":successor" + /** Attributes derived from LB system data * \see jp_job_attrs.h */ diff --git a/org.glite.jp.common/interface/types.h b/org.glite.jp.common/interface/types.h index e86d4c9..e4208c1 100644 --- a/org.glite.jp.common/interface/types.h +++ b/org.glite.jp.common/interface/types.h @@ -22,6 +22,7 @@ typedef struct _glite_jp_context { void *dbhandle; char **trusted_peers; char *myURL; + int noauth; } *glite_jp_context_t; typedef enum { diff --git a/org.glite.jp.common/project/version.properties b/org.glite.jp.common/project/version.properties index 46eb5a1..bcf5d00 100644 --- a/org.glite.jp.common/project/version.properties +++ b/org.glite.jp.common/project/version.properties @@ -1,2 +1,2 @@ -module.version=1.2.0 +module.version=1.2.1 module.age=1 diff --git a/org.glite.jp.index/Makefile b/org.glite.jp.index/Makefile index 87a7776..871e9a7 100644 --- a/org.glite.jp.index/Makefile +++ b/org.glite.jp.index/Makefile @@ -22,7 +22,7 @@ CC=gcc -include Makefile.inc -VPATH=${top_srcdir}/src:${top_srcdir}/examples:${top_srcdir}/examples/query-tests:${top_srcdir}/tests:${top_srcdir}/test:${top_srcdir}/project:${top_srcdir}/doc:${jpproject}:${stagedir}/interface +VPATH=${top_srcdir}/src:${top_srcdir}/examples:${top_srcdir}/project:${top_srcdir}/doc:${jpproject}:${stagedir}/interface GLOBUS_LIBS:=-L${globus_prefix}/lib \ -lglobus_common_${nothrflavour} \ @@ -60,13 +60,25 @@ ps_prefix:=jpps_ SRCS:= conf.c bones_server.c soap_ops.c soap_ps_calls.c \ ws_ps_typeref.c ws_is_typeref.c db_ops.c context.c common_server.c \ +<<<<<<< Makefile ${is_prefix}Server.c ${is_prefix}C.c\ ${ps_prefix}ClientLib.c +======= + ${is_prefix}ServerLib.c \ + ${ps_prefix}ClientLib.c ${ps_prefix}C.c +>>>>>>> 1.32.2.3 +<<<<<<< Makefile EXA_TEST_SRCS:=jpis-test.c ${is_prefix}C.c ${is_prefix}Client.c context.c db_ops.c conf.c ws_is_typeref.c EXA_DB_SRCS:=jpis-db-internal.c db_ops.c conf.c context.c ws_is_typeref.c EXA_CLIENT_SRCS:=jpis-client.c common.c \ ${is_client_prefix}C.c ${is_client_prefix}Client.c +======= +EXA_TEST_SRCS:=jpis-test.c ${is_prefix}C.c ${is_prefix}Client.c context.c db_ops.c conf.c +EXA_DB_SRCS:=jpis-db-internal.c db_ops.c conf.c context.c +EXA_CLIENT_SRCS:=jpis-client.c common.c \ + ${is_client_prefix}C.c ${is_client_prefix}Client.c +>>>>>>> 1.32.2.3 # ${is_prefix}C.c OBJS:=${SRCS:.c=.o} @@ -97,7 +109,9 @@ glite-jpis-client: ${EXA_CLIENT_OBJS} ${LINK} -o $@ $+ ${GSOAPLIB} copy_tests: ${test} ${test_files} - -for i in $?; do cp $$i ${top_srcdir}/build; done + +${test} ${test_files}: + for i in $@; do cp ${top_srcdir}/examples/query-tests/$$i ${top_srcdir}/build; done JobProvenanceIS.xh: JobProvenanceIS.wsdl JobProvenanceTypes.wsdl typemap.dat cp ${stagedir}/interface/JobProvenanceTypes.wsdl . diff --git a/org.glite.jp.index/examples/jpis-test.c b/org.glite.jp.index/examples/jpis-test.c index b0a2783..ef926dd 100644 --- a/org.glite.jp.index/examples/jpis-test.c +++ b/org.glite.jp.index/examples/jpis-test.c @@ -47,7 +47,7 @@ int main(int argc,char *argv[]) glite_jp_init_context(&ctx); - glite_jp_get_conf(0, NULL, NULL, &conf); + glite_jp_get_conf(argc, argv, NULL, &conf); if (default_server) strcpy(server, default_server); else snprintf(server, sizeof(server), "http://localhost:%s", conf->port ? conf->port : GLITE_JPIS_DEFAULT_PORT_STR); printf("JP index server: %s\n", server); diff --git a/org.glite.jp.index/examples/query-tests/authz.out b/org.glite.jp.index/examples/query-tests/authz.out index 18e2d30..21458e0 100644 --- a/org.glite.jp.index/examples/query-tests/authz.out +++ b/org.glite.jp.index/examples/query-tests/authz.out @@ -10,5 +10,4 @@ Attributes: http://egee.cesnet.cz/en/Schema/LB/Attributes:user OK -Result 0 jobs: \ No newline at end of file diff --git a/org.glite.jp.index/examples/query-tests/complex_query.out b/org.glite.jp.index/examples/query-tests/complex_query.out index 65e14e9..bc11975 100644 --- a/org.glite.jp.index/examples/query-tests/complex_query.out +++ b/org.glite.jp.index/examples/query-tests/complex_query.out @@ -13,23 +13,4 @@ Attributes: http://egee.cesnet.cz/en/Schema/LB/Attributes:user OK -Result 2 jobs: - jobid = https://localhost:7846/pokus1, owner = (null) - http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus - value = Done - origin = 0, (null) - time = Thu Jan 1 01:00:00 1970 - http://egee.cesnet.cz/en/Schema/LB/Attributes:user - value = CertSubj - origin = 0, (null) - time = Thu Jan 1 01:00:00 1970 - jobid = https://localhost:7846/pokus2, owner = (null) - http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus - value = Ready - origin = 0, (null) - time = Thu Jan 1 01:00:00 1970 - http://egee.cesnet.cz/en/Schema/LB/Attributes:user - value = CertSubj - origin = 0, (null) - time = Thu Jan 1 01:00:00 1970 -https://localhost:7846/pokus1http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusDone1970-01-01T00:00:00ZSYSTEMhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1970-01-01T00:00:00ZSYSTEMhttp://localhost:8901https://localhost:7846/pokus2http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusReady1970-01-01T00:00:00ZSYSTEMhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1970-01-01T00:00:00ZSYSTEMhttp://localhost:8901 \ No newline at end of file +https://localhost:7846/pokus1/O=CESNET/O=Masaryk University/CN=Milos Mulachttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusDone1969-12-31T23:00:00ZSYSTEMhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1969-12-31T23:00:00ZSYSTEMhttp://localhost:8901https://localhost:7846/pokus2OwnerNamehttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusReady1969-12-31T23:00:00ZSYSTEMhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1969-12-31T23:00:00ZSYSTEMhttp://localhost:8901 \ No newline at end of file diff --git a/org.glite.jp.index/examples/query-tests/run-test.sh b/org.glite.jp.index/examples/query-tests/run-test.sh index e8fda6d..2e27c08 100755 --- a/org.glite.jp.index/examples/query-tests/run-test.sh +++ b/org.glite.jp.index/examples/query-tests/run-test.sh @@ -59,7 +59,7 @@ init() { # handle the configuration ARGS="-u ${GLITE_JPIS_TEST_ROOT_USER:-root}" - [ -z "$GLITE_JPIS_TEST_ROOT_PASSWORD" ] || ARGS="-p ${GLITE_JPIS_TEST_ROOT_PASSWORD} $ARGS" + [ -z "$GLITE_JPIS_TEST_ROOT_PASSWORD" ] || ARGS="--password=${GLITE_JPIS_TEST_ROOT_PASSWORD} $ARGS" GLITE_JPIS_TEST_PORT=${GLITE_JPIS_TEST_PORT:-"10000"} GLITE_JPIS_TEST_PIDFILE=${GLITE_JPIS_TEST_PIDFILE:-"/tmp/glite-jp-indexd.pid"} GLITE_JPIS_TEST_LOGFILE=${GLITE_JPIS_TEST_LOGFILE:-"/tmp/glite-jp-indexd.log"} @@ -80,7 +80,7 @@ create_db() { if [ "x$need_new_db" = "x1" ]; then mysqladmin -f $ARGS drop $DB_NAME > /dev/null 2>&1 mysqladmin -f $ARGS create $DB_NAME && \ - mysql $ARGS -e 'GRANT ALL on $DB_NAME.* to jpis@localhost' && \ + mysql $ARGS -e "GRANT ALL on $DB_NAME.* to jpis@localhost" && \ mysql -u $DB_USER $DB_NAME < $GLITE_LOCATION/etc/glite-jp-index-dbsetup.sql || exit 1 db_created="1" fi @@ -142,7 +142,7 @@ kill_is() { run_test_query() { X509_USER_KEY=${X509_USER_KEY} X509_USER_CERT=${X509_USER_CERT} \ - $GLITE_LOCATION/examples/glite-jpis-client -q $1 \ + $GLITE_LOCATION/examples/glite-jpis-client -f xml -q $1 \ -i http://localhost:$GLITE_JPIS_TEST_PORT &>/tmp/result DIFF=`diff --ignore-matching-lines="query: using JPIS" $2 /tmp/result` if [ -z "$DIFF" -a "$?" -eq "0" ] ; then @@ -151,11 +151,17 @@ run_test_query() { else echo "FAILED!" echo - echo "Expected result:" + echo "Expected result (using query $1):" + echo ------------------------------------------------------------------------------ cat $2 + echo + echo --------------------------------------------------------------------------------------------------- + echo echo "Obtained result (in /tmp/result):" + echo --------------------------------- cat /tmp/result echo + echo --------------------------------------------------------------------------------------------------- drop_db; kill_is; exit 1 @@ -165,9 +171,7 @@ run_test_query() { run_test_feed() { # run the example numok=`X509_USER_KEY=${X509_USER_KEY} X509_USER_CERT=${X509_USER_CERT}\ - GLITE_JPIS_DB=$GLITE_JPIS_TEST_DB \ - GLITE_JPIS_PORT=$GLITE_JPIS_TEST_PORT \ - $GLITE_LOCATION/examples/glite-jpis-test 2>/dev/null| grep "OK" | wc -l` + $GLITE_LOCATION/examples/glite-jpis-test -p $GLITE_JPIS_TEST_PORT -m $GLITE_JPIS_TEST_DB ` if [ "$numok" -eq "2" ]; then echo OK. else @@ -187,16 +191,16 @@ echo echo -n "Simple query test.... " create_db; run_is "-n"; -import_db $GLITE_LOCATION/examples/dump1.sql; -run_test_query $GLITE_LOCATION/examples/simple_query.in $GLITE_LOCATION/examples/simple_query.out; +import_db $GLITE_LOCATION/examples/query-tests/dump1.sql; +run_test_query $GLITE_LOCATION/examples/query-tests/simple_query.in $GLITE_LOCATION/examples/query-tests/simple_query.out; drop_db; kill_is; echo -n "Complex query test... " create_db; run_is "-n"; -import_db $GLITE_LOCATION/examples/dump1.sql; -run_test_query $GLITE_LOCATION/examples/complex_query.in $GLITE_LOCATION/examples/complex_query.out; +import_db $GLITE_LOCATION/examples/query-tests/dump1.sql; +run_test_query $GLITE_LOCATION/examples/query-tests/complex_query.in $GLITE_LOCATION/examples/query-tests/complex_query.out; drop_db; kill_is; @@ -210,8 +214,8 @@ kill_is; echo -n "Authz test........... " create_db; run_is; -import_db $GLITE_LOCATION/examples/dump1.sql; -run_test_query $GLITE_LOCATION/examples/simple_query.in $GLITE_LOCATION/examples/authz.out; +import_db $GLITE_LOCATION/examples/query-tests/dump1.sql; +run_test_query $GLITE_LOCATION/examples/query-tests/simple_query.in $GLITE_LOCATION/examples/query-tests/authz.out; drop_db; kill_is; diff --git a/org.glite.jp.index/examples/query-tests/simple_query.out b/org.glite.jp.index/examples/query-tests/simple_query.out index f63adf2..c3edb73 100644 --- a/org.glite.jp.index/examples/query-tests/simple_query.out +++ b/org.glite.jp.index/examples/query-tests/simple_query.out @@ -10,14 +10,4 @@ Attributes: http://egee.cesnet.cz/en/Schema/LB/Attributes:user OK -Result 1 jobs: - jobid = https://localhost:7846/pokus2, owner = (null) - http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus - value = Ready - origin = 0, (null) - time = Thu Jan 1 01:00:00 1970 - http://egee.cesnet.cz/en/Schema/LB/Attributes:user - value = CertSubj - origin = 0, (null) - time = Thu Jan 1 01:00:00 1970 -https://localhost:7846/pokus2http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusReady1970-01-01T00:00:00ZSYSTEMhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1970-01-01T00:00:00ZSYSTEMhttp://localhost:8901 \ No newline at end of file +https://localhost:7846/pokus2OwnerNamehttp://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusReady1969-12-31T23:00:00ZSYSTEMhttp://egee.cesnet.cz/en/Schema/LB/Attributes:userCertSubj1969-12-31T23:00:00ZSYSTEMhttp://localhost:8901 \ No newline at end of file diff --git a/org.glite.jp.index/project/version.properties b/org.glite.jp.index/project/version.properties index 46eb5a1..80cbe6b 100644 --- a/org.glite.jp.index/project/version.properties +++ b/org.glite.jp.index/project/version.properties @@ -1,2 +1,2 @@ -module.version=1.2.0 +module.version=1.2.2 module.age=1 diff --git a/org.glite.jp.index/src/conf.c b/org.glite.jp.index/src/conf.c index 27ee84a..4797579 100644 --- a/org.glite.jp.index/src/conf.c +++ b/org.glite.jp.index/src/conf.c @@ -102,6 +102,90 @@ int glite_jp_get_conf(int argc, char **argv, char *config_file, glite_jp_is_conf fprintf(stderr,"JP IS configuration file must be specified! "\ "Exiting.\n"); return 1; +/* hardcoded configuration from 3.1 */ +#if 0 + // prefixes & attributes defined in: + // lb.server/build/jp_job_attrs.h (created when build plugin) + // jp.common/interface/known_attr.h + + conf->attrs = calloc(29, sizeof(*conf->attrs)); + conf->attrs[0] = strdup("http://egee.cesnet.cz/en/Schema/JP/System:owner"); + conf->attrs[1] = strdup("http://egee.cesnet.cz/en/Schema/JP/System:jobId"); + conf->attrs[2] = strdup("http://egee.cesnet.cz/en/Schema/JP/System:regtime"); + conf->attrs[3] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:user"); + conf->attrs[4] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:aTag"); + conf->attrs[5] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:eNodes"); + conf->attrs[6] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:RB"); + conf->attrs[7] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:CE"); + conf->attrs[8] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:UIHost"); + conf->attrs[9] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:CPUTime"); + conf->attrs[10] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:NProc"); + conf->attrs[11] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus"); + conf->attrs[12] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatusDate"); + conf->attrs[13] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:retryCount"); + conf->attrs[14] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:jobType"); + conf->attrs[15] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:nsubjobs"); + conf->attrs[16] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:lastStatusHistory"); + conf->attrs[17] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:fullStatusHistory"); + conf->attrs[18] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:parent"); + conf->attrs[19] = strdup("http://egee.cesnet.cz/en/WSDL/jp-lbtag:IPAW_STAGE"); + conf->attrs[20] = strdup("http://egee.cesnet.cz/en/WSDL/jp-lbtag:IPAW_PROGRAM"); + conf->attrs[21] = strdup("http://egee.cesnet.cz/en/WSDL/jp-lbtag:IPAW_INPUT"); + conf->attrs[22] = strdup("http://egee.cesnet.cz/en/WSDL/jp-lbtag:IPAW_OUTPUT"); + conf->attrs[23] = strdup("http://egee.cesnet.cz/en/WSDL/jp-lbtag:IPAW_PARAM"); + conf->attrs[24] = strdup("http://egee.cesnet.cz/en/WSDL/jp-lbtag:IPAW_HEADER"); + conf->attrs[25] = strdup("http://egee.cesnet.cz/en/Schema/JP/Workflow:ancestor"); + conf->attrs[26] = strdup("http://egee.cesnet.cz/en/Schema/JP/Workflow:successor"); + conf->attrs[27] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:host"); + + conf->indexed_attrs = calloc(12, sizeof(*conf->indexed_attrs)); + conf->indexed_attrs[0] = strdup("http://egee.cesnet.cz/en/Schema/JP/System:owner"); + conf->indexed_attrs[1] = strdup("http://egee.cesnet.cz/en/Schema/JP/System:jobId"); + conf->indexed_attrs[2] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:user"); + conf->indexed_attrs[3] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus"); + conf->indexed_attrs[4] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:UIHost"); + conf->indexed_attrs[5] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:CE"); + conf->indexed_attrs[6] = strdup("http://egee.cesnet.cz/en/Schema/LB/Attributes:RB"); + conf->indexed_attrs[7] = strdup("http://egee.cesnet.cz/en/WSDL/jp-lbtag:IPAW_PROGRAM"); + conf->indexed_attrs[8] = strdup("http://egee.cesnet.cz/en/WSDL/jp-lbtag:IPAW_OUTPUT"); + conf->indexed_attrs[9] = strdup("http://egee.cesnet.cz/en/Schema/JP/Workflow:successor"); + conf->indexed_attrs[10] = strdup("http://egee.cesnet.cz/en/Schema/JP/Workflow:ancestor"); + + // XXX: some plugin names should come here in future + conf->plugins = NULL; + + if (!ps) { + // No JP PrimaryStrorage server specified in $GLITE_JPIS_PS -> skip feeds + conf->feeds = calloc(1, sizeof(*(conf->feeds))); + *configuration = conf; + return 0; + } + + /* ask for one feed */ + conf->feeds = calloc(2, sizeof(*(conf->feeds))); + + conf->feeds[0] = calloc(1, sizeof(**(conf->feeds))); + conf->feeds[0]->PS_URL = strdup(ps); + + // all job since Epoche + conf->feeds[0]->query = calloc(2,sizeof(*conf->feeds[0]->query)); + conf->feeds[0]->query[0] = calloc(2,sizeof(**conf->feeds[0]->query)); + conf->feeds[0]->query[0][0].attr = strdup("http://egee.cesnet.cz/en/Schema/JP/System:regtime"); + conf->feeds[0]->query[0][0].op = GLITE_JP_QUERYOP_GREATER; + conf->feeds[0]->query[0][0].value = strdup("0"); + + if (qt && !strcmp(qt,"both")) { + conf->feeds[0]->history = 1; + conf->feeds[0]->continuous = 1; + } + else if ( qt && (!strcmp(qt,"continuous") || !strcmp(qt,"cont")) ) { + conf->feeds[0]->history = 0; + conf->feeds[0]->continuous = 1; + } + else if ( qt && (!strcmp(qt,"history") || !strcmp(qt,"hist")) ) { + conf->feeds[0]->history = 1; + conf->feeds[0]->continuous = 0; +#endif } else { read_conf(conf, conf_file); diff --git a/org.glite.jp.index/src/soap_ops.c b/org.glite.jp.index/src/soap_ops.c index 28d4a42..123fc2e 100644 --- a/org.glite.jp.index/src/soap_ops.c +++ b/org.glite.jp.index/src/soap_ops.c @@ -249,43 +249,66 @@ static int get_jobids(struct soap *soap, glite_jpis_context_t ctx, struct _jpele qwhere = strdup(""); for (i=0; i < in->__sizeconditions; i++) { - attr_md5 = str2md5(in->conditions[i]->attr); - trio_asprintf(&qa,"%s jobs.jobid = attr_%|Ss.jobid AND (", - (i ? "AND" : ""), attr_md5); - - for (j=0; j < in->conditions[i]->__sizerecord; j++) { - if (get_op(in->conditions[i]->record[j]->op, &qop)) goto err; - add_attr_table(attr_md5, &attr_tables); +/* XXX: deal with jobIds diferently (they are not in attr_X table, but in jobs) + if (strcmp(in->conditions[i]->attr,GLITE_JP_ATTR_JOBID)) { + trio_asprintf(&qa,"%s (", (i ? "AND" : "") ); + + for (j=0; j < in->conditions[i]->__sizerecord; j++) { + if (get_op(in->conditions[i]->record[j]->op, &qop)) goto err; - if (in->conditions[i]->record[j]->value->string) { attr.name = in->conditions[i]->attr; attr.value = in->conditions[i]->record[j]->value->string; attr.binary = 0; glite_jpis_SoapToAttrOrig(soap, in->conditions[i]->origin, &(attr.origin)); - trio_asprintf(&qb,"%s%sattr_%|Ss.value %s \"%|Ss\"", + trio_asprintf(&qb,"%s%sjobs.dg_jobid %s \"%|Ss\"", qa, (j ? " OR " : ""), attr_md5, qop, glite_jp_attrval_to_db_index(ctx->jpctx, &attr, 255)); free(qop); free(qa); qa = qb; qb = NULL; + + } + else +*/ + { + attr_md5 = str2md5(in->conditions[i]->attr); + trio_asprintf(&qa,"%s jobs.jobid = attr_%|Ss.jobid AND (", + (i ? "AND" : ""), attr_md5); + + for (j=0; j < in->conditions[i]->__sizerecord; j++) { + if (get_op(in->conditions[i]->record[j]->op, &qop)) goto err; + add_attr_table(attr_md5, &attr_tables); + + if (in->conditions[i]->record[j]->value->string) { + attr.name = in->conditions[i]->attr; + attr.value = in->conditions[i]->record[j]->value->string; + attr.binary = 0; + glite_jpis_SoapToAttrOrig(soap, + in->conditions[i]->origin, &(attr.origin)); + trio_asprintf(&qb,"%s%sattr_%|Ss.value %s \"%|Ss\"", + qa, (j ? " OR " : ""), attr_md5, qop, + glite_jp_attrval_to_db_index(ctx->jpctx, &attr, 255)); + free(qop); + free(qa); qa = qb; qb = NULL; + } + else { + attr.name = in->conditions[i]->attr; + attr.value = in->conditions[i]->record[j]->value->blob->__ptr; + attr.binary = 1; + attr.size = in->conditions[i]->record[j]->value->blob->__size; + glite_jpis_SoapToAttrOrig(soap, + in->conditions[i]->origin, &(attr.origin)); + trio_asprintf(&qb,"%s %s attr_%|Ss.value %s \"%|Ss\"", + qa, (j ? "OR" : ""), attr_md5, qop, + glite_jp_attrval_to_db_index(ctx->jpctx, &attr, 255)); + free(qop); + free(qa); qa = qb; qb = NULL; + } } - else { - attr.name = in->conditions[i]->attr; - attr.value = in->conditions[i]->record[j]->value->blob->__ptr; - attr.binary = 1; - attr.size = in->conditions[i]->record[j]->value->blob->__size; - glite_jpis_SoapToAttrOrig(soap, - in->conditions[i]->origin, &(attr.origin)); - trio_asprintf(&qb,"%s %s attr_%|Ss.value %s \"%|Ss\"", - qa, (j ? "OR" : ""), attr_md5, qop, - glite_jp_attrval_to_db_index(ctx->jpctx, &attr, 255)); - free(qop); - free(qa); qa = qb; qb = NULL; - } + trio_asprintf(&qb,"%s %s)", qwhere, qa); + free(qa); qwhere = qb; qb = NULL; qa = NULL; + free(attr_md5); } - trio_asprintf(&qb,"%s %s)", qwhere, qa); - free(qa); qwhere = qb; qb = NULL; qa = NULL; - free(attr_md5); } qa = strdup(""); @@ -373,6 +396,7 @@ static int get_attr(struct soap *soap, glite_jpis_context_t ctx, char *jobid, ch glite_jp_db_stmt_t stmt; + memset(&jav,0,sizeof(jav)); jobid_md5 = str2md5(jobid); attr_md5 = str2md5(attr_name); trio_asprintf(&query,"SELECT full_value FROM attr_%|Ss WHERE jobid = \"%s\"", @@ -380,7 +404,10 @@ static int get_attr(struct soap *soap, glite_jpis_context_t ctx, char *jobid, ch free(attr_md5); free(jobid_md5); - if ((ret = glite_jp_db_execstmt(ctx->jpctx, query, &stmt)) < 0) goto err; + if ((ret = glite_jp_db_execstmt(ctx->jpctx, query, &stmt)) < 0) + // unknown attribute + // XXX: propagate the error to client + goto err; free(query); i = 0; diff --git a/org.glite.jp.index/src/ws_is_typeref.c b/org.glite.jp.index/src/ws_is_typeref.c index c51c98c..94aaad2 100644 --- a/org.glite.jp.index/src/ws_is_typeref.c +++ b/org.glite.jp.index/src/ws_is_typeref.c @@ -80,7 +80,7 @@ static int SoapToQueryCond( glite_jp_query_rec_t **out) { glite_jp_query_rec_t *qr; - int i; + int i; assert(in); assert(out); diff --git a/org.glite.jp.primary/Makefile b/org.glite.jp.primary/Makefile index 822a0c4..9b2efe0 100644 --- a/org.glite.jp.primary/Makefile +++ b/org.glite.jp.primary/Makefile @@ -32,7 +32,7 @@ GLOBUS_CFLAGS:=-I${globus_prefix}/include/${nothrflavour} DEBUG:=-g -O0 -DDEBUG -CFLAGS:=${DEBUG} -I. -I${top_srcdir}/interface -I${top_srcdir}/src -I${gsoap_prefix}/include -I${gsoap_prefix} -I${stagedir}/include ${GLOBUS_CFLAGS} -I${mysql_prefix}/include -I${mysql_prefix}/include/mysql +CFLAGS:=${DEBUG} -I. -I${top_srcdir}/interface -I${top_srcdir}/src -I${gsoap_prefix}/include -I${gsoap_prefix} -I${stagedir}/include ${GLOBUS_CFLAGS} -I${mysql_prefix}/include -I${mysql_prefix}/include/mysql -I${classads_prefix}/include LDFLAGS:=-L${stagedir}/lib LINK:=libtool --mode=link ${CC} ${LDFLAGS} @@ -41,8 +41,12 @@ SOLINK:=libtool --mode=link ${CC} -module ${LDFLAGS} -rpath ${stagedir}/lib LINKXX:=libtool --mode=link ${CXX} ${LDFLAGS} INSTALL:=libtool --mode=install install +ifneq (${classads_prefix},/usr) + classadslib := -L${classads_prefix}/lib -lclassad +endif + daemon:=glite-jp-primarystoraged -example:=jpps-test +example:=jpps-test dag-deps ps_prefix:=jpps_ is_prefix:=jpis_ sample_jobs:=sample_job_aborted sample_job_cleared sample_job_tagged_done sample_job_waiting @@ -61,12 +65,14 @@ SRCS:= bones_server.c soap_ops.c \ # ${ps_prefix}ServerLib.c \ # ${is_prefix}ClientLib.c jpps_C.c -EXA_SRCS:=jpps-test.c ${ps_prefix}C.c ${ps_prefix}Client.c +TEST_SRCS:=jpps-test.c ${ps_prefix}C.c ${ps_prefix}Client.c +DAG_SRCS:=dag-deps.c ${ps_prefix}C.c ${ps_prefix}Client.c gsoap_bin_prefix:=${shell if [ -x ${gsoap_prefix}/bin/soapcpp2 ]; then echo ${gsoap_prefix}/bin; else echo ${gsoap_prefix}; fi } OBJS:=${SRCS:.c=.o} -EXA_OBJS:=${EXA_SRCS:.c=.o} +TEST_OBJS:=${TEST_SRCS:.c=.o} +DAG_OBJS:=${DAG_SRCS:.c=.o} dotless_soap_ver:=${shell echo ${gsoap_version} | tr -d . } COMMONLIB:=-lglite_jp_common @@ -91,8 +97,12 @@ compile: ${daemon} ${example} ${plugins} ${daemon}: ${OBJS} ${LINK} -o $@ -export-dynamic ${OBJS} ${BONESLIB} ${TRIOLIB} ${COMMONLIB} ${GSOAPLIB} ${GLOBUS_LIBS} ${MYSQLIB} -${example}: ${EXA_OBJS} - ${LINK} -o $@ ${EXA_OBJS} ${GSOAPLIB} ${GLOBUS_LIBS} +jpps-test: ${TEST_OBJS} + ${LINK} -o $@ ${TEST_OBJS} ${GSOAPLIB} + +dag-deps: ${DAG_OBJS} + ${LINKXX} -o $@ ${DAG_OBJS} ${classadslib} ${GSOAPLIB} + JobProvenancePS.xh: %.xh: %.wsdl JobProvenanceTypes.wsdl typemap.dat cp ${stagedir}/interface/JobProvenanceTypes.wsdl . diff --git a/org.glite.jp.primary/examples/dag-deps.c b/org.glite.jp.primary/examples/dag-deps.c new file mode 100644 index 0000000..c68c13b --- /dev/null +++ b/org.glite.jp.primary/examples/dag-deps.c @@ -0,0 +1,215 @@ +#define _GNU_SOURCE /* strndup */ + +#include +#include + +#include +#include +#include + +#include + +#include "glite/security/glite_gsplugin.h" +#include "glite/jp/known_attr.h" + +#include "jpps_H.h" +#include "jpps_.nsmap" + +#include "jptype_map.h" + +#include "soap_version.h" +#if GSOAP_VERSION <= 20602 +#define soap_call___jpsrv__RegisterJob soap_call___ns1__RegisterJob +#define soap_call___jpsrv__StartUpload soap_call___ns1__StartUpload +#define soap_call___jpsrv__CommitUpload soap_call___ns1__CommitUpload +#define soap_call___jpsrv__RecordTag soap_call___ns1__RecordTag +#define soap_call___jpsrv__FeedIndex soap_call___ns1__FeedIndex +#define soap_call___jpsrv__FeedIndexRefresh soap_call___ns1__FeedIndexRefresh +#define soap_call___jpsrv__GetJob soap_call___ns1__GetJob +#endif + + +static void usage(const char *me) +{ + fprintf(stderr,"%s: [-s server-url] jobid\n",me); + + exit (EX_USAGE); +} + +static int check_fault(struct soap *soap,int err) { + struct SOAP_ENV__Detail *detail; + struct jptype__genericFault *f; + char *reason,indent[200] = " "; + + switch(err) { + case SOAP_OK: puts("OK"); + break; + case SOAP_FAULT: + case SOAP_SVR_FAULT: + if (soap->version == 2) { + detail = soap->fault->SOAP_ENV__Detail; + reason = soap->fault->SOAP_ENV__Reason; + } + else { + detail = soap->fault->detail; + reason = soap->fault->faultstring; + } + fputs(reason,stderr); + putc('\n',stderr); + assert(detail->__type == SOAP_TYPE__genericFault); +#if GSOAP_VERSION >=20700 + f = ((struct _genericFault *) detail->fault) +#else + f = ((struct _genericFault *) detail->value) +#endif + -> jpelem__genericFault; + + while (f) { + fprintf(stderr,"%s%s: %s (%s)\n",indent, + f->source,f->text,f->description); + f = f->reason; + strcat(indent," "); + } + return -1; + + default: soap_print_fault(soap,stderr); + return -1; + } + return 0; +} + +static const char *orig2str(enum jptype__attrOrig orig) +{ + switch (orig) { + case jptype__attrOrig__SYSTEM: return "SYSTEM"; + case jptype__attrOrig__USER: return "USER"; + case jptype__attrOrig__FILE_: return "FILE"; + default: return "unknown"; + } +} + + +int main(int argc,char *argv[]) +{ + char *server = NULL; + int opt,ret = 0,i; + struct soap *soap = soap_new(); + struct _jpelem__GetJobAttributes in; + struct _jpelem__GetJobAttributesResponse out; + char *aname = "http://egee.cesnet.cz/en/Schema/LB/Attributes:JDL"; + struct cclassad *ad; + struct { char *a,*s; } *deps = calloc(1,sizeof *deps); + int ndeps = 0; + char *dep_s,*where,*end,*tmp_a,*tmp_s,*wa,*wa_r,*ws,*ws_r; + + if (argc < 2) usage(argv[0]); + + soap_init(soap); + soap_set_namespaces(soap, jpps__namespaces); + + soap_register_plugin(soap,glite_gsplugin); + + while ((opt = getopt(argc,argv,"s:")) >= 0) switch (opt) { + case 's': server = optarg; + break; + case '?': usage(argv[0]); + } + + if (server) { + argv += 2; + argc -= 2; + } + else server = "http://localhost:8901"; + + + + in.jobid = argv[1]; + in.__sizeattributes = 1; + in.attributes = &aname; + + puts("Retrieving JDL ..."); + if ((ret = check_fault(soap,soap_call___jpsrv__GetJobAttributes(soap,server,"",&in,&out)))) + return 1; + + ad = cclassad_create(out.attrValues[0]->value->string); + if (!ad) { + fputs("Can't parse JDL\n",stderr); + return 1; + } + + // cclassad_evaluate_to_string(ad,"dependencies",&dep_s); + cclassad_evaluate_to_expr(ad,"dependencies",&dep_s); + + /* XXX: assumes syntacticly correct dependencies = { ... } */ + where = strchr(dep_s,'{'); assert(where); + where++; + + while ((where = strchr(where, '{'))) { /* 2nd level */ + for (where++; isspace(*where); where++); + + if (*where == '{') end = strchr(where, '}')+1; /* more ancestors */ + else for (end = where; !isspace(*end) && *end != ','; end++); + tmp_a = strndup(where,end - where); + where = end++; + + while(isspace(*where)) where++; + where++; /* comma */ + while(isspace(*where)) where++; + + if (*where == '{') end = strchr(where, '}')+1; /* more successors */ + else for (end = where; !isspace(*end) && *end != ','; end++); + tmp_s = strndup(where,end - where); + where = strchr(end+1,'}'); + +#define DELIM "{} ,\t\n" + for (ws = strtok_r(tmp_s,DELIM,&ws_r); ws; ws = strtok_r(NULL,DELIM,&ws_r)) + for (wa = strtok_r(tmp_a,DELIM,&wa_r); wa; wa = strtok_r(NULL,DELIM,&wa_r)) { + deps[ndeps].a = strdup(wa); + deps[ndeps].s = strdup(ws); + deps = realloc(deps, (ndeps+2) * sizeof *deps); + ndeps++; + deps[ndeps].a = deps[ndeps].s = NULL; + } + free(tmp_a); free(tmp_s); + } + + for (i=0; deps[i].a; i++) { + char attr[1000],*ja,*js; + int have_a,have_s; + + printf("node: %s -> %s\n",deps[i].a,deps[i].s); + sprintf(attr,"nodes.%s.description.edg_jobid",deps[i].a); + have_a = cclassad_evaluate_to_string(ad,attr,&ja); + + sprintf(attr,"nodes.%s.description.edg_jobid",deps[i].s); + have_s = cclassad_evaluate_to_string(ad,attr,&js); + + printf("jobid: %s -> %s\n",ja,js); + + if (have_a && have_s) { + struct _jpelem__RecordTag in; + struct _jpelem__RecordTagResponse empty; + struct jptype__tagValue tagval; + struct jptype__stringOrBlob val; + + in.jobid = ja; + in.tag = &tagval; + tagval.name = GLITE_JP_ATTR_WF_SUCCESSOR; + tagval.value = &val; + val.string = js; + val.blob = NULL; + + printf("Register successor ...\n"); + ret = check_fault(soap,soap_call___jpsrv__RecordTag(soap, server, "",&in, &empty)); + in.jobid = js; + tagval.name = GLITE_JP_ATTR_WF_ANCESTOR; + val.string = ja; + + printf("Register ancestor ...\n"); + ret = check_fault(soap,soap_call___jpsrv__RecordTag(soap, server, "",&in, &empty)); + putchar(10); + } + } + + return ret; +} diff --git a/org.glite.jp.primary/project/configure.properties.xml b/org.glite.jp.primary/project/configure.properties.xml index be55b62..53515a6 100644 --- a/org.glite.jp.primary/project/configure.properties.xml +++ b/org.glite.jp.primary/project/configure.properties.xml @@ -12,6 +12,13 @@ Revision history: $Log$ + Revision 1.6.4.1 2006/08/28 18:49:23 akrenek + pch06 tool to generate subjob relationships from DAG JDL + + Revision 1.6 2006/01/16 17:20:27 mmulac + merge from RC15 branch + - compiles + Revision 1.5.2.1 2005/11/03 17:46:53 mmulac ares to c-ares migration - use dynamic library of c-ares, instead of ares static lib @@ -68,6 +75,7 @@ nothrflavour=${with.globus.nothr.flavor} cppunit=${with.cppunit.prefix} jpproject=${subsystem.project.dir} project=${component.project.dir} +classads_prefix=${with.classads.prefix} diff --git a/org.glite.jp.primary/project/version.properties b/org.glite.jp.primary/project/version.properties index 46eb5a1..80cbe6b 100644 --- a/org.glite.jp.primary/project/version.properties +++ b/org.glite.jp.primary/project/version.properties @@ -1,2 +1,2 @@ -module.version=1.2.0 +module.version=1.2.2 module.age=1 diff --git a/org.glite.jp.primary/src/attrs.c b/org.glite.jp.primary/src/attrs.c index 044b610..7099dd1 100644 --- a/org.glite.jp.primary/src/attrs.c +++ b/org.glite.jp.primary/src/attrs.c @@ -1,6 +1,7 @@ #include #include #include +#include #include #include "glite/jp/types.h" @@ -161,10 +162,22 @@ glite_jpps_get_attrs(glite_jp_context_t ctx,const char *job,char **attr,int natt } p->ops.close(p->fpctx,ph); } + else { + char *e; + fprintf(stderr,"[%d] %s: %s\n",getpid(),known_classes[ci].class, + e = glite_jp_error_chain(ctx)); + free(e); + } } glite_jppsbe_close_file(ctx,beh); } + else { + char *e; + fprintf(stderr,"[%d] %s: %s\n",getpid(),known_classes[ci].class, + e = glite_jp_error_chain(ctx)); + free(e); + } } nout = merge_attrvals(&out,nout,meta); diff --git a/org.glite.jp.primary/src/authz.c b/org.glite.jp.primary/src/authz.c index 3ceba02..39451d4 100644 --- a/org.glite.jp.primary/src/authz.c +++ b/org.glite.jp.primary/src/authz.c @@ -31,8 +31,14 @@ int glite_jpps_authz(glite_jp_context_t ctx,int op,const char *job,const char *o case SOAP_TYPE___jpsrv__GetJobFiles: case SOAP_TYPE___jpsrv__GetJobAttributes: + case SOAP_TYPE___jpsrv__RecordTag: assert(owner); - return strcmp(owner,ctx->peer) ? glite_jp_stack_error(ctx,&err) : 0; + if (!ctx->noauth && strcmp(owner,ctx->peer)) { + err.desc = "you are not a job owner"; + glite_jp_stack_error(ctx,&err); + return 1; + } + return 0; break; default: @@ -76,3 +82,4 @@ int glite_jpps_readauth(glite_jp_context_t ctx,const char *file) fclose(f); return 0; } + diff --git a/org.glite.jp.primary/src/bones_server.c b/org.glite.jp.primary/src/bones_server.c index 9bfe779..49d87ad 100644 --- a/org.glite.jp.primary/src/bones_server.c +++ b/org.glite.jp.primary/src/bones_server.c @@ -71,7 +71,7 @@ int main(int argc, char *argv[]) b_argc = p_argc = 1; - while ((opt = getopt(argc,argv,"B:P:a:p:s:dl:i:c:k:")) != EOF) switch (opt) { + while ((opt = getopt(argc,argv,"B:P:a:p:s:dl:i:c:k:n")) != EOF) switch (opt) { case 'B': assert(b_argc < 20); if (com = strchr(optarg,',')) *com = 0; @@ -106,6 +106,7 @@ int main(int argc, char *argv[]) case 'i': strncpy(pidfile,optarg,PATH_MAX); pidfile[PATH_MAX-1] = 0; break; case 'c': server_cert = optarg; break; case 'k': server_key = optarg; break; + case 'n': ctx->noauth = 1; break; case '?': fprintf(stderr,"usage: %s: -Bb,val ... -Pplugin.so ...\n" "b is backend option\n",argv[0]); exit (1); diff --git a/org.glite.jp.primary/src/ftp_backend.c b/org.glite.jp.primary/src/ftp_backend.c deleted file mode 100644 index 8bf523b..0000000 --- a/org.glite.jp.primary/src/ftp_backend.c +++ /dev/null @@ -1,1744 +0,0 @@ -#ident "$Header$" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "glite/jp/types.h" -#include "glite/jp/context.h" -#include "glite/jp/strmd5.h" - -#include "tags.h" -#include "backend.h" - -#define UPLOAD_SUFFIX ".upload" -#define LOCK_SUFFIX ".lock" - -struct ftpbe_config { - char *internal_path; - char *external_path; - char *gridmap; - char *logname; -}; - -static struct ftpbe_config *config = NULL; - -struct fhandle_rec { - int fd; - int fd_append; -}; -typedef struct fhandle_rec *fhandle; - -static struct option ftpbe_opts[] = { - { "ftp-internal-path", 1, NULL, 'I' }, - { "ftp-external-path", 1, NULL, 'E' }, - { "ftp-gridmap", 1, NULL, 'G' }, - { NULL, 0, NULL, 0 } -}; - -/* obsolete */ -#if 0 -static struct { - glite_jp_fileclass_t type; - char * fname; - } class_to_fname_tab[] = { - { GLITE_JP_FILECLASS_INPUT, "input" }, - { GLITE_JP_FILECLASS_OUTPUT, "output" }, - { GLITE_JP_FILECLASS_LBLOG, "lblog" }, - { GLITE_JP_FILECLASS_TAGS, "tags" }, - { GLITE_JP_FILECLASS_UNDEF, NULL } - }; - -static char *class_to_fname(glite_jp_fileclass_t type) -{ - int i; - - for (i = 0; class_to_fname_tab[i].type != GLITE_JP_FILECLASS_UNDEF; i++) - if (type == class_to_fname_tab[i].type) - return class_to_fname_tab[i].fname; - - return NULL; -} - -static glite_jp_fileclass_t fname_to_class(char* fname) -{ - int i; - - for (i = 0; class_to_fname_tab[i].type != GLITE_JP_FILECLASS_UNDEF; i++) - if (!strcmp(fname, class_to_fname_tab[i].fname)) - return class_to_fname_tab[i].type; - - return GLITE_JP_FILECLASS_UNDEF; -} -#endif - -static int config_check( - glite_jp_context_t ctx, - struct ftpbe_config *config) -{ - return config == NULL || - config->internal_path == NULL || - config->external_path == NULL || - config->gridmap == NULL || - config->logname == NULL; - - /* XXX check reality */ -} - -static int jobid_unique_pathname(glite_jp_context_t ctx, const char *job, - char **unique, char **ju_path, int get_path) -{ - char *p; - glite_jp_error_t err; - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - p = strrchr(job, '/'); - if (!p) { - err.code = EINVAL; - err.desc = "Malformed jobid"; - return glite_jp_stack_error(ctx,&err); - } - /* XXX thorough checks */ - if (!(*unique = strdup(p+1))) { - err.code = ENOMEM; - return glite_jp_stack_error(ctx,&err); - } - if (get_path) { - if (!(*ju_path = strdup(p+1))) { - free(*unique); - err.code = ENOMEM; - return glite_jp_stack_error(ctx,&err); - } - *(*ju_path + 10) = '\0'; - } - return 0; -} - -static int mkdirpath(const char* path, int prefixlen) -{ - char *wpath, *p; - int goout, ret; - - wpath = strdup(path); - if (!wpath) { - errno = ENOMEM; - return -1; - } - - p = wpath + prefixlen; - goout = 0; - while (!goout) { - while (*p == '/') p++; - while (*p != '/' && *p != '\0') p++; - goout = (*p == '\0'); - *p = '\0'; - ret = mkdir(wpath, S_IRUSR | S_IWUSR | S_IXUSR); - if (ret < 0 && errno != EEXIST) break; - *p = '/'; - } - free(wpath); - return goout ? 0 : ret; -} - -static long regtime_trunc(long tv_sec) -{ - return tv_sec / (86400*7); -} - -static long regtime_ceil(long tv_sec) -{ - return (tv_sec % (86400*7)) ? tv_sec/(86400*7)+1 : tv_sec/(86400*7) ; -} - -/********************************************************************************/ -int glite_jppsbe_init( - glite_jp_context_t ctx, - int argc, - char *argv[] -) -{ - glite_jp_error_t err; - int opt; - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - config = (struct ftpbe_config *) calloc(1, sizeof *config); - if (!config) { - err.code = ENOMEM; - return glite_jp_stack_error(ctx,&err); - } - - config->logname = getlogin(); - - while ((opt = getopt_long(argc, argv, "I:E:G:", ftpbe_opts, NULL)) != EOF) { - switch (opt) { - case 'I': config->internal_path = optarg; break; - case 'E': config->external_path = optarg; break; - case 'G': config->gridmap = optarg; break; - default: break; - } - } - - if (config_check(ctx, config)) { - err.code = EINVAL; - err.desc = "Invalid FTP backend configuration"; - return glite_jp_stack_error(ctx,&err); - } - - return 0; -} - -int glite_jppsbe_init_slave( - glite_jp_context_t ctx -) -{ - /* Nothing to do */ -} - -int glite_jppsbe_register_job( - glite_jp_context_t ctx, - const char *job, - const char *owner -) -{ - glite_jp_error_t err; - char *int_dir = NULL; - char *int_fname = NULL; - char *data_dir = NULL; - char *data_fname = NULL; - char *ju = NULL; - char *ju_path = NULL; - char *ownerhash = NULL; - FILE *regfile = NULL; - struct timeval reg_tv; - long reg_tv_trunc; - struct stat statbuf; - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - assert(job != NULL); - assert(owner != NULL); - - gettimeofday(®_tv, NULL); - reg_tv_trunc = regtime_trunc(reg_tv.tv_sec); - - if (jobid_unique_pathname(ctx, job, &ju, &ju_path, 1) != 0) { - err.code = ctx->error->code; - err.desc = "Cannot obtain jobid unique path/name"; - return glite_jp_stack_error(ctx,&err); - } - - if (asprintf(&int_dir, "%s/regs/%s", - config->internal_path, ju_path) == -1) { - err.code = ENOMEM; - return glite_jp_stack_error(ctx,&err); - } - - if (mkdirpath(int_dir, strlen(config->internal_path)) < 0 && - errno != EEXIST) { - free(int_dir); - err.code = errno; - err.desc = "Cannot mkdir jobs's reg directory"; - return glite_jp_stack_error(ctx,&err); - } - free(int_dir); - - if (asprintf(&int_fname, "%s/regs/%s/%s.info", - config->internal_path, ju_path, ju) == -1) { - err.code = ENOMEM; - return glite_jp_stack_error(ctx,&err); - } - - if (stat(int_fname, &statbuf) < 0) { - if (errno != ENOENT) { - err.code = errno; - err.desc = "Cannot stat jobs's reg info file"; - goto error_out; - } - } else { - err.code = EEXIST; - err.desc = "Job already registered"; - goto error_out; - } - - regfile = fopen(int_fname, "w"); - if (regfile == NULL) { - err.code = errno; - err.desc = "Cannot open jobs's reg info file"; - goto error_out; - } - - ownerhash = str2md5(owner); /* static buffer */ - - if (fprintf(regfile, "%d %ld.%06ld %s %s %d %s\n", 1, - (long)reg_tv.tv_sec, (long)reg_tv.tv_usec, job, - ownerhash, strlen(owner), owner) < 1 || ferror(regfile)) { - fclose(regfile); - err.code = errno; - err.desc = "Cannot write jobs's reg info file"; - goto error_out; - } - if (fclose(regfile) != 0 ) { - err.code = errno; - err.desc = "Cannot close(write) jobs's reg info file"; - goto error_out; - } - - if (asprintf(&data_dir, "%s/data/%s/%d/%s", - config->internal_path, ownerhash, regtime_trunc(reg_tv.tv_sec), ju) == -1) { - err.code = ENOMEM; - goto error_out; - } - if (asprintf(&data_fname, "%s/_info", data_dir) == -1) { - err.code = ENOMEM; - goto error_out; - } - if (mkdirpath(data_dir, strlen(config->internal_path)) < 0 && - errno != EEXIST) { - err.code = errno; - err.desc = "Cannot mkdir jobs's data directory"; - goto error_out; - } - - if (link(int_fname, data_fname) < 0) { - err.code = errno; - err.desc = "Cannot link job's reg and data info files"; - goto error_out; - } - -error_out: - free(int_fname); - free(data_fname); - if (err.code && data_dir) rmdir(data_dir); - free(data_dir); - if (err.code) { - return glite_jp_stack_error(ctx,&err); - } else { - return 0; - } -} - -static int add_to_gridmap(glite_jp_context_t ctx, const char *dn) -{ - FILE *gridmap = NULL; - glite_jp_error_t err; - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - gridmap = fopen(config->gridmap, "a"); - if (!gridmap) { - err.code = errno; - err.desc = "Cannot open gridmap file"; - return glite_jp_stack_error(ctx,&err); - } - if (fprintf(gridmap, "\"%s\" %s\n", dn, config->logname) < 6 || - ferror(gridmap)) { - err.code = EIO; - err.desc = "Cannot write to gridmap file"; - fclose(gridmap); - return glite_jp_stack_error(ctx,&err); - } - fclose(gridmap); - return 0; -} - -static int remove_from_gridmap(glite_jp_context_t ctx, const char *dn) -{ - FILE *gridmap = NULL; - char *temp_name = NULL; - FILE *temp_file = NULL; - glite_jp_error_t err; - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - /* XXX */ - return 0; -} - -int glite_jppsbe_start_upload( - glite_jp_context_t ctx, - const char *job, - const char *class, - const char *name, /* TODO */ - const char *content_type, - char **destination_out, - time_t *commit_before_inout -) -{ - char *int_fname = NULL; - char *lock_fname = NULL; - FILE *lockfile = NULL; - FILE *regfile = NULL; - char *data_dir = NULL; - char *data_lock = NULL; - char *ju = NULL; - char *ju_path = NULL; - char *peername = NULL; - int info_version; - long reg_time; - char ownerhash[33]; - glite_jp_error_t err; - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - assert(job!=NULL); - assert(destination_out!=NULL); - - assert(class!=NULL); - - if (jobid_unique_pathname(ctx, job, &ju, &ju_path, 1) != 0) { - err.code = ctx->error->code; - err.desc = "Cannot obtain jobid unique path/name"; - return glite_jp_stack_error(ctx,&err); - } - - peername = glite_jp_peer_name(ctx); - - if (asprintf(&int_fname, "%s/regs/%s/%s.info", - config->internal_path, ju_path, ju) == -1) { - err.code = ENOMEM; - goto error_out; - } - regfile = fopen(int_fname, "r"); - if (regfile == NULL) { - err.code = errno; - if (errno == ENOENT) - err.desc = "Job not registered"; - else - err.desc = "Cannot open jobs's reg info file"; - goto error_out; - } - if (fscanf(regfile, "%d %ld.%*ld %*s %s ", &info_version, - ®_time, ownerhash) < 3 || ferror(regfile)) { - fclose(regfile); - err.code = errno; - err.desc = "Cannot read jobs's reg info file"; - goto error_out; - } - fclose(regfile); - - /* XXX authorization */ - - if (asprintf(&data_dir, "%s/data/%s/%d/%s", - config->internal_path, ownerhash, regtime_trunc(reg_time), ju) == -1) { - err.code = ENOMEM; - goto error_out; - } - - if (asprintf(&lock_fname, "%s/%s" LOCK_SUFFIX, - data_dir, class) == -1) { - err.code = ENOMEM; - goto error_out; - } - - if (commit_before_inout != NULL) - *commit_before_inout = (time_t) LONG_MAX; /* XXX no timeout enforced */ - - lockfile = fopen(lock_fname, "w"); - if (lockfile == NULL) { - err.code = errno; - err.desc = "Cannot open uploads's lock file"; - goto error_out; - } - - if (fprintf(lockfile, "%ld %d %s\n", (long)*commit_before_inout, - peername ? peername : 0, - peername ? peername : "") < 1 || ferror(regfile)) { - fclose(lockfile); - err.code = errno; - err.desc = "Cannot write upload's lock file"; - goto error_out; - } - if (fclose(lockfile) != 0 ) { - err.code = errno; - err.desc = "Cannot close(write) upload's lock file"; - goto error_out; - } - - if (asprintf(destination_out, "%s/data/%s/%d/%s/%s" UPLOAD_SUFFIX, - config->external_path, ownerhash, regtime_trunc(reg_time), ju, class) == -1) { - err.code = ENOMEM; - goto error_out; - } - - if (add_to_gridmap(ctx, peername)) { - err.code = EIO; - err.desc = "Cannot add peer DN to ftp server authorization file"; - goto error_out; - } - -error_out: - free(int_fname); - free(data_dir); - if (err.code && data_lock) unlink(data_lock); - free(data_lock); - free(ju); free(ju_path); - if (err.code) { - return glite_jp_stack_error(ctx,&err); - } else { - return 0; - } -} - -int glite_jppsbe_commit_upload( - glite_jp_context_t ctx, - const char *destination -) -{ - size_t dest_len; - size_t suff_len; - size_t extp_len; - long commit_before; - int lockpeerlen; - char *lockpeername = NULL; - char *peername = NULL; - char *dest_rw = NULL; - char *dest_rw_suff = NULL; - char *dest_rw_lock = NULL; - FILE *lockfile = NULL; - glite_jp_error_t err; - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - assert(destination != NULL); - - suff_len = strlen(UPLOAD_SUFFIX); - dest_len = strlen(destination); - extp_len = strlen(config->external_path); - - if (dest_len < suff_len || - strcmp(UPLOAD_SUFFIX, destination + (dest_len - suff_len)) || - strncmp(destination, config->external_path, extp_len)) { - err.code = EINVAL; - err.desc = "Forged destination path"; - return glite_jp_stack_error(ctx,&err); - } - - if (asprintf(&dest_rw_suff, "%s%s", config->internal_path, - destination + extp_len) == -1) { - err.code = ENOMEM; - return glite_jp_stack_error(ctx,&err); - } - dest_rw = strdup(dest_rw_suff); - if (!dest_rw) { - err.code = ENOMEM; - goto error_out; - } - *(dest_rw + (strlen(dest_rw_suff) - suff_len)) = '\0'; - - if (asprintf(&dest_rw_lock, "%s" LOCK_SUFFIX, dest_rw) == -1) { - err.code = ENOMEM; - goto error_out; - } - - lockfile = fopen(dest_rw_lock, "r"); - if (lockfile == NULL) { - err.code = errno; - err.desc = "Cannot open upload's lock file"; - goto error_out; - } - if (fscanf(lockfile, "%ld %d ", &commit_before, &lockpeerlen) < 2 || ferror(lockfile)) { - fclose(lockfile); - err.code = errno; - err.desc = "Cannot read upload's lock file"; - goto error_out; - } - if (lockpeerlen) { - lockpeername = (char*) calloc(1, lockpeerlen+1); - if (!lockpeername) { - err.code = ENOMEM; - goto error_out; - } - if (fgets(lockpeername, lockpeerlen+1, lockfile) == NULL) { - fclose(lockfile); - err.code = errno; - err.desc = "Cannot read upload's lock file"; - goto error_out; - } - } - fclose(lockfile); - - peername = glite_jp_peer_name(ctx); - if (lockpeername && (!peername || strcmp(lockpeername, peername))) { - err.code = EPERM; - err.desc = "Upload started by client of different identity"; - goto error_out; - } - - if (rename(dest_rw_suff, dest_rw) < 0) { - err.code = errno; - err.desc = "Cannot move upload file to the final place"; - goto error_out; - } - - if (unlink(dest_rw_lock) < 0) { - err.code = errno; - err.desc = "Cannot unlink upload's lock file"; - goto error_out; - } - -error_out: - free(dest_rw); - free(dest_rw_suff); - free(dest_rw_lock); - free(peername); - free(lockpeername); - if (err.code) { - return glite_jp_stack_error(ctx,&err); - } else { - return 0; - } -} - -int glite_jppsbe_destination_info( - glite_jp_context_t ctx, - const char *destination, - char **job, - char **class, - char **name -) -{ - size_t dest_len; - size_t suff_len; - size_t extp_len; - char *dest_rw = NULL; - char *dest_rw_suff = NULL; - char *dest_rw_info = NULL; - FILE *infofile = NULL; - char *classname = NULL; - char jobstr[256+1]; - glite_jp_error_t err; - - assert(destination != NULL); - assert(job != NULL); - assert(class != NULL); - assert(name != NULL); - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - suff_len = strlen(UPLOAD_SUFFIX); - dest_len = strlen(destination); - extp_len = strlen(config->external_path); - - if (dest_len < suff_len || - strcmp(UPLOAD_SUFFIX, destination + (dest_len - suff_len)) || - strncmp(destination, config->external_path, extp_len)) { - err.code = EINVAL; - err.desc = "Forged destination path"; - return glite_jp_stack_error(ctx,&err); - } - - if (asprintf(&dest_rw_suff, "%s%s", config->internal_path, - destination + extp_len) == -1) { - err.code = ENOMEM; - return glite_jp_stack_error(ctx,&err); - } - dest_rw = strdup(dest_rw_suff); - if (!dest_rw) { - err.code = ENOMEM; - goto error_out; - } - *(dest_rw + (strlen(dest_rw_suff) - suff_len)) = '\0'; - - classname = strrchr(dest_rw,'/'); - if (classname == NULL) { - err.code = EINVAL; - err.desc = "Forged destination path"; - goto error_out; - } - *classname++ ='\0'; - *class = strdup(classname); - -/* XXX: do we need similar check? - if (!class == GLITE_JP_FILECLASS_UNDEF) { - err.code = EINVAL; - err.desc = "Forged destination path"; - goto error_out; - } -*/ - - /* TODO: */ - *name = NULL; - - if (asprintf(&dest_rw_info, "%s/_info", dest_rw) == -1) { - err.code = ENOMEM; - goto error_out; - } - - infofile = fopen(dest_rw_info, "r"); - if (infofile == NULL) { - err.code = errno; - err.desc = "Cannot open _info file"; - goto error_out; - } - if (fscanf(infofile, "%*d %*ld.%*ld %256s ", jobstr) < 1 || ferror(infofile)) { - fclose(infofile); - err.code = errno; - err.desc = "Cannot read _info file"; - goto error_out; - } - *job = strdup(jobstr); - fclose(infofile); - -error_out: - free(dest_rw); - free(dest_rw_suff); - free(dest_rw_info); - if (err.code) { - return glite_jp_stack_error(ctx,&err); - } else { - return 0; - } -} - - -int glite_jppsbe_get_job_url( - glite_jp_context_t ctx, - const char *job, - const char *class, - const char *name, /* TODO */ - char **url_out -) -{ - FILE *regfile = NULL; - char *int_fname = NULL; - char *ju = NULL; - char *ju_path = NULL; - int info_version; - long reg_time; - char ownerhash[33]; - glite_jp_error_t err; - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - assert(job!=NULL); - assert(url_out != NULL); - - assert(class!=NULL); - - if (jobid_unique_pathname(ctx, job, &ju, &ju_path, 1) != 0) { - err.code = ctx->error->code; - err.desc = "Cannot obtain jobid unique path/name"; - return glite_jp_stack_error(ctx,&err); - } - - if (asprintf(&int_fname, "%s/regs/%s/%s.info", - config->internal_path, ju_path, ju) == -1) { - err.code = ENOMEM; - goto error_out; - } - regfile = fopen(int_fname, "r"); - if (regfile == NULL) { - err.code = errno; - if (errno == ENOENT) - err.desc = "Job not registered"; - else - err.desc = "Cannot open jobs's reg info file"; - goto error_out; - } - if (fscanf(regfile, "%d %ld.%*ld %*s %s", &info_version, - ®_time, ownerhash) < 3 || ferror(regfile)) { - fclose(regfile); - err.code = errno; - err.desc = "Cannot read jobs's reg info file"; - goto error_out; - } - fclose(regfile); - - if (asprintf(url_out, "%s/data/%s/%d/%s/%s", - config->external_path, ownerhash, regtime_trunc(reg_time), ju, class) == -1) { - err.code = ENOMEM; - goto error_out; - } - -error_out: - free(int_fname); - free(ju); free(ju_path); - if (err.code) { - return glite_jp_stack_error(ctx,&err); - } else { - return 0; - } -} - -static int get_job_fname( - glite_jp_context_t ctx, - const char *job, - const char *class, - const char *name, /* TODO */ - char **fname_out -) -{ - FILE *regfile = NULL; - char *int_fname = NULL; - char *ju = NULL; - char *ju_path = NULL; - int info_version; - long reg_time; - char ownerhash[33]; - glite_jp_error_t err; - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - assert(job!=NULL); - assert(fname_out != NULL); - - assert(class!=NULL); - - if (jobid_unique_pathname(ctx, job, &ju, &ju_path, 1) != 0) { - err.code = ctx->error->code; - err.desc = "Cannot obtain jobid unique path/name"; - return glite_jp_stack_error(ctx,&err); - } - - if (asprintf(&int_fname, "%s/regs/%s/%s.info", - config->internal_path, ju_path, ju) == -1) { - err.code = ENOMEM; - goto error_out; - } - regfile = fopen(int_fname, "r"); - if (regfile == NULL) { - err.code = errno; - if (errno == ENOENT) - err.desc = "Job not registered"; - else - err.desc = "Cannot open jobs's reg info file"; - goto error_out; - } - if (fscanf(regfile, "%d %ld.%*ld %*s %s", &info_version, - ®_time, ownerhash) < 3 || ferror(regfile)) { - fclose(regfile); - err.code = errno; - err.desc = "Cannot read jobs's reg info file"; - goto error_out; - } - fclose(regfile); - - if (asprintf(fname_out, "%s/data/%s/%d/%s/%s", - config->internal_path, ownerhash, regtime_trunc(reg_time), ju, class) == -1) { - err.code = ENOMEM; - goto error_out; - } - -error_out: - free(int_fname); - free(ju); free(ju_path); - if (err.code) { - return glite_jp_stack_error(ctx,&err); - } else { - return 0; - } -} - -int glite_jppsbe_open_file( - glite_jp_context_t ctx, - const char *job, - const char *class, - const char *name, /* TODO */ - int mode, - void **handle_out -) -{ - fhandle handle = NULL; - char* fname = NULL; - glite_jp_error_t err; - - assert(handle_out != NULL); - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - if (get_job_fname(ctx, job, class, name, &fname)) { - err.code = ctx->error->code; - err.desc = "Cannot construct internal filename"; - return glite_jp_stack_error(ctx,&err); - } - - handle = (fhandle) calloc(1,sizeof(*handle)); - if (handle == NULL) { - err.code = ENOMEM; - goto error_out; - } - - handle->fd = open(fname, mode, S_IRUSR | S_IWUSR); - if (handle->fd < 0) { - err.code = errno; - err.desc = "Cannot open requested file"; - free(handle); - goto error_out; - } - handle->fd_append = open(fname, mode | O_APPEND, S_IRUSR | S_IWUSR); - if (handle->fd_append < 0) { - err.code = errno; - err.desc = "Cannot open requested file for append"; - close(handle->fd); - free(handle); - goto error_out; - } - *handle_out = (void*) handle; - -error_out: - free(fname); - if (err.code) { - return glite_jp_stack_error(ctx,&err); - } else { - return 0; - } -} - -int glite_jppsbe_close_file( - glite_jp_context_t ctx, - void *handle -) -{ - glite_jp_error_t err; - - assert(handle != NULL); - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - if (close(((fhandle)handle)->fd_append) < 0) { - err.code = errno; - err.desc = "Error closing file descriptor (fd_append)"; - goto error_out; - } - if (close(((fhandle)handle)->fd) < 0) { - err.code = errno; - err.desc = "Error closing file descriptor"; - goto error_out; - } - -error_out: - free(handle); - if (err.code) { - return glite_jp_stack_error(ctx,&err); - } else { - return 0; - } -} - -int glite_jppsbe_pread( - glite_jp_context_t ctx, - void *handle, - void *buf, - size_t nbytes, - off_t offset, - ssize_t *nbytes_ret -) -{ - ssize_t ret; - glite_jp_error_t err; - - assert(handle != NULL); - assert(buf != NULL); - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - if ((ret = pread(((fhandle)handle)->fd, buf, nbytes, offset)) < 0) { - err.code = errno; - err.desc = "Error in pread()"; - return glite_jp_stack_error(ctx,&err); - } - *nbytes_ret = ret; - - return 0; -} - -int glite_jppsbe_pwrite( - glite_jp_context_t ctx, - void *handle, - void *buf, - size_t nbytes, - off_t offset -) -{ - glite_jp_error_t err; - - assert(handle != NULL); - assert(buf != NULL); - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - if (pwrite(((fhandle)handle)->fd, buf, nbytes, offset) < 0) { - err.code = errno; - err.desc = "Error in pwrite()"; - return glite_jp_stack_error(ctx,&err); - } - - return 0; -} - -int glite_jppsbe_append( - glite_jp_context_t ctx, - void *handle, - void *buf, - size_t nbytes -) -{ - glite_jp_error_t err; - - assert(handle != NULL); - assert(buf != NULL); - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - if (write(((fhandle)handle)->fd_append, buf, nbytes) < 0) { - err.code = errno; - err.desc = "Error in write()"; - return glite_jp_stack_error(ctx,&err); - } - - return 0; -} - -static int get_job_info( - glite_jp_context_t ctx, - const char *job, - char **owner, - struct timeval *tv_reg -) -{ - char *ju = NULL; - char *ju_path = NULL; - FILE *regfile = NULL; - long reg_time_sec; - long reg_time_usec; - int ownerlen = 0; - int info_version; - char *int_fname = NULL; - glite_jp_error_t err; - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - if (jobid_unique_pathname(ctx, job, &ju, &ju_path, 1) != 0) { - err.code = ctx->error->code; - err.desc = "Cannot obtain jobid unique path/name"; - return glite_jp_stack_error(ctx,&err); - } - - if (asprintf(&int_fname, "%s/regs/%s/%s.info", - config->internal_path, ju_path, ju) == -1) { - err.code = ENOMEM; - goto error_out; - } - regfile = fopen(int_fname, "r"); - if (regfile == NULL) { - err.code = errno; - if (errno == ENOENT) - err.desc = "Job not registered"; - else - err.desc = "Cannot open jobs's reg info file"; - goto error_out; - } - if (fscanf(regfile, "%d %ld.%ld %*s %*s %d ", &info_version, - ®_time_sec, ®_time_usec, &ownerlen) < 4 || ferror(regfile)) { - fclose(regfile); - err.code = errno; - err.desc = "Cannot read jobs's reg info file"; - goto error_out; - } - if (ownerlen) { - *owner = (char *) calloc(1, ownerlen+1); - if (!*owner) { - err.code = ENOMEM; - goto error_out; - } - if (fgets(*owner, ownerlen+1, regfile) == NULL) { - fclose(regfile); - free(*owner); - err.code = errno; - err.desc = "Cannot read jobs's reg info file"; - goto error_out; - } - } - fclose(regfile); - - tv_reg->tv_sec = reg_time_sec; - tv_reg->tv_usec = reg_time_usec; - -error_out: - free(int_fname); - free(ju); - free(ju_path); - if (err.code) { - return glite_jp_stack_error(ctx,&err); - } else { - return 0; - } -} - -static int get_job_info_int( - glite_jp_context_t ctx, - const char *int_fname, - char **jobid, - char **owner, - struct timeval *tv_reg -) -{ - FILE *regfile = NULL; - long reg_time_sec; - long reg_time_usec; - int ownerlen = 0; - int info_version; - char jobid_buf[256]; - glite_jp_error_t err; - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - regfile = fopen(int_fname, "r"); - if (regfile == NULL) { - err.code = errno; - err.desc = "Cannot open jobs's reg info file"; - goto error_out; - } - if (fscanf(regfile, "%d %ld.%ld %s %*s %d ", &info_version, - ®_time_sec, ®_time_usec, jobid_buf, &ownerlen) < 5 || ferror(regfile)) { - fclose(regfile); - err.code = errno; - err.desc = "Cannot read jobs's reg info file"; - goto error_out; - } - *jobid = strdup(jobid_buf); - if (ownerlen) { - *owner = (char *) calloc(1, ownerlen+1); - if (!*owner) { - err.code = ENOMEM; - goto error_out; - } - if (fgets(*owner, ownerlen+1, regfile) == NULL) { - fclose(regfile); - free(*owner); - err.code = errno; - err.desc = "Cannot read jobs's reg info file"; - goto error_out; - } - } - fclose(regfile); - - tv_reg->tv_sec = reg_time_sec; - tv_reg->tv_usec = reg_time_usec; - -error_out: - if (err.code) { - return glite_jp_stack_error(ctx,&err); - } else { - return 0; - } -} - -int glite_jppsbe_get_job_metadata( - glite_jp_context_t ctx, - const char *job, - glite_jp_attrval_t attrs_inout[] -) -{ - int got_info = 0; - struct timeval tv_reg; - char *owner = NULL; - int got_tags = 0; - void *tags_handle = NULL; - glite_jp_tagval_t* tags = NULL; - int i,j; - glite_jp_error_t err; - - assert(job != NULL); - assert(attrs_inout != NULL); - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - for (i = 0; attrs_inout[i].attr.type != GLITE_JP_ATTR_UNDEF; i++) { - switch (attrs_inout[i].attr.type) { - case GLITE_JP_ATTR_OWNER: - -/* must be implemented via filetype plugin - case GLITE_JP_ATTR_TIME: -*/ - if (!got_info) { - if (get_job_info(ctx, job, &owner, &tv_reg)) { - err.code = ctx->error->code; - err.desc = "Cannot retrieve job info"; - goto error_out; - } - got_info = 1; - } - break; - -/* must be implemented via filetype plugin - case GLITE_JP_ATTR_TAG: - if (!got_tags) { - if (glite_jppsbe_open_file(ctx, job, GLITE_JP_FILECLASS_TAGS, - O_RDONLY, &tags_handle)) { - err.code = ctx->error->code; - err.desc = "Cannot open tag file"; - goto error_out; - } - if (glite_jpps_tag_readall(ctx, tags_handle, &tags)) { - err.code = ctx->error->code; - err.desc = "Cannot read tags"; - glite_jppsbe_close_file(ctx, tags_handle); - goto error_out; - } - glite_jppsbe_close_file(ctx, tags_handle); - got_tags = 1; - } - break; -*/ - default: - err.code = EINVAL; - err.desc = "Invalid attribute type"; - goto error_out; - break; - } - - switch (attrs_inout[i].attr.type) { - case GLITE_JP_ATTR_OWNER: - attrs_inout[i].value.s = strdup(owner); - if (!attrs_inout[i].value.s) { - err.code = ENOMEM; - err.desc = "Cannot copy owner string"; - goto error_out; - } - break; - case GLITE_JP_ATTR_TIME: - attrs_inout[i].value.time = tv_reg; - break; - -/* must be implemented via filetype plugin - case GLITE_JP_ATTR_TAG: - for (j = 0; tags[j].name != NULL; j++) { - if (!strcmp(tags[j].name, attrs_inout[i].attr.name)) { - if (glite_jpps_tagval_copy(ctx, &tags[j], - &attrs_inout[i].value.tag)) { - err.code = ENOMEM; - err.desc = "Cannot copy tag value"; - goto error_out; - } - break; - } - } - if (!tags[j].name) attrs_inout[i].value.tag.name = NULL; - break; -*/ - default: - break; - } - } - -error_out: - free(owner); - if (tags) for (j = 0; tags[j].name != NULL; j++) { - free(tags[j].name); - free(tags[j].value); - } - free(tags); - - if (err.code) { - while (i > 0) { - i--; - switch (attrs_inout[i].attr.type) { - case GLITE_JP_ATTR_OWNER: - free(attrs_inout[i].value.s); - break; - case GLITE_JP_ATTR_TAG: - free(attrs_inout[i].value.tag.name); - free(attrs_inout[i].value.tag.value); - default: - break; - } - } - return glite_jp_stack_error(ctx,&err); - } else { - return 0; - } -} -static int compare_timeval(struct timeval a, struct timeval b) -{ - if (a.tv_sec < b.tv_sec) return -1; - if (a.tv_sec > b.tv_sec) return 1; - if (a.tv_usec < b.tv_usec) return -1; - if (a.tv_usec > b.tv_usec) return 1; - return 0; -} - - -/* FIXME: disabled -- clarification wrt. filetype plugin needed */ - -#if 0 - -static int query_phase2( - glite_jp_context_t ctx, - const char *ownerhash, - long regtime_tr, - int q_tags, - int md_tags, - const glite_jp_query_rec_t query[], - glite_jp_attrval_t metadata[], - int (*callback)( - glite_jp_context_t ctx, - const char *job, - const glite_jp_attrval_t metadata[] - ) -); - -static int query_phase2( - glite_jp_context_t ctx, - const char *ownerhash, - long regtime_tr, - int q_tags, - int md_tags, - const glite_jp_query_rec_t query[], - glite_jp_attrval_t metadata[], - int (*callback)( - glite_jp_context_t ctx, - const char *job, - const glite_jp_attrval_t metadata[] - ) -) -{ - char *time_dirname = NULL; - DIR *time_dirp = NULL; - struct dirent *jobent; - char *info_fname = NULL; - char *jobid = NULL; - char *owner = NULL; - struct timeval tv_reg; - void *tags_handle = NULL; - int matching; - int i, j; - glite_jp_tagval_t* tags = NULL; - glite_jp_error_t err; - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - if (asprintf(&time_dirname, "%s/data/%s/%d", config->internal_path, - ownerhash, regtime_tr) == -1) { - err.code = ENOMEM; - return glite_jp_stack_error(ctx,&err); - } - time_dirp = opendir(time_dirname); - if (!time_dirp) { - free(time_dirname); - return 0; /* found nothing */ - } - while ((jobent = readdir(time_dirp)) != NULL) { - if (!strcmp(jobent->d_name, ".")) continue; - if (!strcmp(jobent->d_name, "..")) continue; - if (asprintf(&info_fname, "%s/%s/_info", time_dirname, - jobent->d_name) == -1) { - err.code = ENOMEM; - goto error_out; - } - if (get_job_info_int(ctx, info_fname, &jobid, &owner, &tv_reg)) { - err.code = EIO; - err.desc = "Cannot retrieve job info"; - goto error_out; - } - if (q_tags || md_tags) { - if (glite_jppsbe_open_file(ctx, jobid, GLITE_JP_FILECLASS_TAGS, - O_RDONLY, &tags_handle)) { - err.code = ctx->error->code; - err.desc = "Cannot open tag file"; - goto error_out; - } - if (glite_jpps_tag_readall(ctx, tags_handle, &tags)) { - err.code = ctx->error->code; - err.desc = "Cannot read tags"; - glite_jppsbe_close_file(ctx, tags_handle); - goto error_out; - } - glite_jppsbe_close_file(ctx, tags_handle); - tags_handle = NULL; - } - - matching = 1; - for (i = 0; matching && query[i].attr.type != GLITE_JP_ATTR_UNDEF; i++) { - switch (query[i].attr.type) { - case GLITE_JP_ATTR_OWNER: - if (query[i].value.s == NULL || - strcmp(query[i].value.s, owner)) matching = 0; - break; - case GLITE_JP_ATTR_TIME: - switch (query[i].op) { - case GLITE_JP_QUERYOP_EQUAL: - matching = !compare_timeval(tv_reg, query[i].value.time); - break; - case GLITE_JP_QUERYOP_UNEQUAL: - matching = compare_timeval(tv_reg, query[i].value.time); - break; - case GLITE_JP_QUERYOP_LESS: - matching = compare_timeval(tv_reg, query[i].value.time) < 0; - break; - case GLITE_JP_QUERYOP_GREATER: - matching = compare_timeval(tv_reg, query[i].value.time) > 0; - break; - case GLITE_JP_QUERYOP_WITHIN: - matching = compare_timeval(tv_reg, query[i].value.time) >= 0 - && compare_timeval(tv_reg, query[i].value2.time) <= 0; - break; - } - break; - case GLITE_JP_ATTR_TAG: - if (!tags) { - matching = 0; - break; - } - for (j = 0; tags[j].name != NULL; j++) { - if (!strcmp(tags[j].name, query[i].attr.name)) { - switch (query[i].op) { - case GLITE_JP_QUERYOP_EQUAL: - matching = !strcmp(tags[j].value, query[i].value.s); - break; - case GLITE_JP_QUERYOP_UNEQUAL: - matching = strcmp(tags[j].value, query[i].value.s); - break; - case GLITE_JP_QUERYOP_LESS: - matching = strcmp(tags[j].value, query[i].value.s) < 0; - break; - case GLITE_JP_QUERYOP_GREATER: - matching = strcmp(tags[j].value, query[i].value.s) > 0; - break; - case GLITE_JP_QUERYOP_WITHIN: - matching = strcmp(tags[j].value, query[i].value.s) >= 0 \ - && strcmp(tags[j].value, query[i].value2.s) <= 0 ; - break; - default: - break; - } - } - } - break; - default: - break; - } - } - if (!matching) { - free(info_fname); info_fname = NULL; - free(jobid); jobid = NULL; - if (tags) for (j = 0; tags[j].name != NULL; j++) { - free(tags[j].name); - free(tags[j].value); - } - free(tags); tags = NULL; - continue; - } - - for (i = 0; metadata[i].attr.type != GLITE_JP_ATTR_UNDEF; i++) { - switch (metadata[i].attr.type) { - case GLITE_JP_ATTR_OWNER: - metadata[i].value.s = owner; - break; - case GLITE_JP_ATTR_TIME: - metadata[i].value.time = tv_reg; - break; - case GLITE_JP_ATTR_TAG: - for (j = 0; tags[j].name != NULL; j++) { - if (!strcmp(tags[j].name, metadata[i].attr.name)) { - if (glite_jpps_tagval_copy(ctx, &tags[j], - &metadata[i].value.tag)) { - err.code = ENOMEM; - err.desc = "Cannot copy tag value"; - goto error_out; - } - break; - } - } - if (!tags[j].name) { - metadata[i].value.tag.name = NULL; - metadata[i].value.tag.value = NULL; - } - break; - default: - break; - } - } - (*callback)(ctx, jobid, metadata); - free(jobid); jobid = NULL; - while (i > 0) { - i--; - switch (metadata[i].attr.type) { - case GLITE_JP_ATTR_TAG: - free(metadata[i].value.tag.name); - free(metadata[i].value.tag.value); - default: - break; - } - } - } - -error_out: - if (tags) for (j = 0; tags[j].name != NULL; j++) { - free(tags[j].name); - free(tags[j].value); - } - if (tags_handle) glite_jppsbe_close_file(ctx, tags_handle); - free(info_fname); - free(owner); - free(jobid); - closedir(time_dirp); - free(time_dirname); - if (err.code) { - while (i > 0) { - i--; - switch (metadata[i].attr.type) { - case GLITE_JP_ATTR_TAG: - free(metadata[i].value.tag.name); - free(metadata[i].value.tag.value); - default: - break; - } - } - return glite_jp_stack_error(ctx,&err); - } else - return 0; -} - -int glite_jppsbe_query( - glite_jp_context_t ctx, - const glite_jp_query_rec_t query[], - const glite_jp_attrval_t metadata[], - int (*callback)( - glite_jp_context_t ctx, - const char *job, - const glite_jp_attrval_t metadata[] - ) -) -{ - /* XXX clone metadata */ - int i; - char *q_exact_owner = NULL; - char *ownerhash = NULL; - long q_min_time = 0; - long q_max_time = LONG_MAX; - long q_min_time_tr; - long q_max_time_tr; - int q_with_tags = 0; - int md_info = 0; - int md_tags = 0; - char *owner_dirname = NULL; - DIR *owner_dirp = NULL; - struct dirent *ttimeent; - char *data_dirname = NULL; - DIR *data_dirp = NULL; - struct dirent *ownerent; - long ttime = 0; - glite_jp_attrval_t *metadata_templ = NULL; - glite_jp_error_t err; - - glite_jp_clear_error(ctx); - memset(&err,0,sizeof err); - err.source = __FUNCTION__; - - for (i = 0; query[i].attr.type != GLITE_JP_ATTR_UNDEF; i++) { - if (query[i].attr.type == GLITE_JP_ATTR_OWNER && query[i].op == GLITE_JP_QUERYOP_EQUAL) { - q_exact_owner = query[i].value.s; - } - if (query[i].attr.type == GLITE_JP_ATTR_TIME) { - switch (query[i].op) { - case GLITE_JP_QUERYOP_EQUAL: - q_min_time = query[i].value.time.tv_sec; - q_max_time = query[i].value.time.tv_sec + 1; - break; - case GLITE_JP_QUERYOP_LESS: - if (q_max_time > query[i].value.time.tv_sec + 1) - q_max_time = query[i].value.time.tv_sec + 1; - break; - case GLITE_JP_QUERYOP_WITHIN: - if (q_max_time > query[i].value2.time.tv_sec + 1) - q_max_time = query[i].value2.time.tv_sec + 1; - /* fallthrough */ - case GLITE_JP_QUERYOP_GREATER: - if (q_min_time < query[i].value.time.tv_sec) - q_min_time = query[i].value.time.tv_sec; - break; - default: - err.code = EINVAL; - err.desc = "Invalid query op"; - return glite_jp_stack_error(ctx,&err); - break; - } - } - if (query[i].attr.type == GLITE_JP_ATTR_TAG) - q_with_tags = 1; - - } - - for (i = 0; metadata[i].attr.type != GLITE_JP_ATTR_UNDEF; i++) { - switch (metadata[i].attr.type) { - case GLITE_JP_ATTR_OWNER: - case GLITE_JP_ATTR_TIME: - md_info = 1; - break; - case GLITE_JP_ATTR_TAG: - md_tags = 1; - break; - default: - err.code = EINVAL; - err.desc = "Invalid attribute type in metadata parameter"; - return glite_jp_stack_error(ctx,&err); - break; - } - } - metadata_templ = (glite_jp_attrval_t *) calloc(i + 1, sizeof(glite_jp_attrval_t)); - if (!metadata_templ) { - err.code = ENOMEM; - return glite_jp_stack_error(ctx,&err); - } - memcpy(metadata_templ, metadata, (i + 1) * sizeof(glite_jp_attrval_t)); - - q_min_time_tr = regtime_trunc(q_min_time); - q_max_time_tr = regtime_ceil(q_max_time); - - if (q_exact_owner) { - ownerhash = str2md5(q_exact_owner); /* static buffer */ - if (asprintf(&owner_dirname, "%s/data/%s", config->internal_path, ownerhash) == -1) { - err.code = ENOMEM; - return glite_jp_stack_error(ctx,&err); - } - owner_dirp = opendir(owner_dirname); - free(owner_dirname); - if (!owner_dirp) { - free(metadata_templ); - return 0; /* found nothing */ - } - while ((ttimeent = readdir(owner_dirp)) != NULL) { - if (!strcmp(ttimeent->d_name, ".")) continue; - if (!strcmp(ttimeent->d_name, "..")) continue; - ttime = atol(ttimeent->d_name); - if (ttime >= q_min_time_tr && ttime < q_max_time_tr) { - if (query_phase2(ctx, ownerhash, ttime, q_with_tags, md_tags, - query, metadata_templ, callback)) { - err.code = EIO; - err.desc = "query_phase2() error"; - goto error_out; - } - } - } - } else { /* !q_exact_owner */ - if (asprintf(&data_dirname, "%s/data", config->internal_path) == -1) { - err.code = ENOMEM; - goto error_out; - } - data_dirp = opendir(data_dirname); - if (!data_dirp) { - err.code = EIO; - err.desc = "Cannot open data directory"; - goto error_out; - } - while ((ownerent = readdir(data_dirp)) != NULL) { - if (!strcmp(ownerent->d_name, ".")) continue; - if (!strcmp(ownerent->d_name, "..")) continue; - if (asprintf(&owner_dirname, "%s/data/%s", config->internal_path, - ownerent->d_name) == -1) { - err.code = ENOMEM; - goto error_out; - } - owner_dirp = opendir(owner_dirname); - free(owner_dirname); - if (!owner_dirp) { - err.code = EIO; - err.desc = "Cannot open owner data directory"; - goto error_out; - } - while ((ttimeent = readdir(owner_dirp)) != NULL) { - if (!strcmp(ttimeent->d_name, ".")) continue; - if (!strcmp(ttimeent->d_name, "..")) continue; - ttime = atol(ttimeent->d_name); - if (ttime >= q_min_time_tr && ttime < q_max_time_tr) { - if (query_phase2(ctx, ownerent->d_name, ttime, q_with_tags, md_tags, - query, metadata_templ, callback)) { - err.code = EIO; - err.desc = "query_phase2() error"; - goto error_out; - } - } - } - closedir(owner_dirp); owner_dirp = NULL; - } - closedir(data_dirp); data_dirp = NULL; - } - return 0; - -error_out: - if (owner_dirp) closedir(owner_dirp); - if (data_dirp) closedir(data_dirp); - free(data_dirname); - free(metadata_templ); - return glite_jp_stack_error(ctx,&err); -} - -#else - -/* placeholder instead */ -int glite_jppsbe_query( - glite_jp_context_t ctx, - const glite_jp_query_rec_t query[], - const glite_jp_attrval_t metadata[], - int (*callback)( - glite_jp_context_t ctx, - const char *job, - const glite_jp_attrval_t metadata[] - ) -) -{ - glite_jp_error_t err; - err.code = ENOSYS; - err.desc = "not implemented"; - return glite_jp_stack_error(ctx,&err); -} - -#endif - -/* XXX: -- no primary authorization yet -- no concurrency control yet -- partial success in pwrite,append -- "unique" part of jobid is assumed to be unique across bookkeeping servers -- repository versioning not fully implemented yet -*/ diff --git a/org.glite.jp.primary/src/new_ftp_backend.c b/org.glite.jp.primary/src/new_ftp_backend.c index 5774b51..27da176 100644 --- a/org.glite.jp.primary/src/new_ftp_backend.c +++ b/org.glite.jp.primary/src/new_ftp_backend.c @@ -1784,7 +1784,10 @@ int glite_jppsbe_query( memset(metadata,0, sizeof metadata); /* XXX: const discarding is OK */ - for (i=0;attrs[i]; i++) metadata[i].name = (char *) attrs[i]; + for (i=0;attrs[i]; i++) { + assert(i<2); + metadata[i].name = (char *) attrs[i]; + } for (i=0; query[i].attr; i++) { char *qitem; diff --git a/org.glite.jp.primary/src/soap_ops.c b/org.glite.jp.primary/src/soap_ops.c index 97db1b1..d933311 100644 --- a/org.glite.jp.primary/src/soap_ops.c +++ b/org.glite.jp.primary/src/soap_ops.c @@ -195,11 +195,23 @@ SOAP_FMAC5 int SOAP_FMAC6 __jpsrv__RecordTag( CONTEXT_FROM_SOAP(soap,ctx); void *file_be,*file_p; glite_jpps_fplug_data_t **pd = NULL; - glite_jp_attrval_t attr[2]; + glite_jp_attrval_t attr[2], meta[2]; + file_be = file_p = NULL; memset(attr, 0, 2 * sizeof(glite_jp_attrval_t)); + memset(meta,0,sizeof meta); + meta[0].name = strdup(GLITE_JP_ATTR_OWNER); + + if (glite_jppsbe_get_job_metadata(ctx,in->jobid,meta)) { + goto err; + } + + if (glite_jpps_authz(ctx,SOAP_TYPE___jpsrv__RecordTag,in->jobid,meta[0].value)) { + goto err; + } + attr[0].name = in->tag->name; if (in->tag->value->string) { attr[0].value = in->tag->value->string; @@ -252,6 +264,10 @@ SOAP_FMAC5 int SOAP_FMAC6 __jpsrv__RecordTag( free(pd); return SOAP_OK; +err: + glite_jp_attrval_free(meta,0); + err2fault(ctx,soap); + return SOAP_FAULT; } static void s2jp_qval(const struct jptype__stringOrBlob *in, char **value, int *binary, size_t *size) @@ -407,10 +423,22 @@ SOAP_FMAC5 int SOAP_FMAC6 __jpsrv__GetJobFiles( glite_jp_error_t err; void **pd; struct jptype__jppsFile **f = NULL; + glite_jp_attrval_t meta[2]; memset(&err,0,sizeof err); out->__sizefiles = 0; + memset(meta,0,sizeof meta); + meta[0].name = strdup(GLITE_JP_ATTR_OWNER); + + if (glite_jppsbe_get_job_metadata(ctx,in->jobid,meta)) { + goto err; + } + + if (glite_jpps_authz(ctx,SOAP_TYPE___jpsrv__GetJobFiles,in->jobid,meta[0].value)) { + goto err; + } + for (pd = ctx->plugins; *pd; pd++) { glite_jpps_fplug_data_t *plugin = *pd; @@ -454,6 +482,10 @@ SOAP_FMAC5 int SOAP_FMAC6 __jpsrv__GetJobFiles( memcpy(out->files,f,out->__sizefiles * sizeof *f); return SOAP_OK; +err: + glite_jp_attrval_free(meta,0); + err2fault(ctx,soap); + return SOAP_FAULT; } SOAP_FMAC5 int SOAP_FMAC6 __jpsrv__GetJobAttributes( @@ -461,11 +493,22 @@ SOAP_FMAC5 int SOAP_FMAC6 __jpsrv__GetJobAttributes( struct _jpelem__GetJobAttributes *in, struct _jpelem__GetJobAttributesResponse *out) { - glite_jp_attrval_t *attr; + glite_jp_attrval_t *attr, meta[2]; int i,n; CONTEXT_FROM_SOAP(soap,ctx); + memset(meta,0,sizeof meta); + meta[0].name = strdup(GLITE_JP_ATTR_OWNER); + + if (glite_jppsbe_get_job_metadata(ctx,in->jobid,meta)) { + goto err; + } + + if (glite_jpps_authz(ctx,SOAP_TYPE___jpsrv__GetJobAttributes,in->jobid,meta[0].value)) { + goto err; + } + if (glite_jpps_get_attrs(ctx,in->jobid, in->attributes, in->__sizeattributes,&attr)) { @@ -477,4 +520,8 @@ SOAP_FMAC5 int SOAP_FMAC6 __jpsrv__GetJobAttributes( out->__sizeattrValues = jp2s_attrValues(soap,attr,&out->attrValues,1); return SOAP_OK; +err: + glite_jp_attrval_free(meta,0); + err2fault(ctx,soap); + return SOAP_FAULT; } -- 1.8.2.3