From 95929b0e7260dbb082095aec7a4f963ed0b97248 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Ale=C5=A1=20K=C5=99enek?= Date: Fri, 15 Oct 2004 09:49:24 +0000 Subject: [PATCH] *** empty log message *** --- org.glite.jp.common/Makefile | 85 ++++ org.glite.jp.common/build.xml | 115 +++++ org.glite.jp.common/interface/context.h | 17 + org.glite.jp.common/interface/strmd5.h | 28 ++ org.glite.jp.common/interface/types.h | 83 ++++ .../project/configure.properties.xml | 62 +++ org.glite.jp.common/project/properties.xml | 65 +++ org.glite.jp.common/src/context.c | 164 +++++++ org.glite.jp.common/src/strmd5.c | 115 +++++ org.glite.jp.index/Makefile | 129 +++++ org.glite.jp.index/build.xml | 118 +++++ org.glite.jp.index/project/JobProvenanceIS.wsdl | 531 +++++++++++++++++++++ .../project/configure.properties.xml | 68 +++ org.glite.jp.index/project/properties.xml | 71 +++ org.glite.jp.index/src/simple_server.c | 39 ++ org.glite.jp.index/src/soap_ops.c | 81 ++++ org.glite.jp.index/src/typemap.dat | 2 + org.glite.jp.primary/Makefile | 141 ++++++ org.glite.jp.primary/build.xml | 115 +++++ org.glite.jp.primary/examples/jpps-test.c | 129 +++++ .../project/configure.properties.xml | 65 +++ org.glite.jp.primary/project/properties.xml | 68 +++ org.glite.jp.primary/src/backend.h | 104 ++++ org.glite.jp.primary/src/feed.c | 263 ++++++++++ org.glite.jp.primary/src/feed.h | 21 + org.glite.jp.primary/src/ftp_backend.c | 166 +++++++ org.glite.jp.primary/src/is_client.c | 28 ++ org.glite.jp.primary/src/simple_server.c | 46 ++ org.glite.jp.primary/src/soap_ops.c | 354 ++++++++++++++ org.glite.jp.primary/src/tags.c | 62 +++ org.glite.jp.primary/src/tags.h | 1 + org.glite.jp.primary/src/typemap.dat | 2 + org.glite.jp/project/JobProvenance.wsdl | 531 +++++++++++++++++++++ org.glite.jp/project/JobProvenanceIS.wsdl | 108 +++++ org.glite.jp/project/JobProvenancePS.wsdl | 307 ++++++++++++ org.glite.jp/project/JobProvenanceTypes.wsdl | 162 +++++++ org.glite.jp/project/JobProvenanceTypes.xsd | 146 ++++++ org.glite.jp/project/properties.xml | 61 +++ org.glite.jp/project/taskdefs.xml | 38 ++ 39 files changed, 4691 insertions(+) create mode 100644 org.glite.jp.common/Makefile create mode 100755 org.glite.jp.common/build.xml create mode 100644 org.glite.jp.common/interface/context.h create mode 100755 org.glite.jp.common/interface/strmd5.h create mode 100644 org.glite.jp.common/interface/types.h create mode 100644 org.glite.jp.common/project/configure.properties.xml create mode 100755 org.glite.jp.common/project/properties.xml create mode 100644 org.glite.jp.common/src/context.c create mode 100755 org.glite.jp.common/src/strmd5.c create mode 100644 org.glite.jp.index/Makefile create mode 100755 org.glite.jp.index/build.xml create mode 100644 org.glite.jp.index/project/JobProvenanceIS.wsdl create mode 100644 org.glite.jp.index/project/configure.properties.xml create mode 100755 org.glite.jp.index/project/properties.xml create mode 100644 org.glite.jp.index/src/simple_server.c create mode 100644 org.glite.jp.index/src/soap_ops.c create mode 100644 org.glite.jp.index/src/typemap.dat create mode 100644 org.glite.jp.primary/Makefile create mode 100755 org.glite.jp.primary/build.xml create mode 100644 org.glite.jp.primary/examples/jpps-test.c create mode 100644 org.glite.jp.primary/project/configure.properties.xml create mode 100755 org.glite.jp.primary/project/properties.xml create mode 100644 org.glite.jp.primary/src/backend.h create mode 100644 org.glite.jp.primary/src/feed.c create mode 100644 org.glite.jp.primary/src/feed.h create mode 100644 org.glite.jp.primary/src/ftp_backend.c create mode 100644 org.glite.jp.primary/src/is_client.c create mode 100644 org.glite.jp.primary/src/simple_server.c create mode 100644 org.glite.jp.primary/src/soap_ops.c create mode 100644 org.glite.jp.primary/src/tags.c create mode 100644 org.glite.jp.primary/src/tags.h create mode 100644 org.glite.jp.primary/src/typemap.dat create mode 100644 org.glite.jp/project/JobProvenance.wsdl create mode 100644 org.glite.jp/project/JobProvenanceIS.wsdl create mode 100644 org.glite.jp/project/JobProvenancePS.wsdl create mode 100644 org.glite.jp/project/JobProvenanceTypes.wsdl create mode 100644 org.glite.jp/project/JobProvenanceTypes.xsd create mode 100755 org.glite.jp/project/properties.xml create mode 100755 org.glite.jp/project/taskdefs.xml diff --git a/org.glite.jp.common/Makefile b/org.glite.jp.common/Makefile new file mode 100644 index 0000000..8626261 --- /dev/null +++ b/org.glite.jp.common/Makefile @@ -0,0 +1,85 @@ +# defaults +top_srcdir=. +builddir=build +top_builddir=${top_srcdir}/${builddir} +stagedir=. +distdir=. +globalprefix=glite +lbprefix=lb +package=glite-jp-common +version=0.0.0 +PREFIX=/opt/glite + +glite_location=/opt/glite +globus_prefix=/opt/globus +nothrflavour=gcc32 +thrflavour=gcc32pthr +expat_prefix=/opt/expat +ares_prefix=/opt/ares +gsoap_prefix=/software/gsoap-2.6 + +CC=gcc + +-include Makefile.inc + + +VPATH=${top_srcdir}/src:${top_srcdir}/test:${top_srcdir}/project:${jpproject} + +GLOBUSINC:= -I${globus_prefix}/include/${nothrflavour} + + +DEBUG:=-g -O0 +CFLAGS:=${DEBUG} -I. -I${top_srcdir}/interface -I${stagedir}/include \ + ${GLOBUSINC} + +LINK:=libtool --mode=link ${CC} ${LDFLAGS} -rpath ${stagedir}/lib +LINKXX:=libtool --mode=link ${CXX} ${LDFLAGS} +INSTALL:=libtool --mode=install install +COMPILE:=libtool --mode=compile ${CC} ${CFLAGS} + +HDRS:=types.h context.h strmd5.h + +SRCS:=context.c strmd5.c +OBJS:=${SRCS:.c=.lo} + +commonlib:= libglite_jp_common.la + +default all: compile + +compile: ${commonlib} + +${commonlib}: ${OBJS} + ${LINK} -o $@ ${OBJS} + +check: + -echo nothing yet + +doc: + +stage: compile + $(MAKE) install PREFIX=${stagedir} + +install: + -mkdir -p ${PREFIX}/include/${globalprefix}/${jpprefix} + cd ${top_srcdir}/interface && install -m 644 ${HDRS} ${PREFIX}/include/${globalprefix}/${jpprefix} + -mkdir -p ${PREFIX}/lib + ${INSTALL} -m 755 ${commonlib} ${PREFIX}/lib + +dist: distsrc distbin + +# FIXME: just copied from LB +distsrc: + mkdir -p ${top_srcdir}/${package}-${version} + cd ${top_srcdir} && GLOBIGNORE="${package}-${version}" && cp -Rf * ${package}-${version} + cd ${top_srcdir} && tar -czf ${distdir}/${package}-${version}_src.tar.gz --exclude-from=project/tar_exclude ${package}-${version} + rm -rf ${top_srcdir}/${package}-${version} + +distbin: + $(MAKE) install PREFIX=`pwd`/tmpbuilddir${stagedir} + save_dir=`pwd`; cd tmpbuilddir${stagedir} && tar -czf $$save_dir/${top_srcdir}/${distdir}/${package}-${version}_bin.tar.gz *; cd $$save_dir + rm -rf tmpbuilddir + +clean: + +%.lo: %.c + ${COMPILE} -o $@ -c $< diff --git a/org.glite.jp.common/build.xml b/org.glite.jp.common/build.xml new file mode 100755 index 0000000..0b93f98 --- /dev/null +++ b/org.glite.jp.common/build.xml @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.glite.jp.common/interface/context.h b/org.glite.jp.common/interface/context.h new file mode 100644 index 0000000..20effb2 --- /dev/null +++ b/org.glite.jp.common/interface/context.h @@ -0,0 +1,17 @@ +#ifndef __GLITE_JP_CONTEXT +#define __GLITE_JP_CONTEXT + +int glite_jp_init_context(glite_jp_context_t *); +void glite_jp_free_query_rec(glite_jp_query_rec_t *); + +char *glite_jp_peer_name(glite_jp_context_t); +char *glite_jp_error_chain(glite_jp_context_t); + +int glite_jp_stack_error(glite_jp_context_t, const glite_jp_error_t *); +int glite_jp_clear_error(glite_jp_context_t); + +int glite_jp_add_deferred(glite_jp_context_t,int (*)(glite_jp_context_t,void *),void *); +int glite_jp_run_deferred(glite_jp_context_t); + + +#endif diff --git a/org.glite.jp.common/interface/strmd5.h b/org.glite.jp.common/interface/strmd5.h new file mode 100755 index 0000000..c5d76b6 --- /dev/null +++ b/org.glite.jp.common/interface/strmd5.h @@ -0,0 +1,28 @@ +#ifndef _GLITE_STRMD5_H +#define _GLITE_STRMD5_H + +#ident "$Header$" + +/* Compute MD5 sum of the first argument. + * The sum is returned in the 16-byte array pointed to by 2nd argument + * (if not NULL) + * + * Return value: ASCII string of the sum, i.e. 32 characters [0-9a-f] + * (pointer to static area, changed by subsequent calls) + */ + +char *strmd5(const char *src, unsigned char *dst); + +/** + * Returns: allocated 32bytes long ASCII string with md5 sum + * of the first argument + */ +char *str2md5(const char *src); + +/** + * Returns: allocated 22bytes long ASCII string with md5 sum in base64 + * format of the source argument + */ +char *str2md5base64(const char *src); + +#endif /* _GLITE_STRMD5_H */ diff --git a/org.glite.jp.common/interface/types.h b/org.glite.jp.common/interface/types.h new file mode 100644 index 0000000..70e8482 --- /dev/null +++ b/org.glite.jp.common/interface/types.h @@ -0,0 +1,83 @@ +#ifndef __GLITE_JP_TYPES +#define __GLITE_JP_TYPES + +#include + +typedef struct _glite_jp_error_t { + int code; + char *desc; + char *source; + struct _glite_jp_error_t *reason; +} glite_jp_error_t; + +typedef struct _glite_jp_context { + glite_jp_error_t *error; + int (**deferred_func)(struct _glite_jp_context *,void *); + void **deferred_arg; + void *feeds; + struct soap *other_soap; +} *glite_jp_context_t; + +typedef enum { + GLITE_JP_FILECLASS_UNDEF, + GLITE_JP_FILECLASS_INPUT, + GLITE_JP_FILECLASS_OUTPUT, + GLITE_JP_FILECLASS_LBLOG, + GLITE_JP_FILECLASS_TAGS, + GLITE_JP_FILECLASS__LAST +} glite_jp_fileclass_t; + +typedef struct { + char *name; + int sequence; + time_t timestamp; + int binary; + size_t size; + char *value; +} glite_jp_tagval_t; + +typedef enum { + GLITE_JP_ATTR_UNDEF, + GLITE_JP_ATTR_OWNER, + GLITE_JP_ATTR_TIME, + GLITE_JP_ATTR_TAG, + GLITE_JP_ATTR__LAST +} glite_jp_attrtype_t; + +typedef struct { + glite_jp_attrtype_t type; + char *name; +} glite_jp_attr_t; + +typedef struct { + glite_jp_attr_t attr; + union { + char *s; + int i; + struct timeval time; + glite_jp_tagval_t tag; + } value; +} glite_jp_attrval_t; + + +typedef enum { + GLITE_JP_QUERYOP_UNDEF, + GLITE_JP_QUERYOP_EQUAL, + GLITE_JP_QUERYOP_UNEQUAL, + GLITE_JP_QUERYOP_LESS, + GLITE_JP_QUERYOP_GREATER, + GLITE_JP_QUERYOP_WITHIN, + GLITE_JP_QUERYOP__LAST, +} glite_jp_queryop_t; + +typedef struct { + glite_jp_attr_t attr; + glite_jp_queryop_t op; + union _glite_jp_query_rec_val { + char *s; + int i; + struct timeval time; + } value,value2; +} glite_jp_query_rec_t; + +#endif diff --git a/org.glite.jp.common/project/configure.properties.xml b/org.glite.jp.common/project/configure.properties.xml new file mode 100644 index 0000000..9e1f7ed --- /dev/null +++ b/org.glite.jp.common/project/configure.properties.xml @@ -0,0 +1,62 @@ + + + + + + + + + + + +top_srcdir=.. +builddir=build +stagedir=${stage.abs.dir} +distdir=${dist.dir} +globalprefix=${global.prefix} +jpprefix=${subsystem.prefix} +package=${module.package.name} +PREFIX=${install.dir} +version=${module.version} +glite_location=${with.glite.location} +globus_prefix=${with.globus.prefix} +expat_prefix=${with.expat.prefix} +ares_prefix=${with.ares.prefix} +thrflavour=${with.globus.thr.flavor} +nothrflavour=${with.globus.nothr.flavor} +cppunit=${with.cppunit.prefix} +jpproject=${subsystem.project.dir} +project=${component.project.dir} + + + diff --git a/org.glite.jp.common/project/properties.xml b/org.glite.jp.common/project/properties.xml new file mode 100755 index 0000000..2d7b685 --- /dev/null +++ b/org.glite.jp.common/project/properties.xml @@ -0,0 +1,65 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.glite.jp.common/src/context.c b/org.glite.jp.common/src/context.c new file mode 100644 index 0000000..016bc59 --- /dev/null +++ b/org.glite.jp.common/src/context.c @@ -0,0 +1,164 @@ +#include +#include + +#include "types.h" +#include "context.h" + +int glite_jp_init_context(glite_jp_context_t *ctx) +{ + *ctx = calloc(1,sizeof **ctx); +} + +char *glite_jp_peer_name(glite_jp_context_t ctx) +{ + return strdup("unknown"); +} + +char *glite_jp_error_chain(glite_jp_context_t ctx) +{ + char *ret = NULL,indent[300] = ""; + int len = 0,add; + char buf[2000]; + + glite_jp_error_t *ep = ctx->error; + + do { + add = snprintf(buf,sizeof buf,"%s%s: %s (%s)\n", + indent, + ep->source, + strerror(ep->code), + ep->desc ? ep->desc : ""); + ret = realloc(ret,len + add + 1); + strncpy(ret + len,buf,add); ret[len += add] = 0; + strcat(indent," "); + } while (ep = ep->reason); + + return ret; +} + +int glite_jp_stack_error(glite_jp_context_t ctx, const glite_jp_error_t *err) +{ + glite_jp_error_t *reason = ctx->error; + + ctx->error = calloc(1,sizeof *ctx->error); + ctx->error->code = err->code; + ctx->error->desc = err->desc ? strdup(err->desc) : NULL; + ctx->error->source = err->source ? strdup(err->source) : NULL; + ctx->error->reason = reason; + + return err->code; +} + +int glite_jp_clear_error(glite_jp_context_t ctx) +{ + glite_jp_error_t *e = ctx->error, *r; + + while (e) { + r = e->reason; + free(e->source); + free(e->desc); + free(e); + e = r; + } + ctx->error = NULL; + return 0; +} + + +void glite_jp_free_query_rec(glite_jp_query_rec_t *q) +{ + free(q->attr.name); q->attr.name = NULL; + switch (q->attr.type) { + case GLITE_JP_ATTR_OWNER: + case GLITE_JP_ATTR_TAG: + free(q->value.s); q->value.s = NULL; + if (q->op == GLITE_JP_QUERYOP_WITHIN) { + free(q->value2.s); + q->value2.s = NULL; + } + break; + default: break; + } +} + +int glite_jp_attr_copy(glite_jp_attr_t *dst,const glite_jp_attr_t *src) +{ + dst->name = src->name ? strdup(src->name) : NULL; + dst->type = src->type; + return 0; +} + +int glite_jp_queryrec_copy(glite_jp_query_rec_t *dst, const glite_jp_query_rec_t *src) +{ + glite_jp_attr_copy(&dst->attr,&src->attr); + dst->op = src->op; + switch (src->attr.type) { + case GLITE_JP_ATTR_OWNER: + case GLITE_JP_ATTR_TAG: + dst->value.s = strdup(src->value.s); + if (dst->op == GLITE_JP_QUERYOP_WITHIN) + dst->value2.s = strdup(src->value2.s); + break; + case GLITE_JP_ATTR_TIME: + memcpy(&dst->value.time,&src->value.time,sizeof dst->value.time); + if (dst->op == GLITE_JP_QUERYOP_WITHIN) + memcpy(&dst->value2.time,&src->value2.time,sizeof dst->value2.time); + break; + } + return 0; +} + +int glite_jp_run_deferred(glite_jp_context_t ctx) +{ + int i,cnt,ret; + + if (!ctx->deferred_func) return 0; + + glite_jp_clear_error(ctx); + for (cnt=0;ctx->deferred_func[cnt];cnt++); + for (i=0; ideferred_func)(ctx,*ctx->deferred_arg)) { + glite_jp_error_t err; + char desc[100]; + + sprintf(desc,"calling func #%d, %p",i,*ctx->deferred_func); + err.code = ret; + err.desc = desc; + err.source = "glite_jp_run_deferred()"; + + glite_jp_stack_error(ctx,&err); + return ret; + } + else { + memmove(ctx->deferred_func,ctx->deferred_func+1, + (cnt-i) * sizeof *ctx->deferred_func); + memmove(ctx->deferred_arg,ctx->deferred_arg+1, + (cnt-i) * sizeof *ctx->deferred_arg); + } + } + free(ctx->deferred_func); ctx->deferred_func = NULL; + free(ctx->deferred_arg); ctx->deferred_arg = NULL; + return 0; +} + +int glite_jp_add_deferred( + glite_jp_context_t ctx, + int (*func)(glite_jp_context_t, void *), + void *arg +) +{ + int (**v)(glite_jp_context_t, void *) = ctx->deferred_func; + int i; + + for (i=0; v && *v; i++); + + ctx->deferred_func = realloc(ctx->deferred_func, (i+1) * sizeof *ctx->deferred_func); + ctx->deferred_func[i] = func; + ctx->deferred_func[i+1] = NULL; + + ctx->deferred_arg = realloc(ctx->deferred_arg,(i+1) * sizeof *ctx->deferred_arg); + ctx->deferred_arg[i] = arg; + ctx->deferred_arg[i+1] = NULL; + + return 0; +} diff --git a/org.glite.jp.common/src/strmd5.c b/org.glite.jp.common/src/strmd5.c new file mode 100755 index 0000000..87fd400 --- /dev/null +++ b/org.glite.jp.common/src/strmd5.c @@ -0,0 +1,115 @@ +#include +#include +#include +#include + +#include "strmd5.h" + +static char mbuf[33]; + +static int base64_encode(const void *enc, int enc_size, char *out, int out_max_size) +{ + static const char* b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + + unsigned char* enc_buf = (unsigned char*)enc; + int out_size = 0; + unsigned int bits = 0; + unsigned int shift = 0; + + while ( out_size < out_max_size ) { + if ( enc_size>0 ) { + // Shift in byte + bits <<= 8; + bits |= *enc_buf; + shift += 8; + // Next byte + enc_buf++; + enc_size--; + } else if ( shift>0 ) { + // Pad last bits to 6 bits - will end next loop + bits <<= 6 - shift; + shift = 6; + } else { + // Terminate with Mime style '=' + *out = '='; + out_size++; + + return out_size; + } + + // Encode 6 bit segments + while ( shift>=6 ) { + shift -= 6; + *out = b64[ (bits >> shift) & 0x3F ]; + out++; + out_size++; + } + } + + // Output overflow + return -1; +} + +char *strmd5(const char *s, unsigned char *digest) +{ + MD5_CTX md5; + unsigned char d[16]; + int i; + + MD5_Init(&md5); + MD5_Update(&md5,s,strlen(s)); + MD5_Final(d,&md5); + + if (digest) memcpy(digest,d,sizeof(d)); + + for (i=0; i<16; i++) { + int dd = d[i] & 0x0f; + mbuf[2*i+1] = dd<10 ? dd+'0' : dd-10+'a'; + dd = d[i] >> 4; + mbuf[2*i] = dd<10 ? dd+'0' : dd-10+'a'; + } + mbuf[32] = 0; + return (char *) mbuf; +} + +char *str2md5(const char *s) +{ + MD5_CTX md5; + unsigned char d[16]; + char* ret = malloc(33); + int i; + + if (!ret) + return NULL; + + MD5_Init(&md5); + MD5_Update(&md5, s, strlen(s)); + MD5_Final(d, &md5); + + for (i=0; i<16; i++) { + int dd = d[i] & 0x0f; + ret[2*i+1] = dd<10 ? dd+'0' : dd-10+'a'; + dd = d[i] >> 4; + ret[2*i] = dd<10 ? dd+'0' : dd-10+'a'; + } + ret[32] = 0; + return ret; +} + +char *str2md5base64(const char *s) +{ + MD5_CTX md5; + unsigned char d[16]; + char buf[50]; + int l; + + MD5_Init(&md5); + MD5_Update(&md5, s, strlen(s)); + MD5_Final(d, &md5); + + l = base64_encode(d, 16, buf, sizeof(buf) - 1); + if (l < 1) + return NULL; + buf[l - 1] = 0; + return strdup(buf); +} diff --git a/org.glite.jp.index/Makefile b/org.glite.jp.index/Makefile new file mode 100644 index 0000000..3b893cd --- /dev/null +++ b/org.glite.jp.index/Makefile @@ -0,0 +1,129 @@ +# defaults +top_srcdir=. +builddir=build +top_builddir=${top_srcdir}/${builddir} +stagedir=. +distdir=. +globalprefix=glite +lbprefix=lb +package=glite-lb-server +version=0.0.0 +PREFIX=/opt/glite + +glite_location=/opt/glite +globus_prefix=/opt/globus +nothrflavour=gcc32 +thrflavour=gcc32pthr +expat_prefix=/opt/expat +ares_prefix=/opt/ares +gsoap_prefix=/software/gsoap-2.6 + +CC=gcc + +-include Makefile.inc + + +VPATH=${top_srcdir}/src:${top_srcdir}/examples:${top_srcdir}/test:${top_srcdir}/project:${jpproject} + +GLOBUS_LIBS:=-L${globus_prefix}/lib \ + -lglobus_common_${nothrflavour} \ + -lglobus_gssapi_gsi_${nothrflavour} + +DEBUG:=-g -O0 -DDEBUG + +CFLAGS:=${DEBUG} -I. -I${gsoap_prefix}/include -I${stagedir}/include +LDFLAGS:=-L${stagedir}/lib + +LINK:=libtool --mode=link ${CC} ${LDFLAGS} +LINKXX:=libtool --mode=link ${CXX} ${LDFLAGS} +INSTALL:=libtool --mode=install install + + +daemon:=glite-jp-indexd +example:=jpis-test +soap_prefix:=jpis_ + +SRCS:= simple_server.c soap_ops.c \ + ${soap_prefix}C.c \ + ${soap_prefix}Server.c + +EXA_SRCS:=jpis-test.c ${soap_prefix}C.c ${soap_prefix}Client.c + + +OBJS:=${SRCS:.c=.o} stdsoap2.o +EXA_OBJS:=${EXA_SRCS:.c=.o} stdsoap2.o + +COMMONLIB:=-lglite_jp_common + +default all: compile + +compile: ${daemon} ${example} + +${daemon}: ${OBJS} + ${LINK} -o $@ ${OBJS} ${COMMONLIB} ${GLOBUS_LIBS} + +${example}: ${EXA_OBJS} + ${LINK} -o $@ ${EXA_OBJS} + +JobProvenanceIS.xh: JobProvenanceIS.wsdl JobProvenanceTypes.wsdl typemap.dat + cp ${jpproject}/JobProvenanceTypes.wsdl . + ${gsoap_prefix}/bin/wsdl2h -t ${top_srcdir}/src/typemap.dat -c -o $@ $< + rm -f JobProvenanceTypes.wsdl + +${soap_prefix}C.c ${soap_prefix}H.h: JobProvenanceIS.xh + ${gsoap_prefix}/bin/soapcpp2 -w -c -p ${soap_prefix} JobProvenanceIS.xh + +#$(SOAP_PREFIX)H.h $(SOAP_PREFIX)C.c: LB.xh +# $(GSOAP_BIN_PATH)/soapcpp2 -w -c -p $(SOAP_PREFIX) LB.xh +# +#LB.xh: LB.wsdl typemap.dat +# $(GSOAP_BIN_PATH)/wsdl2h -c -o $@ LB.wsdl +# + + +check: + -echo nothing yet + +doc: + +stage: compile + ${INSTALL} -m 755 ${daemon} ${stagedir}/bin + +dist: distsrc distbin + +# FIXME: just copied from LB +distsrc: + mkdir -p ${top_srcdir}/${package}-${version} + cd ${top_srcdir} && GLOBIGNORE="${package}-${version}" && cp -Rf * ${package}-${version} + cd ${top_srcdir} && tar -czf ${distdir}/${package}-${version}_src.tar.gz --exclude-from=project/tar_exclude ${package}-${version} + rm -rf ${top_srcdir}/${package}-${version} + +distbin: + $(MAKE) install PREFIX=`pwd`/tmpbuilddir${stagedir} + save_dir=`pwd`; cd tmpbuilddir${stagedir} && tar -czf $$save_dir/${top_srcdir}/${distdir}/${package}-${version}_bin.tar.gz *; cd $$save_dir + rm -rf tmpbuilddir + +install: + -mkdir -p ${PREFIX}/bin ${PREFIX}/etc ${PREFIX}/etc/init.d + for p in bkserverd bkindex; do \ + ${INSTALL} -m 755 "glite_lb_$$p" "${PREFIX}/bin/glite-lb-$$p"; \ + done + + for f in dbsetup.sql index.conf.template; do \ + ${INSTALL} -m 644 ${top_srcdir}/config/"glite-lb-$$f" ${PREFIX}/etc; \ + done + + ${INSTALL} -m 755 ${top_srcdir}/config/startup ${PREFIX}/etc/init.d/glite-lb-bkserverd + +clean: + +soap_ops.o jpis-test.o simple_server.o: ${soap_prefix}H.h + +# we have no real config.h but have to force gSoap not to use +# linux ftime with broken (aka obsolete) DST information + +stdsoap2.o: ${gsoap_prefix}/devel/stdsoap2.c + test -f config.h || touch config.h + @echo 'The following warning "time_t (de)serialization is not MT safe on this platform" is harmless' + ${CC} -o $@ -c -DHAVE_CONFIG_H ${CFLAGS} ${gsoap_prefix}/devel/stdsoap2.c + diff --git a/org.glite.jp.index/build.xml b/org.glite.jp.index/build.xml new file mode 100755 index 0000000..7f0b6cb --- /dev/null +++ b/org.glite.jp.index/build.xml @@ -0,0 +1,118 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.glite.jp.index/project/JobProvenanceIS.wsdl b/org.glite.jp.index/project/JobProvenanceIS.wsdl new file mode 100644 index 0000000..26f97a4 --- /dev/null +++ b/org.glite.jp.index/project/JobProvenanceIS.wsdl @@ -0,0 +1,531 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Register job with JP primary storage. +Job registration in LB is propagated to JP immediately so that JP is aware of the job, +despite no furhter information is available in it. + +Input: JobId + +Output: N/A + +Faults: GenericJPFault + + + + + + + Initiate upload of of sandbox/dump of job life log from LB. +WM component responsible for job sandbox management and LB server call JP to declare +intention to upload intput/output sandbox and job life log. + +Input: + +uclass: type of the upload - INPUT_SANDBOX, OUTPUT_SANDBOX, JOB_LOG + +commitTimeout: upper limit on time for which JP waits for committing this upload transaction + +contentType: MIME type of the uploaded file + +Output: + +destination: URL where the client should upload the file + +commitBefore: acutual time when the upload transaction times out + +Faults: GenericJPFault + + +Initiate upload of of sandbox/dump of job life log from LB. + + + + + + Confirm upload. +Should be called after a file upload initiaded with StartUpload is finished. + +Input: + +destination: Upload destination URL (to match with the original request) + +Output: + +Faults: GenericJPFault + + + + + + + Record a value of user tag. +JP tags are either standalone or override values of their LB counterparts. +However, JP tag values are still distinguishable those inherited from LB. +JP tags may be either strings or blobs. + +Input: + +jobid: + +tag: structure containing name, timestamp, optional sequence number to order tag values +without relying on timestamps, and string or blob value. + +Output: N/A + +Faults: GenericJPFault + + + + + + + Start feeding JP index server. +Called by the index server to start batch feed, and optionally also subscribe for incremental feed. + +JP index server subscribes with JP primary storage using a query +containing conditions on primary metadata and a list of queryable attributes +of the index server (i.e. data which should be sent to the index server). + +When a matching job record is created or modified within the primary storage +the job record data are sent to the subscribed index server. + +The subscription is soft-state, it expires after certain time unless refreshed by the client explicitely. + +In the batch mode the query has the same form +with additional flag asking for all matching records (i.e.\ not only +arriving afterwards). + +Input: + +destination: where to send the job record data + +attributes: which job record attributes should be sent to the requesting index server + +conditions: list of query conditions. Each conditions has the form Attribute Operator Value, +where Attribute is any of job record attributes and Operator is one of EQUAL, UNEQUAL, LESS, GREATER, WITHIN. + +continuous: flag determining that the query is incremental (not batch) + +Output: + +feedId: unique Id of the feed request, to be used in refresh, cancelation etc. + +expires: when the feed times out. Must be refreshed before this time. + +Faults: GenericJPFault + + + + + + + Extend batch feed subscription (used by index server) + +Input: feedId returned previously by FeedIndex + +Output: the same as for FeedIndex + +Faults: GenericJPFault + + + + + + + Retrieve job record URL's when jobid is known +Used either to bypass JP index server query for this specific case, or after the index server query to +retrieve actual job record. + +Input: jobid + +Output: + +jobLog, inputSandbox, outputSandbox, tags: URL's to components of the job record. + +Faults: GenericJPFault + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Job Provenance Primary Storage service + + + + + + + + + + + + + + + + + + + + + + Store or update information on jobs within the JP index server. +Called directly by the primary storage, used for both batch and incremental feed. + +Input: + +data: list of job record updates. Each contains jobid, list of JP attribute values and user tag values. + +feedDone: flag indicating end of batch feed. (In order to avoid potential problems with buffer allocation +the huge dataset of batch feed is split into reasonable chunks and delivered with more UpdateJobs calls.) + +Output: N/A + +Faults: GenericJPFault + + + + + + + + + Retrieve pointers to job records of jobs matching a query. +Input: conditions - list of lists of query conditions. + Elements of the inner lists refer to a single job attribute, the conditions are or-ed. + Elements of the outer list may refer to different job attributes, they are and-ed. + +Output: + +jobs: list of JobId, PSContact (URL of the primary storage which manges this job) pairs + +Faults: GenericJPFault + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Job Provenance Index service + + + + + + + diff --git a/org.glite.jp.index/project/configure.properties.xml b/org.glite.jp.index/project/configure.properties.xml new file mode 100644 index 0000000..e72730c --- /dev/null +++ b/org.glite.jp.index/project/configure.properties.xml @@ -0,0 +1,68 @@ + + + + + + + + + + + +top_srcdir=.. +builddir=build +stagedir=${stage.abs.dir} +distdir=${dist.dir} +globalprefix=${global.prefix} +lbprefix=${subsystem.prefix} +package=${module.package.name} +PREFIX=${install.dir} +version=${module.version} +glite_location=${with.glite.location} +globus_prefix=${with.globus.prefix} +expat_prefix=${with.expat.prefix} +ares_prefix=${with.ares.prefix} +thrflavour=${with.globus.thr.flavor} +nothrflavour=${with.globus.nothr.flavor} +cppunit=${with.cppunit.prefix} +jpproject=${subsystem.project.dir} +project=${component.project.dir} + + + diff --git a/org.glite.jp.index/project/properties.xml b/org.glite.jp.index/project/properties.xml new file mode 100755 index 0000000..cdd32d9 --- /dev/null +++ b/org.glite.jp.index/project/properties.xml @@ -0,0 +1,71 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.glite.jp.index/src/simple_server.c b/org.glite.jp.index/src/simple_server.c new file mode 100644 index 0000000..ccaa5a7 --- /dev/null +++ b/org.glite.jp.index/src/simple_server.c @@ -0,0 +1,39 @@ +#include "glite/jp/types.h" +#include "glite/jp/context.h" + +#include "jpis_H.h" + +int main() { + struct soap soap; + int i, m, s; // master and slave sockets + + glite_jp_context_t ctx; + + soap_init(&soap); + glite_jp_init_context(&ctx); + soap.user = (void *) ctx; + + srand48(time(NULL)); /* feed id generation */ + + m = soap_bind(&soap, NULL, 8902, 100); + if (m < 0) + soap_print_fault(&soap, stderr); + else + { + fprintf(stderr, "Socket connection successful: master socket = %d\n", m); + for (i = 1; ; i++) { + s = soap_accept(&soap); + if (s < 0) { + soap_print_fault(&soap, stderr); + break; + } + soap_serve(&soap); // process RPC request + soap_destroy(&soap); // clean up class instances + soap_end(&soap); // clean up everything and close socket + glite_jp_run_deferred(ctx); + } + } + soap_done(&soap); // close master socket + + return 0; +} diff --git a/org.glite.jp.index/src/soap_ops.c b/org.glite.jp.index/src/soap_ops.c new file mode 100644 index 0000000..5af7352 --- /dev/null +++ b/org.glite.jp.index/src/soap_ops.c @@ -0,0 +1,81 @@ +#include +#include + +#include "glite/jp/types.h" +#include "glite/jp/context.h" + +#include "jpis_H.h" +#include "JobProvenanceIS.nsmap" + +static struct jptype__GenericJPFaultType *jp2s_error(struct soap *soap, + const glite_jp_error_t *err) +{ + struct jptype__GenericJPFaultType *ret = NULL; + if (err) { + ret = soap_malloc(soap,sizeof *ret); + memset(ret,0,sizeof *ret); + ret->code = err->code; + ret->source = soap_strdup(soap,err->source); + ret->text = soap_strdup(soap,strerror(err->code)); + ret->description = soap_strdup(soap,err->desc); + ret->reason = jp2s_error(soap,err->reason); + } + return ret; +} + +static void err2fault(const glite_jp_context_t ctx,struct soap *soap) +{ + char *et; + struct SOAP_ENV__Detail *detail = soap_malloc(soap,sizeof *detail); + struct _GenericJPFault *f = soap_malloc(soap,sizeof *f); + + + f->jptype__GenericJPFault = jp2s_error(soap,ctx->error); + + detail->__type = SOAP_TYPE__GenericJPFault; + detail->value = f; + detail->__any = NULL; + + soap_receiver_fault(soap,"Oh, shit!",NULL); + if (soap->version == 2) soap->fault->SOAP_ENV__Detail = detail; + else soap->fault->detail = detail; +} + +static void s2jp_tag(const struct jptype__TagValue *stag,glite_jp_tagval_t *jptag) +{ + memset(jptag,0,sizeof *jptag); + jptag->name = strdup(stag->name); + jptag->sequence = stag->sequence ? *stag->sequence : 0; + jptag->timestamp = stag->timestamp ? *stag->timestamp : 0; + if (stag->stringValue) jptag->value = strdup(stag->stringValue); + else if (stag->blobValue) { + jptag->binary = 1; + jptag->size = stag->blobValue->__size; + jptag->value = (char *) stag->blobValue->__ptr; + } +} + +#define CONTEXT_FROM_SOAP(soap,ctx) glite_jp_context_t ctx = (glite_jp_context_t) ((soap)->user) + +SOAP_FMAC5 int SOAP_FMAC6 jpsrv__UpdateJobs( + struct soap *soap, + char *feed_id, + struct jptype__UpdateJobsData *jobs, + enum xsd__boolean done +) +{ + printf("%s items %d jobid %s\n",__FUNCTION__,jobs->__sizejob, + jobs->job[0]->jobid); + return SOAP_OK; +} + +SOAP_FMAC5 int SOAP_FMAC6 jpsrv__QueryJobs( + struct soap *soap, + struct jptype__IndexQuery *query, + struct jpsrv__QueryJobsResponse *resp +) +{ + puts(__FUNCTION__); + return SOAP_OK; +} + diff --git a/org.glite.jp.index/src/typemap.dat b/org.glite.jp.index/src/typemap.dat new file mode 100644 index 0000000..7032cb2 --- /dev/null +++ b/org.glite.jp.index/src/typemap.dat @@ -0,0 +1,2 @@ +jpsrv = http://glite.org/wsdl/services/jp +jptype = http://glite.org/wsdl/types/jp diff --git a/org.glite.jp.primary/Makefile b/org.glite.jp.primary/Makefile new file mode 100644 index 0000000..65c6380 --- /dev/null +++ b/org.glite.jp.primary/Makefile @@ -0,0 +1,141 @@ +# defaults +top_srcdir=. +builddir=build +top_builddir=${top_srcdir}/${builddir} +stagedir=. +distdir=. +globalprefix=glite +lbprefix=lb +package=glite-lb-server +version=0.0.0 +PREFIX=/opt/glite + +glite_location=/opt/glite +globus_prefix=/opt/globus +nothrflavour=gcc32 +thrflavour=gcc32pthr +expat_prefix=/opt/expat +ares_prefix=/opt/ares +gsoap_prefix=/software/gsoap-2.6 + +CC=gcc + +-include Makefile.inc + + +VPATH=${top_srcdir}/src:${top_srcdir}/examples:${top_srcdir}/test:${top_srcdir}/project:${jpproject} + +GLOBUS_LIBS:=-L${globus_prefix}/lib \ + -lglobus_common_${nothrflavour} \ + -lglobus_gssapi_gsi_${nothrflavour} + +DEBUG:=-g -O0 -DDEBUG + +CFLAGS:=${DEBUG} -I. -I${gsoap_prefix}/include -I${stagedir}/include +LDFLAGS:=-L${stagedir}/lib + +LINK:=libtool --mode=link ${CC} ${LDFLAGS} +LINKXX:=libtool --mode=link ${CXX} ${LDFLAGS} +INSTALL:=libtool --mode=install install + + +daemon:=glite-jp-primarystoraged +example:=jpps-test +ps_prefix:=jpps_ +is_prefix:=jpis_ + +SRCS:= simple_server.c soap_ops.c \ + ftp_backend.c \ + feed.c tags.c\ + ${ps_prefix}ServerLib.c \ + ${is_prefix}ClientLib.c \ + env_C.c + +EXA_SRCS:=jpps-test.c ${ps_prefix}C.c ${ps_prefix}Client.c + + +OBJS:=${SRCS:.c=.o} stdsoap2.o +EXA_OBJS:=${EXA_SRCS:.c=.o} stdsoap2.o + +COMMONLIB:=-lglite_jp_common + +default all: compile + +compile: ${daemon} ${example} + +${daemon}: ${OBJS} + ${LINK} -o $@ ${OBJS} ${COMMONLIB} ${GLOBUS_LIBS} + +${example}: ${EXA_OBJS} + ${LINK} -o $@ ${EXA_OBJS} + +JobProvenanceIS.xh JobProvenancePS.xh: %.xh: %.wsdl JobProvenanceTypes.wsdl typemap.dat + cp ${jpproject}/JobProvenanceTypes.wsdl . + ${gsoap_prefix}/bin/wsdl2h -t ${top_srcdir}/src/typemap.dat -c -o $@ $< + rm -f JobProvenanceTypes.wsdl + +${ps_prefix}ServerLib.c ${ps_prefix}H.h: JobProvenancePS.xh + ${gsoap_prefix}/bin/soapcpp2 -n -w -c -p ${ps_prefix} JobProvenancePS.xh + +${is_prefix}ClientLib.c ${is_prefix}H.h: JobProvenanceIS.xh + ${gsoap_prefix}/bin/soapcpp2 -n -w -c -p ${is_prefix} JobProvenanceIS.xh + +env_C.c env_Server.c: + touch env.xh + ${gsoap_prefix}/bin/soapcpp2 -w -c -p env_ env.xh + +#$(SOAP_PREFIX)H.h $(SOAP_PREFIX)C.c: LB.xh +# $(GSOAP_BIN_PATH)/soapcpp2 -w -c -p $(SOAP_PREFIX) LB.xh +# +#LB.xh: LB.wsdl typemap.dat +# $(GSOAP_BIN_PATH)/wsdl2h -c -o $@ LB.wsdl +# + + + +check: + -echo nothing yet + +doc: + +stage: compile + ${INSTALL} -m 755 ${daemon} ${stagedir}/bin + +dist: distsrc distbin + +# FIXME: just copied from LB +distsrc: + mkdir -p ${top_srcdir}/${package}-${version} + cd ${top_srcdir} && GLOBIGNORE="${package}-${version}" && cp -Rf * ${package}-${version} + cd ${top_srcdir} && tar -czf ${distdir}/${package}-${version}_src.tar.gz --exclude-from=project/tar_exclude ${package}-${version} + rm -rf ${top_srcdir}/${package}-${version} + +distbin: + $(MAKE) install PREFIX=`pwd`/tmpbuilddir${stagedir} + save_dir=`pwd`; cd tmpbuilddir${stagedir} && tar -czf $$save_dir/${top_srcdir}/${distdir}/${package}-${version}_bin.tar.gz *; cd $$save_dir + rm -rf tmpbuilddir + +install: + -mkdir -p ${PREFIX}/bin ${PREFIX}/etc ${PREFIX}/etc/init.d + for p in bkserverd bkindex; do \ + ${INSTALL} -m 755 "glite_lb_$$p" "${PREFIX}/bin/glite-lb-$$p"; \ + done + + for f in dbsetup.sql index.conf.template; do \ + ${INSTALL} -m 644 ${top_srcdir}/config/"glite-lb-$$f" ${PREFIX}/etc; \ + done + + ${INSTALL} -m 755 ${top_srcdir}/config/startup ${PREFIX}/etc/init.d/glite-lb-bkserverd + +clean: + +simple_server.o soap_ops.o jpps-test.o: ${ps_prefix}H.h + +# we have no real config.h but have to force gSoap not to use +# linux ftime with broken (aka obsolete) DST information + +stdsoap2.o: ${gsoap_prefix}/devel/stdsoap2.c + test -f config.h || touch config.h + @echo 'The following warning "time_t (de)serialization is not MT safe on this platform" is harmless' + ${CC} -o $@ -c -DWITH_NONAMESPACES -DHAVE_CONFIG_H ${CFLAGS} ${gsoap_prefix}/devel/stdsoap2.c + diff --git a/org.glite.jp.primary/build.xml b/org.glite.jp.primary/build.xml new file mode 100755 index 0000000..b6f440e --- /dev/null +++ b/org.glite.jp.primary/build.xml @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.glite.jp.primary/examples/jpps-test.c b/org.glite.jp.primary/examples/jpps-test.c new file mode 100644 index 0000000..60c9c5f --- /dev/null +++ b/org.glite.jp.primary/examples/jpps-test.c @@ -0,0 +1,129 @@ +#include +#include +#include +#include + +#include "jpps_H.h" +#include "jpps_.nsmap" + +static void usage(const char *me) +{ + fprintf(stderr,"%s: [-s server-url] operation args \n\n" + " operations are:\n" + " RegisterJob jobid\n" + " StartUpload\n" + " CommitUpload\n" + " RecordTag\n" + " GetJob\n" + " FeedIndex destination query_number history continuous\n" + ,me); + + exit (EX_USAGE); +} + +static int check_fault(struct soap *soap,int err) { + struct SOAP_ENV__Detail *detail; + struct jptype__GenericJPFaultType *f; + char *reason,indent[200] = " "; + + switch(err) { + case SOAP_OK: puts("OK"); + break; + case SOAP_FAULT: + case SOAP_SVR_FAULT: + if (soap->version == 2) { + detail = soap->fault->SOAP_ENV__Detail; + reason = soap->fault->SOAP_ENV__Reason; + } + else { + detail = soap->fault->detail; + reason = soap->fault->faultstring; + } + fputs(reason,stderr); + putc(10,stderr); + assert(detail->__type == SOAP_TYPE__GenericJPFault); + f = ((struct _GenericJPFault *) detail->value) + -> jptype__GenericJPFault; + + while (f) { + fprintf(stderr,"%s%s: %s (%s)\n",indent, + f->source,f->text,f->description); + f = f->reason; + strcat(indent," "); + } + return -1; + + default: soap_print_fault(soap,stderr); + return -1; + } + return 0; +} + +static struct jptype__Attribute sample_attr[] = { + { OWNER, NULL }, + { TIME, "submitted" }, + { TAG, "test" }, +}; + +static struct jptype__PrimaryQueryElement sample_query[][5] = { + { + { sample_attr+OWNER, EQUAL, "unknown", NULL }, + { NULL, 0, NULL, NULL } + }, +}; + +int main(int argc,char *argv[]) +{ + char *server = "http://localhost:8901"; + int opt; + struct soap *soap = soap_new(); + + if (argc < 2) usage(argv[0]); + + soap_init(soap); + soap_set_namespaces(soap, jpps__namespaces); + + while ((opt = getopt(argc,argv,"s:")) >= 0) switch (opt) { + case 's': server = optarg; + break; + case '?': usage(argv[0]); + } + + if (!strcasecmp(argv[1],"RegisterJob")) { + struct jpsrv__RegisterJobResponse r; + + if (argc != 3) usage(argv[0]); + check_fault(soap, + soap_call_jpsrv__RegisterJob(soap,server,"",argv[2],&r)); + } + else if (!strcasecmp(argv[1],"FeedIndex")) { + struct jpsrv__FeedIndexResponse r; + struct jptype__Attribute *ap[2]; + struct jptype__Attributes attr = { 2, ap }; + struct jptype__PrimaryQueryElement *qp[100]; + struct jptype__PrimaryQuery qry = { 0, qp }; + + int i,j,qi = atoi(argv[3])-1; + + if (argc != 6) usage(argv[0]); + + for (i=0; i + + + + + + + + + + +top_srcdir=.. +builddir=build +stagedir=${stage.abs.dir} +distdir=${dist.dir} +globalprefix=${global.prefix} +lbprefix=${subsystem.prefix} +package=${module.package.name} +PREFIX=${install.dir} +version=${module.version} +glite_location=${with.glite.location} +globus_prefix=${with.globus.prefix} +expat_prefix=${with.expat.prefix} +ares_prefix=${with.ares.prefix} +thrflavour=${with.globus.thr.flavor} +nothrflavour=${with.globus.nothr.flavor} +cppunit=${with.cppunit.prefix} +jpproject=${subsystem.project.dir} +project=${component.project.dir} + + + diff --git a/org.glite.jp.primary/project/properties.xml b/org.glite.jp.primary/project/properties.xml new file mode 100755 index 0000000..182d6e4 --- /dev/null +++ b/org.glite.jp.primary/project/properties.xml @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.glite.jp.primary/src/backend.h b/org.glite.jp.primary/src/backend.h new file mode 100644 index 0000000..8dde271 --- /dev/null +++ b/org.glite.jp.primary/src/backend.h @@ -0,0 +1,104 @@ +#ifndef __GLITE_JP_BACKEND +#define __GLITE_JP_BACKEND + +#include +#include + +int glite_jppsbe_init( + glite_jp_context_t ctx, + int *argc, + char *argv[] +); + +int glite_jppsbe_init_slave( + glite_jp_context_t ctx +); + +int glite_jppsbe_register_job( + glite_jp_context_t ctx, + const char *job, + const char *owner +); + +int glite_jppsbe_start_upload( + glite_jp_context_t ctx, + const char *job, + glite_jp_fileclass_t class, + const char *content_type, + char **destination_out, + time_t *commit_before_inout +); + +int glite_jppsbe_commit_upload( + glite_jp_context_t ctx, + const char *destination +); + +int glite_jppsbe_destination_info( + glite_jp_context_t ctx, + const char *destination, + char **job_out, + glite_jp_fileclass_t *class_out +); + +int glite_jppsbe_get_job_url( + glite_jp_context_t ctx, + const char *job, + glite_jp_fileclass_t class, + char **url_out +); + +int glite_jppsbe_open_file( + glite_jp_context_t ctx, + const char *job, + glite_jp_fileclass_t class, + int mode, + void **handle_out +); + +int glite_jppsbe_close_file( + glite_jp_context_t ctx, + void *handle +); + +int glite_jppsbe_pread( + glite_jp_context_t ctx, + void *handle, + void *buf, + size_t nbytes, + off_t offset +); + +int glite_jppsbe_pwrite( + glite_jp_context_t ctx, + void *handle, + void *buf, + size_t nbytes, + off_t offset +); + +int glite_jppsbe_append( + glite_jp_context_t ctx, + void *handle, + void *buf, + size_t nbytes +); + +int glite_jppsbe_get_job_metadata( + glite_jp_context_t ctx, + const char *job, + glite_jp_attrval_t attrs_inout[] +); + +int glite_jppsbe_query( + glite_jp_context_t ctx, + const glite_jp_query_rec_t query[], + const glite_jp_attrval_t metadata[], + int (*callback)( + glite_jp_context_t ctx, + const char *job, + const glite_jp_attrval_t metadata[] + ) +); + +#endif diff --git a/org.glite.jp.primary/src/feed.c b/org.glite.jp.primary/src/feed.c new file mode 100644 index 0000000..1f7c2ac --- /dev/null +++ b/org.glite.jp.primary/src/feed.c @@ -0,0 +1,263 @@ +#include +#include +#include +#include +#include +#include + +#include "glite/jp/types.h" +#include "glite/jp/strmd5.h" +#include "feed.h" + + +/* + * seconds before feed expires: should be + * XXX: should be configurable, default for real deployment sort of 1 hour + */ +#define FEED_TTL 120 + +static int check_qry_item( + glite_jp_context_t ctx, + const glite_jp_query_rec_t *qry, + const glite_jp_attrval_t *attr +) +{ + int cmp,cmp2; + long scmp,ucmp; + + switch (qry->attr.type) { + case GLITE_JP_ATTR_OWNER: + case GLITE_JP_ATTR_TAG: + cmp = strcmp(attr->value.s,qry->value.s); + break; + case GLITE_JP_ATTR_TIME: + scmp = (ucmp = attr->value.time.tv_usec - qry->value.time.tv_usec) > 0 ? 0 : -1; + ucmp -= 1000000 * scmp; + scmp += attr->value.time.tv_sec - qry->value.time.tv_sec; + cmp = scmp ? scmp : ucmp; + break; + } + switch (qry->op) { + case GLITE_JP_QUERYOP_EQUAL: return !cmp; + case GLITE_JP_QUERYOP_UNEQUAL: return cmp; + case GLITE_JP_QUERYOP_LESS: return cmp < 0; + case GLITE_JP_QUERYOP_GREATER: return cmp > 0; + + case GLITE_JP_QUERYOP_WITHIN: + switch (qry->attr.type) { + case GLITE_JP_ATTR_OWNER: + case GLITE_JP_ATTR_TAG: + cmp2 = strcmp(attr->value.s,qry->value2.s); + break; + case GLITE_JP_ATTR_TIME: + scmp = (ucmp = attr->value.time.tv_usec - qry->value2.time.tv_usec) > 0 ? 0 : -1; + ucmp -= 1000000 * scmp; + scmp += attr->value.time.tv_sec - qry->value2.time.tv_sec; + cmp2 = scmp ? scmp : ucmp; + break; + } + return cmp >= 0 && cmp2 <= 0; + } +} + +/* XXX: limit on query size -- I'm lazy to malloc() */ +#define QUERY_MAX 100 + +static int match_feed( + glite_jp_context_t ctx, + const struct jpfeed *feed, + const char *job, + const glite_jp_attrval_t attrs[] /* XXX: not checked for correctness */ +) +{ + int i; + int attri[GLITE_JP_ATTR__LAST]; + int qi[QUERY_MAX]; + + glite_jp_attrval_t *newattr = NULL; + + glite_jp_clear_error(ctx); + + for (i=0; iqry) { + int j,complete = 1; + + memset(qi,0,sizeof qi); + for (i=0; feed->qry[i].attr.type; i++) { + assert(iqry[i].attr.type]) >=0) { + if (check_qry_item(ctx,feed->qry+i,attrs+j)) + qi[i] = 1; /* matched */ + else return 0; /* can't be satisfied */ + } + else complete = 0; + } + + /* not all attributes in query are known from input + * we have to retrieve job metadata from the backend + */ + if (!complete) { + glite_jp_attrval_t meta[GLITE_JP_ATTR__LAST+1]; + int qai[GLITE_JP_ATTR__LAST]; + + memset(meta,0,sizeof meta); + j=0; + for (i=0; feed->qry[i].attr.type; i++) if (!qi[i]) { + meta[j].attr.type = feed->qry[i].attr.type; + meta[j].attr.name = feed->qry[i].attr.name; + qai[feed->qry[i].attr.type] = i; + j++; + } + + if (glite_jppsbe_get_job_metadata(ctx,job,meta)) { + glite_jp_error_t err; + err.code = EIO; + err.source = __FUNCTION__; + err.desc = "complete query"; + return glite_jp_stack_error(ctx,&err); + } + + for (i=0; j=meta[i].attr.type; i++) + if (!check_qry_item(ctx,feed->qry+qai[j],meta+i)) + return 0; + } + } + + /* matched completely */ + return glite_jpps_single_feed(ctx,feed->destination,job,attrs); +} + +int glite_jpps_match_attr( + glite_jp_context_t ctx, + const char *job, + const glite_jp_attrval_t attrs[] +) +{ + struct jpfeed *f = (struct jpfeed *) ctx->feeds; + int i,j; + int attri[GLITE_JP_ATTR__LAST]; + + glite_jp_clear_error(ctx); + + for (i=0; i= GLITE_JP_ATTR__LAST || + attrs[i].attr.type <= 0) + { + glite_jp_error_t err; + err.code = EINVAL; + err.source = __FUNCTION__; + err.desc = "unknown attribute"; + return glite_jp_stack_error(ctx,&err); + } + if (attri[attrs[i].attr.type] >= 0) { + glite_jp_error_t err; + err.code = EINVAL; + err.source = __FUNCTION__; + err.desc = "double attribute change"; + return glite_jp_stack_error(ctx,&err); + } + + attri[attrs[i].attr.type] = i; + } + + for (;f; f = f->next) { + for (i=0; f->attrs[i].type && attri[f->attrs[i].type] == -1; i++); + /* XXX: ignore any errors */ + if (f->attrs[i].type) match_feed(ctx,f,job,attrs); + } + + return glite_jp_clear_error(ctx); +} + +int glite_jpps_match_file( + glite_jp_context_t ctx, + const char *job, + glite_jp_fileclass_t class +) +{ + fprintf(stderr,"%s: \n",__FUNCTION__); + return 0; +} + +int glite_jpps_match_tag( + glite_jp_context_t ctx, + const char *job, + const glite_jp_tagval_t *tag +) +{ + fprintf(stderr,"%s: \n",__FUNCTION__); + return 0; +} + +static char *generate_feedid(void) +{ + char hname[200],buf[1000]; + + gethostname(hname,sizeof hname); + snprintf(buf,sizeof buf,"%s%d%ld",hname,getpid(),lrand48()); + buf[sizeof buf-1] = 0; + return str2md5base64(buf); +} + + +int glite_jpps_run_feed( + glite_jp_context_t ctx, + const char *destination, + const glite_jp_attr_t *attrs, + const glite_jp_query_rec_t *qry, + char **feed_id) +{ + fprintf(stderr,"%s: \n",__FUNCTION__); + return 0; +} + +static int register_feed_deferred(glite_jp_context_t ctx,void *feed) +{ + struct jpfeed *f = feed; + + f->next = ctx->feeds; + ctx->feeds = f; + return 0; +} + +/* FIXME: + * - volatile implementation: should store the registrations in a file + * and recover after restart + * - should communicate the data among all server slaves + */ +int glite_jpps_register_feed( + glite_jp_context_t ctx, + const char *destination, + const glite_jp_attr_t *attrs, + const glite_jp_query_rec_t *qry, + char **feed_id, + time_t *expires) +{ + int i; + struct jpfeed *f = calloc(1,sizeof *f); + + if (!*feed_id) *feed_id = generate_feedid(); + time(expires); *expires += FEED_TTL; + + f->id = strdup(*feed_id); + f->destination = strdup(destination); + f->expires = *expires; + for (i=0; attrs[i].type; i++) { + f->attrs = realloc(f->attrs,(i+2) * sizeof *f->attrs); + glite_jp_attr_copy(f->attrs+i,attrs+i); + memset(f->attrs+i+1,0,sizeof *f->attrs); + } + for (i=0; qry[i].attr.type; i++) { + f->qry = realloc(f->qry,(i+2) * sizeof *f->qry); + glite_jp_queryrec_copy(f->qry+i,qry+i); + memset(f->qry+i+1,0,sizeof *f->qry); + } + + glite_jp_add_deferred(ctx,register_feed_deferred,f); + + return 0; +} + diff --git a/org.glite.jp.primary/src/feed.h b/org.glite.jp.primary/src/feed.h new file mode 100644 index 0000000..1551a82 --- /dev/null +++ b/org.glite.jp.primary/src/feed.h @@ -0,0 +1,21 @@ +#ifndef __GLITE_JP_FEED +#define __GLITE_JP_FEED + + +struct jpfeed { + char *id,*destination; + time_t expires; + glite_jp_attr_t *attrs; + glite_jp_query_rec_t *qry; + struct jpfeed *next; +}; + + +int glite_jpps_match_attr(glite_jp_context_t,const char *,const glite_jp_attrval_t[]); +int glite_jpps_match_file(glite_jp_context_t,const char *,glite_jp_fileclass_t); +int glite_jpps_match_tag(glite_jp_context_t,const char *,const glite_jp_tagval_t *); +int glite_jpps_run_feed(glite_jp_context_t,const char *,const glite_jp_attr_t *,const glite_jp_query_rec_t *,char **); +int glite_jpps_register_feed(glite_jp_context_t,const char *,const glite_jp_attr_t *,const glite_jp_query_rec_t *,char **,time_t *); + +#endif + diff --git a/org.glite.jp.primary/src/ftp_backend.c b/org.glite.jp.primary/src/ftp_backend.c new file mode 100644 index 0000000..d076125 --- /dev/null +++ b/org.glite.jp.primary/src/ftp_backend.c @@ -0,0 +1,166 @@ +#include "glite/jp/types.h" +#include "glite/jp/context.h" + +#include "backend.h" + +int glite_jppsbe_init( + glite_jp_context_t ctx, + int *argc, + char *argv[] +) +{ +} + +int glite_jppsbe_init_slave( + glite_jp_context_t ctx +) +{ +} + +int glite_jppsbe_register_job( + glite_jp_context_t ctx, + const char *job, + const char *owner +) +{ + glite_jp_clear_error(ctx); + puts(__FUNCTION__); + return 0; +} + +int glite_jppsbe_start_upload( + glite_jp_context_t ctx, + const char *job, + glite_jp_fileclass_t class, + const char *content_type, + char **destination_out, + time_t *commit_before_inout +) +{ + glite_jp_clear_error(ctx); + puts(__FUNCTION__); + return 0; +} + +int glite_jppsbe_commit_upload( + glite_jp_context_t ctx, + const char *destination +) +{ + glite_jp_clear_error(ctx); + puts(__FUNCTION__); + return 0; +} + +int glite_jppsbe_destination_info( + glite_jp_context_t ctx, + const char *destination, + char **job, + glite_jp_fileclass_t *class +) +{ + glite_jp_clear_error(ctx); + puts(__FUNCTION__); + return 0; +} + + +int glite_jppsbe_get_job_url( + glite_jp_context_t ctx, + const char *job, + glite_jp_fileclass_t class, + char **url_out +) +{ + glite_jp_clear_error(ctx); + puts(__FUNCTION__); + return 0; +} + +int glite_jppsbe_open_file( + glite_jp_context_t ctx, + const char *job, + glite_jp_fileclass_t class, + int mode, + void **handle_out +) +{ + glite_jp_clear_error(ctx); + puts(__FUNCTION__); + return 0; +} + +int glite_jppsbe_close_file( + glite_jp_context_t ctx, + void *handle +) +{ + glite_jp_clear_error(ctx); + puts(__FUNCTION__); + return 0; +} + +int glite_jppsbe_pread( + glite_jp_context_t ctx, + void *handle, + void *buf, + size_t nbytes, + off_t offset +) +{ + glite_jp_clear_error(ctx); + puts(__FUNCTION__); + return 0; +} + +int glite_jppsbe_pwrite( + glite_jp_context_t ctx, + void *handle, + void *buf, + size_t nbytes, + off_t offset +) +{ + glite_jp_clear_error(ctx); + puts(__FUNCTION__); + return 0; +} + +int glite_jppsbe_append( + glite_jp_context_t ctx, + void *handle, + void *buf, + size_t nbytes +) +{ + glite_jp_clear_error(ctx); + puts(__FUNCTION__); + return 0; +} + +int glite_jppsbe_get_job_metadata( + glite_jp_context_t ctx, + const char *job, + glite_jp_attrval_t attrs_inout[] +) +{ + glite_jp_clear_error(ctx); + puts(__FUNCTION__); + return 0; +} + +int glite_jppsbe_query( + glite_jp_context_t ctx, + const glite_jp_query_rec_t query[], + const glite_jp_attrval_t metadata[], + int (*callback)( + glite_jp_context_t ctx, + const char *job, + const glite_jp_attrval_t metadata[] + ) +) +{ + glite_jp_clear_error(ctx); + puts(__FUNCTION__); + return 0; +} diff --git a/org.glite.jp.primary/src/is_client.c b/org.glite.jp.primary/src/is_client.c new file mode 100644 index 0000000..cc594db --- /dev/null +++ b/org.glite.jp.primary/src/is_client.c @@ -0,0 +1,28 @@ +#include +#include +#include +#include +#include +#include + +#include "glite/jp/types.h" + +#include "feed.h" +#include "jpis_H.h" + +int glite_jpps_single_feed( + glite_jp_context_t ctx, + const char *destination, + const char *job, + const glite_jp_attrval_t attrs[] +) +{ + /* TODO: really call JP Index server (via interlogger) */ + printf("feed to %s, job %s\n",destination,job); + + /* FIXME: check fault */ + soap_call_jpsrv__UpdateJobs(ctx->other_soap,destination,"", + + + return 0; +} diff --git a/org.glite.jp.primary/src/simple_server.c b/org.glite.jp.primary/src/simple_server.c new file mode 100644 index 0000000..110c10c --- /dev/null +++ b/org.glite.jp.primary/src/simple_server.c @@ -0,0 +1,46 @@ +#include "glite/jp/types.h" +#include "glite/jp/context.h" + +#include "jpps_H.h" +#include "jpps_.nsmap" +#include "jpis_.nsmap" + +int main() { + struct soap soap; + int i, m, s; // master and slave sockets + + glite_jp_context_t ctx; + + soap_init(&soap); + soap_set_namespaces(&soap, jpps__namespaces); + + glite_jp_init_context(&ctx); + soap.user = (void *) ctx; + ctx->other_soap = soap_new(); + soap_init(ctx->other_soap); + soap_set_namespaces(ctx->other_soap,jpis__namespaces); + + srand48(time(NULL)); /* feed id generation */ + + m = soap_bind(&soap, NULL, 8901, 100); + if (m < 0) + soap_print_fault(&soap, stderr); + else + { + fprintf(stderr, "Socket connection successful: master socket = %d\n", m); + for (i = 1; ; i++) { + s = soap_accept(&soap); + if (s < 0) { + soap_print_fault(&soap, stderr); + break; + } + jpps__serve(&soap); // process RPC request + soap_destroy(&soap); // clean up class instances + soap_end(&soap); // clean up everything and close socket + glite_jp_run_deferred(ctx); + } + } + soap_done(&soap); // close master socket + + return 0; +} diff --git a/org.glite.jp.primary/src/soap_ops.c b/org.glite.jp.primary/src/soap_ops.c new file mode 100644 index 0000000..305f650 --- /dev/null +++ b/org.glite.jp.primary/src/soap_ops.c @@ -0,0 +1,354 @@ +#include +#include + +#include "glite/jp/types.h" +#include "glite/jp/context.h" + +#include "feed.h" + +#include "jpps_H.h" + +static struct jptype__GenericJPFaultType *jp2s_error(struct soap *soap, + const glite_jp_error_t *err) +{ + struct jptype__GenericJPFaultType *ret = NULL; + if (err) { + ret = soap_malloc(soap,sizeof *ret); + memset(ret,0,sizeof *ret); + ret->code = err->code; + ret->source = soap_strdup(soap,err->source); + ret->text = soap_strdup(soap,strerror(err->code)); + ret->description = soap_strdup(soap,err->desc); + ret->reason = jp2s_error(soap,err->reason); + } + return ret; +} + +static void err2fault(const glite_jp_context_t ctx,struct soap *soap) +{ + char *et; + struct SOAP_ENV__Detail *detail = soap_malloc(soap,sizeof *detail); + struct _GenericJPFault *f = soap_malloc(soap,sizeof *f); + + + f->jptype__GenericJPFault = jp2s_error(soap,ctx->error); + + detail->__type = SOAP_TYPE__GenericJPFault; + detail->value = f; + detail->__any = NULL; + + soap_receiver_fault(soap,"Oh, shit!",NULL); + if (soap->version == 2) soap->fault->SOAP_ENV__Detail = detail; + else soap->fault->detail = detail; +} + +static glite_jp_fileclass_t s2jp_fileclass(enum jptype__UploadClass class) +{ + switch (class) { + case INPUT_SANDBOX: return GLITE_JP_FILECLASS_INPUT; + case OUTPUT_SANDBOX: return GLITE_JP_FILECLASS_OUTPUT; + case JOB_LOG: return GLITE_JP_FILECLASS_LBLOG; + default: return GLITE_JP_FILECLASS_UNDEF; + } +} + +static void s2jp_tag(const struct jptype__TagValue *stag,glite_jp_tagval_t *jptag) +{ + memset(jptag,0,sizeof *jptag); + jptag->name = strdup(stag->name); + jptag->sequence = stag->sequence ? *stag->sequence : 0; + jptag->timestamp = stag->timestamp ? *stag->timestamp : 0; + if (stag->stringValue) jptag->value = strdup(stag->stringValue); + else if (stag->blobValue) { + jptag->binary = 1; + jptag->size = stag->blobValue->__size; + jptag->value = (char *) stag->blobValue->__ptr; + } +} + +#define CONTEXT_FROM_SOAP(soap,ctx) glite_jp_context_t ctx = (glite_jp_context_t) ((soap)->user) + +SOAP_FMAC5 int SOAP_FMAC6 jpsrv__RegisterJob( + struct soap *soap, + char *job, + struct jpsrv__RegisterJobResponse *response) +{ + CONTEXT_FROM_SOAP(soap,ctx); + char *owner = glite_jp_peer_name(ctx); + glite_jp_attrval_t owner_val[2]; + + if (glite_jppsbe_register_job(ctx,job,owner)) { + err2fault(ctx,soap); + free(owner); + return SOAP_FAULT; + } + + owner_val[0].attr.type = GLITE_JP_ATTR_OWNER; + owner_val[0].value.s = owner; + owner_val[1].attr.type = GLITE_JP_ATTR_UNDEF; + +/* XXX: errrors should be ingored but not silently */ + glite_jpps_match_attr(ctx,job,owner_val); + free(owner); + + return SOAP_OK; +} + + +SOAP_FMAC5 int SOAP_FMAC6 jpsrv__StartUpload( + struct soap *soap, + char *job, + enum jptype__UploadClass class, + time_t commit_before, + char *content_type, + struct jpsrv__StartUploadResponse *response) +{ + CONTEXT_FROM_SOAP(soap,ctx); + glite_jp_fileclass_t jclass = s2jp_fileclass(class); + char *destination; + + if (glite_jppsbe_start_upload(ctx,job,jclass,content_type,&destination,&commit_before)) { + err2fault(ctx,soap); + return SOAP_FAULT; + } + + response->destination = soap_strdup(soap,destination); + free(destination); + response->commitBefore = commit_before; + + return SOAP_OK; +} + +SOAP_FMAC5 int SOAP_FMAC6 jpsrv__CommitUpload( + struct soap *soap, + char *destination, + struct jpsrv__CommitUploadResponse *response) +{ + CONTEXT_FROM_SOAP(soap,ctx); + char *job = NULL; + glite_jp_fileclass_t class; + + if (glite_jppsbe_commit_upload(ctx,destination)) { + err2fault(ctx,soap); + return SOAP_FAULT; + } + + /* XXX: should not fail when commit_upload was OK */ + glite_jppsbe_destination_info(ctx,destination,&job,&class); + + /* XXX: ignore errors but don't fail silenty */ + glite_jpps_match_file(ctx,job,class); + + return SOAP_OK; +} + +SOAP_FMAC5 int SOAP_FMAC6 jpsrv__RecordTag( + struct soap *soap, + char *job, + struct jptype__TagValue *tag, + struct jpsrv__RecordTagResponse *response) +{ + CONTEXT_FROM_SOAP(soap,ctx); + void *tagfile; + + glite_jp_tagval_t mytag; + + if (glite_jppsbe_open_file(ctx,job,GLITE_JP_FILECLASS_TAGS, + O_WRONLY|O_CREAT,&tagfile)) + { + err2fault(ctx,soap); + return SOAP_FAULT; + } + + if (glite_jpps_tag_append(ctx,tagfile,&mytag)) { + err2fault(ctx,soap); + glite_jppsbe_close_file(ctx,tagfile); + return SOAP_FAULT; + } + + if (glite_jppsbe_close_file(ctx,tagfile)) { + err2fault(ctx,soap); + return SOAP_FAULT; + } + + /* XXX: ignore errors but don't fail silenty */ + + s2jp_tag(tag,&mytag); + glite_jpps_match_tag(ctx,job,&mytag); + + return SOAP_OK; +} + +static void s2jp_attr(const struct jptype__Attribute *in,glite_jp_attr_t *out) +{ + switch (in->type) { + case OWNER: out->type = GLITE_JP_ATTR_OWNER; break; + case TIME: out->type = GLITE_JP_ATTR_TIME; + out->name = strdup(in->name); + break; + case TAG: out->type = GLITE_JP_ATTR_TAG; + out->name = strdup(in->name); + break; + default: break; + } +} + +static void s2jp_queryval( + const char *in, + glite_jp_attrtype_t type, + union _glite_jp_query_rec_val *out) +{ + switch (type) { + case GLITE_JP_ATTR_OWNER: + case GLITE_JP_ATTR_TAG: + out->s = strdup(in); + break; + case GLITE_JP_ATTR_TIME: + out->time.tv_sec = atoi(in); + break; + } +} + +static void s2jp_query(const struct jptype__PrimaryQueryElement *in, glite_jp_query_rec_t *out) +{ + s2jp_attr(in->attr,&out->attr); + + switch (in->op) { + case EQUAL: out->op = GLITE_JP_QUERYOP_EQUAL; break; + case UNEQUAL: out->op = GLITE_JP_QUERYOP_UNEQUAL; break; + case LESS: out->op = GLITE_JP_QUERYOP_LESS; break; + case GREATER: out->op = GLITE_JP_QUERYOP_GREATER; break; + case WITHIN: + out->op = GLITE_JP_QUERYOP_WITHIN; + s2jp_queryval(in->value2,out->attr.type,&out->value2); + break; + } + + s2jp_queryval(in->value,out->attr.type,&out->value); +} + +SOAP_FMAC5 int SOAP_FMAC6 jpsrv__FeedIndex( + struct soap *soap, + char *destination, + struct jptype__Attributes *attributes, + struct jptype__PrimaryQuery *query, + enum xsd__boolean history, + enum xsd__boolean continuous, + struct jpsrv__FeedIndexResponse *response) +{ + +/* deferred processing: return feed_id to the index server first, + * start feeding it afterwards -- not before the index server actually + * knows feed_id and is ready to accept the feed. + * + * Has to be done within the same server slave, + * passed through the context */ + + CONTEXT_FROM_SOAP(soap,ctx); + char *feed_id = NULL; + time_t expires = 0; + int ret = SOAP_OK; + + glite_jp_attr_t *attrs = calloc(attributes->__sizeitem+1,sizeof *attrs); + glite_jp_query_rec_t *qry = calloc(query->__sizeitem+1,sizeof *qry); + int i; + + glite_jp_clear_error(ctx); + + for (i = 0; i__sizeitem; i++) s2jp_attr(attributes->item[i],attrs+i); + for (i = 0; i__sizeitem; i++) s2jp_query(query->item[i],qry+i); + + if (history) { + if (glite_jpps_run_feed(ctx,destination,attrs,qry,&feed_id)) { + err2fault(ctx,soap); + ret = SOAP_FAULT; + goto cleanup; + } + } + + if (continuous) { + if (glite_jpps_register_feed(ctx,destination,attrs,qry,&feed_id,&expires)) { + err2fault(ctx,soap); + ret = SOAP_FAULT; + goto cleanup; + } + } + + if (!history && !continuous) { + glite_jp_error_t err; + err.code = EINVAL; + err.source = __FUNCTION__; + err.desc = "at least one of and must be true"; + glite_jp_stack_error(ctx,&err); + err2fault(ctx,soap); + ret = SOAP_FAULT; + goto cleanup; + } + + response->expires = expires; + response->feedId = soap_strdup(soap,feed_id); + +cleanup: + free(feed_id); + for (i=0; attrs[i].type; i++) free(attrs[i].name); + free(attrs); + for (i=0; qry[i].attr.type; i++) glite_jp_free_query_rec(qry+i); + free(qry); + + return ret; +} + +SOAP_FMAC5 int SOAP_FMAC6 jpsrv__FeedIndexRefresh( + struct soap *soap, + char *feed_id, + struct jpsrv__FeedIndexRefreshResponse *response) +{ + fprintf(stderr,"%s: not implemented\n",__FUNCTION__); + abort(); +} + +SOAP_FMAC5 int SOAP_FMAC6 jpsrv__GetJob( + struct soap *soap, + char *job, + struct jpsrv__GetJobResponse *response) +{ + CONTEXT_FROM_SOAP(soap,ctx); + char *url; + + struct { + glite_jp_fileclass_t type; + char **url; + char *name; + } tab[] = { + { GLITE_JP_FILECLASS_INPUT, &response->inputSandbox, "input sandbox" }, + { GLITE_JP_FILECLASS_OUTPUT, &response->outputSandbox, "output sandbox" }, + { GLITE_JP_FILECLASS_LBLOG, &response->jobLog, "L&B log" }, + { GLITE_JP_FILECLASS_TAGS, &response->tags, "JP tags" }, + { GLITE_JP_FILECLASS_UNDEF, NULL, NULL } + }; + + int i; + glite_jp_error_t err; + + for (i=0; tab[i].type; i++) { + glite_jp_clear_error(ctx); + switch (glite_jppsbe_get_job_url(ctx,job,tab[i].type,&url)) { + case 0: *tab[i].url = soap_strdup(soap,url); + free(url); + break; + case ENOENT: + *tab[i].url = NULL; + break; + default: + err.code = ctx->error->code; + err.source = "jpsrv__GetJob()"; + err.desc = tab[i].name; + glite_jp_stack_error(ctx,&err); + err2fault(ctx,soap); + glite_jp_clear_error(ctx); + return SOAP_FAULT; + } + } + return SOAP_OK; +} + diff --git a/org.glite.jp.primary/src/tags.c b/org.glite.jp.primary/src/tags.c new file mode 100644 index 0000000..25474f4 --- /dev/null +++ b/org.glite.jp.primary/src/tags.c @@ -0,0 +1,62 @@ +#include +#include +#include +#include + +#include +#include "tags.h" +#include "backend.h" + +/* magic name_len value_len binary sequence timestamp */ +#define HEADER "JP#TAG# %05u %012lu %c %05u %012lu#" +#define HEADER_SIZE 48 + +int glite_jpps_tag_append( + glite_jp_context_t ctx, + void *handle, + const glite_jp_tagval_t *tag +) +{ + char hdr[HEADER_SIZE+1]; + glite_jp_error_t err; + + unsigned long vlen = tag->binary ? tag->size : + (tag->value ? strlen(tag->value) : 0); + int nlen; + + memset(&err,0,sizeof err); + err.source = "glite_jpps_tag_append()"; + + if (!tag->name) { + err.code = EINVAL; + err.desc = "tag name"; + return glite_jp_stack_error(ctx,&err); + } + + nlen = strlen(tag->name); + + assert(sprintf(hdr,HEADER,nlen,vlen, + tag->binary ? "B" : "S", + tag->sequence, tag->timestamp) == HEADER_SIZE); + + if (glite_jppsbe_append(ctx,handle,hdr,HEADER_SIZE)) { + err.code = EIO; + err.desc = "write tag header"; + return glite_jp_stack_error(ctx,&err); + } + + if (glite_jppsbe_append(ctx,handle,tag->name,nlen)) { + err.code = EIO; + err.desc = "write tag name"; + return glite_jp_stack_error(ctx,&err); + } + + if (glite_jppsbe_append(ctx,handle,tag->value,vlen)) { + err.code = EIO; + err.desc = "write tag value"; + return glite_jp_stack_error(ctx,&err); + } + + return 0; +} + diff --git a/org.glite.jp.primary/src/tags.h b/org.glite.jp.primary/src/tags.h new file mode 100644 index 0000000..0d8afa8 --- /dev/null +++ b/org.glite.jp.primary/src/tags.h @@ -0,0 +1 @@ +int glite_jpps_tag_append(glite_jp_context_t,void *,const glite_jp_tagval_t *); diff --git a/org.glite.jp.primary/src/typemap.dat b/org.glite.jp.primary/src/typemap.dat new file mode 100644 index 0000000..7032cb2 --- /dev/null +++ b/org.glite.jp.primary/src/typemap.dat @@ -0,0 +1,2 @@ +jpsrv = http://glite.org/wsdl/services/jp +jptype = http://glite.org/wsdl/types/jp diff --git a/org.glite.jp/project/JobProvenance.wsdl b/org.glite.jp/project/JobProvenance.wsdl new file mode 100644 index 0000000..26f97a4 --- /dev/null +++ b/org.glite.jp/project/JobProvenance.wsdl @@ -0,0 +1,531 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Register job with JP primary storage. +Job registration in LB is propagated to JP immediately so that JP is aware of the job, +despite no furhter information is available in it. + +Input: JobId + +Output: N/A + +Faults: GenericJPFault + + + + + + + Initiate upload of of sandbox/dump of job life log from LB. +WM component responsible for job sandbox management and LB server call JP to declare +intention to upload intput/output sandbox and job life log. + +Input: + +uclass: type of the upload - INPUT_SANDBOX, OUTPUT_SANDBOX, JOB_LOG + +commitTimeout: upper limit on time for which JP waits for committing this upload transaction + +contentType: MIME type of the uploaded file + +Output: + +destination: URL where the client should upload the file + +commitBefore: acutual time when the upload transaction times out + +Faults: GenericJPFault + + +Initiate upload of of sandbox/dump of job life log from LB. + + + + + + Confirm upload. +Should be called after a file upload initiaded with StartUpload is finished. + +Input: + +destination: Upload destination URL (to match with the original request) + +Output: + +Faults: GenericJPFault + + + + + + + Record a value of user tag. +JP tags are either standalone or override values of their LB counterparts. +However, JP tag values are still distinguishable those inherited from LB. +JP tags may be either strings or blobs. + +Input: + +jobid: + +tag: structure containing name, timestamp, optional sequence number to order tag values +without relying on timestamps, and string or blob value. + +Output: N/A + +Faults: GenericJPFault + + + + + + + Start feeding JP index server. +Called by the index server to start batch feed, and optionally also subscribe for incremental feed. + +JP index server subscribes with JP primary storage using a query +containing conditions on primary metadata and a list of queryable attributes +of the index server (i.e. data which should be sent to the index server). + +When a matching job record is created or modified within the primary storage +the job record data are sent to the subscribed index server. + +The subscription is soft-state, it expires after certain time unless refreshed by the client explicitely. + +In the batch mode the query has the same form +with additional flag asking for all matching records (i.e.\ not only +arriving afterwards). + +Input: + +destination: where to send the job record data + +attributes: which job record attributes should be sent to the requesting index server + +conditions: list of query conditions. Each conditions has the form Attribute Operator Value, +where Attribute is any of job record attributes and Operator is one of EQUAL, UNEQUAL, LESS, GREATER, WITHIN. + +continuous: flag determining that the query is incremental (not batch) + +Output: + +feedId: unique Id of the feed request, to be used in refresh, cancelation etc. + +expires: when the feed times out. Must be refreshed before this time. + +Faults: GenericJPFault + + + + + + + Extend batch feed subscription (used by index server) + +Input: feedId returned previously by FeedIndex + +Output: the same as for FeedIndex + +Faults: GenericJPFault + + + + + + + Retrieve job record URL's when jobid is known +Used either to bypass JP index server query for this specific case, or after the index server query to +retrieve actual job record. + +Input: jobid + +Output: + +jobLog, inputSandbox, outputSandbox, tags: URL's to components of the job record. + +Faults: GenericJPFault + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Job Provenance Primary Storage service + + + + + + + + + + + + + + + + + + + + + + Store or update information on jobs within the JP index server. +Called directly by the primary storage, used for both batch and incremental feed. + +Input: + +data: list of job record updates. Each contains jobid, list of JP attribute values and user tag values. + +feedDone: flag indicating end of batch feed. (In order to avoid potential problems with buffer allocation +the huge dataset of batch feed is split into reasonable chunks and delivered with more UpdateJobs calls.) + +Output: N/A + +Faults: GenericJPFault + + + + + + + + + Retrieve pointers to job records of jobs matching a query. +Input: conditions - list of lists of query conditions. + Elements of the inner lists refer to a single job attribute, the conditions are or-ed. + Elements of the outer list may refer to different job attributes, they are and-ed. + +Output: + +jobs: list of JobId, PSContact (URL of the primary storage which manges this job) pairs + +Faults: GenericJPFault + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Job Provenance Index service + + + + + + + diff --git a/org.glite.jp/project/JobProvenanceIS.wsdl b/org.glite.jp/project/JobProvenanceIS.wsdl new file mode 100644 index 0000000..2ef587d --- /dev/null +++ b/org.glite.jp/project/JobProvenanceIS.wsdl @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + Store or update information on jobs within the JP index server. +Called directly by the primary storage, used for both batch and incremental feed. + +Input: + +data: list of job record updates. Each contains jobid, list of JP attribute values and user tag values. + +feedDone: flag indicating end of batch feed. (In order to avoid potential problems with buffer allocation +the huge dataset of batch feed is split into reasonable chunks and delivered with more UpdateJobs calls.) + +Output: N/A + +Faults: GenericJPFault + + + + + + + + Retrieve pointers to job records of jobs matching a query. +Input: conditions - list of lists of query conditions. + Elements of the inner lists refer to a single job attribute, the conditions are or-ed. + Elements of the outer list may refer to different job attributes, they are and-ed. + +Output: + +jobs: list of JobId, PSContact (URL of the primary storage which manges this job) pairs + +Faults: GenericJPFault + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Job Provenance Index service + + + + + + + diff --git a/org.glite.jp/project/JobProvenancePS.wsdl b/org.glite.jp/project/JobProvenancePS.wsdl new file mode 100644 index 0000000..7c777dd --- /dev/null +++ b/org.glite.jp/project/JobProvenancePS.wsdl @@ -0,0 +1,307 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Register job with JP primary storage. +Job registration in LB is propagated to JP immediately so that JP is aware of the job, +despite no furhter information is available in it. + +Input: JobId + +Output: N/A + +Faults: GenericJPFault + + + + + + + Initiate upload of of sandbox/dump of job life log from LB. +WM component responsible for job sandbox management and LB server call JP to declare +intention to upload intput/output sandbox and job life log. + +Input: + +uclass: type of the upload - INPUT_SANDBOX, OUTPUT_SANDBOX, JOB_LOG + +commitTimeout: upper limit on time for which JP waits for committing this upload transaction + +contentType: MIME type of the uploaded file + +Output: + +destination: URL where the client should upload the file + +commitBefore: acutual time when the upload transaction times out + +Faults: GenericJPFault + + +Initiate upload of of sandbox/dump of job life log from LB. + + + + + + Confirm upload. +Should be called after a file upload initiaded with StartUpload is finished. + +Input: + +destination: Upload destination URL (to match with the original request) + +Output: + +Faults: GenericJPFault + + + + + + + Record a value of user tag. +JP tags are either standalone or override values of their LB counterparts. +However, JP tag values are still distinguishable those inherited from LB. +JP tags may be either strings or blobs. + +Input: + +jobid: + +tag: structure containing name, timestamp, optional sequence number to order tag values +without relying on timestamps, and string or blob value. + +Output: N/A + +Faults: GenericJPFault + + + + + + + Start feeding JP index server. +Called by the index server to start batch feed, and optionally also subscribe for incremental feed. + +JP index server subscribes with JP primary storage using a query +containing conditions on primary metadata and a list of queryable attributes +of the index server (i.e. data which should be sent to the index server). + +When a matching job record is created or modified within the primary storage +the job record data are sent to the subscribed index server. + +The subscription is soft-state, it expires after certain time unless refreshed by the client explicitely. + +In the batch mode the query has the same form +with additional flag asking for all matching records (i.e.\ not only +arriving afterwards). + +Input: + +destination: where to send the job record data + +attributes: which job record attributes should be sent to the requesting index server + +conditions: list of query conditions. Each conditions has the form Attribute Operator Value, +where Attribute is any of job record attributes and Operator is one of EQUAL, UNEQUAL, LESS, GREATER, WITHIN. + +continuous: flag determining that the query is incremental (not batch) + +Output: + +feedId: unique Id of the feed request, to be used in refresh, cancelation etc. + +expires: when the feed times out. Must be refreshed before this time. + +Faults: GenericJPFault + + + + + + + Extend batch feed subscription (used by index server) + +Input: feedId returned previously by FeedIndex + +Output: the same as for FeedIndex + +Faults: GenericJPFault + + + + + + + Retrieve job record URL's when jobid is known +Used either to bypass JP index server query for this specific case, or after the index server query to +retrieve actual job record. + +Input: jobid + +Output: + +jobLog, inputSandbox, outputSandbox, tags: URL's to components of the job record. + +Faults: GenericJPFault + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Job Provenance Primary Storage service + + + + + diff --git a/org.glite.jp/project/JobProvenanceTypes.wsdl b/org.glite.jp/project/JobProvenanceTypes.wsdl new file mode 100644 index 0000000..e1a6db4 --- /dev/null +++ b/org.glite.jp/project/JobProvenanceTypes.wsdl @@ -0,0 +1,162 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.glite.jp/project/JobProvenanceTypes.xsd b/org.glite.jp/project/JobProvenanceTypes.xsd new file mode 100644 index 0000000..743126f --- /dev/null +++ b/org.glite.jp/project/JobProvenanceTypes.xsd @@ -0,0 +1,146 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.glite.jp/project/properties.xml b/org.glite.jp/project/properties.xml new file mode 100755 index 0000000..eb879c4 --- /dev/null +++ b/org.glite.jp/project/properties.xml @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.glite.jp/project/taskdefs.xml b/org.glite.jp/project/taskdefs.xml new file mode 100755 index 0000000..0e0a223 --- /dev/null +++ b/org.glite.jp/project/taskdefs.xml @@ -0,0 +1,38 @@ + + + + + + + + \ No newline at end of file -- 1.8.2.3