From: František Dvořák Date: Tue, 4 Apr 2006 07:35:39 +0000 (+0000) Subject: Final cleanup LB/JP interaction: X-Git-Tag: glite-jp-client_R_1_1_1~2 X-Git-Url: http://scientific.zcu.cz/git/?a=commitdiff_plain;h=ffbdb9302b46e7e1f64def9c2555f090d553951d;p=jra1mw.git Final cleanup LB/JP interaction: - JOBSDIR required - added JOBSDIR_KEEP for future using - purging on localhost on configured port (so removed bkserver config) - removing used dumps and job files as default - glite-lb-export.sh to /sbin - removed LB/JP part from LB deployment patch --- diff --git a/org.glite.jp.client/config/startup b/org.glite.jp.client/config/startup index a17dd4d..0c2c9a0 100755 --- a/org.glite.jp.client/config/startup +++ b/org.glite.jp.client/config/startup @@ -41,10 +41,14 @@ start() jpdump_maildir="--dump-mdir $GLITE_LB_EXPORT_JPDUMP_MAILDIR " [ -d "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" ] || mkdir "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" && chown $GLITE_USER:$GLITE_GROUP "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" [ -n "$GLITE_LB_EXPORT_JPPS" ] && jpps="--jpps $GLITE_LB_EXPORT_JPPS " + if [ -n "$GLITE_LB_EXPORT_JOBSDIR_KEEP" ]; then + keep_jobs="--store ${GLITE_LB_EXPORT_JOBSDIR_KEEP} " + [ -d $GLITE_LB_EXPORT_JOBSDIR_KEEP ] || mkdir -p $GLITE_LB_EXPORT_JOBSDIR_KEEP + fi echo -n Starting glite-jp-importer ... su - $GLITE_USER -c "$GLITE_LOCATION/bin/glite-jp-importer \ - -i $pidfile $jpreg_maildir $jpdump_maildir $jpps $creds" \ + -i $pidfile $jpreg_maildir $jpdump_maildir $jpps $keep_jobs $creds" \ && echo " done" || echo " FAILED" } diff --git a/org.glite.jp.client/examples/glite-jp-importer.sh b/org.glite.jp.client/examples/glite-jp-importer.sh index 804447e..483caea 100644 --- a/org.glite.jp.client/examples/glite-jp-importer.sh +++ b/org.glite.jp.client/examples/glite-jp-importer.sh @@ -21,11 +21,9 @@ if [ -z "$GLITE_LB_EXPORT_JPPS" ]; then echo "Please specify the Job Provanance Primary Storage server." exit 1 fi -# bookkeeping server -if [ -z "$GLITE_LB_EXPORT_BKSERVER" ]; then - echo "Please specify the Book Keeping server." - exit 1 -fi +# book keeping server +GLITE_LB_SERVER_PORT=${GLITE_LB_SERVER_PORT:-9000} +GLITE_LB_EXPORT_BKSERVER=${GLITE_LB_EXPORT_BKSERVER:-localhost:$GLITE_LB_SERVER_PORT} # certificates if [ -z "$X509_USER_CERT" -o -z "$X509_USER_KEY" ]; then echo "Please set X509_USER_CERT and X509_USER_KEY." @@ -41,25 +39,30 @@ if [ -z "$GLITE_LB_EXPORT_JPREG_MAILDIR" ]; then GLITE_LB_EXPORT_JPREG_MAILDIR=/tmp/jpreg echo "GLITE_LB_EXPORT_JPREG_MAILDIR not specified (-J arguent of the bkserver), used $GLITE_LB_EXPORT_JPREG_MAILDIR" fi +# pidfile +[ -n "$GLITE_JP_IMPORTER_PIDFILE" ] && pidfile="-i $GLITE_JP_IMPORTER_PIDFILE " CERT_ARGS="-c $X509_USER_CERT -k $X509_USER_KEY" GLITE_LB_EXPORT_JPDUMP_MAILDIR=${GLITE_LB_EXPORT_JPDUMP_MAILDIR:-/tmp/jpdump} -GLITE_LB_EXPORT_DUMPDIR_OLD=${GLITE_LB_EXPORT_DUMPDIR_OLD:-$GLITE_LB_EXPORT_DUMPDIR.old} -GLITE_LB_EXPORT_EXPORTDIR=${GLITE_LB_EXPORT_EXPORTDIR:-/tmp/lbexport} +GLITE_LB_EXPORT_JOBSDIR=${GLITE_LB_EXPORT_JOBSDIR:-/tmp/lbexport} PREFIX=${PREFIX:-`dirname $0`/..} LOGDIR=$GLITE_LOCATION_VAR -GLITE_LB_EXPORT_PURGE_ARGS=${GLITE_LB_EXPORT_PURGE_ARGS:--a 1h -c 1h -n 1h -o 1d} +GLITE_LB_EXPORT_PURGE_ARGS=${GLITE_LB_EXPORT_PURGE_ARGS:---aborted 1d --cleared 1d --cancelled 1d --other 7d} [ -d $GLITE_LB_EXPORT_JPDUMP_MAILDIR ] || mkdir -p $GLITE_LB_EXPORT_JPDUMP_MAILDIR [ -d $GLITE_LB_EXPORT_DUMPDIR ] || mkdir -p $GLITE_LB_EXPORT_DUMPDIR -[ -d $GLITE_LB_EXPORT_DUMPDIR_OLD ] || mkdir -p $GLITE_LB_EXPORT_DUMPDIR_OLD -[ -d $GLITE_LB_EXPORT_EXPORTDIR ] || mkdir -p $GLITE_LB_EXPORT_EXPORTDIR +[ -n "$GLITE_LB_EXPORT_DUMPDIR_KEEP" -a ! -d $GLITE_LB_EXPORT_DUMPDIR_KEEP ] && mkdir -p $GLITE_LB_EXPORT_DUMPDIR_KEEP +[ -d $GLITE_LB_EXPORT_JOBSDIR ] || mkdir -p $GLITE_LB_EXPORT_JOBSDIR +if [ -n "$GLITE_LB_EXPORT_JOBSDIR_KEEP" ]; then + keep_jobs="--store ${GLITE_LB_EXPORT_JOBSDIR_KEEP} " + [ -d $GLITE_LB_EXPORT_JOBSDIR_KEEP ] || mkdir -p $GLITE_LB_EXPORT_JOBSDIR_KEEP +fi [ -d $LOGDIR ] || mkdir -p $LOGDIR echo "Using cert args $CERT_ARGS" -$PREFIX/bin/glite-jp-importer -r $GLITE_LB_EXPORT_JPREG_MAILDIR -d $GLITE_LB_EXPORT_JPDUMP_MAILDIR $CERT_ARGS -g -p $GLITE_LB_EXPORT_JPPS > $LOGDIR/jp-importer.log 2>&1 & +$PREFIX/bin/glite-jp-importer --reg-mdir $GLITE_LB_EXPORT_JPREG_MAILDIR --dump-mdir $GLITE_LB_EXPORT_JPDUMP_MAILDIR $CERT_ARGS -g --jpps $GLITE_LB_EXPORT_JPPS $pidfile$keep_jobs> $LOGDIR/jp-importer.log 2>&1 & JP_PID=$! trap "kill $JP_PID; exit 0" SIGINT @@ -68,8 +71,12 @@ while [ 1 ]; do for file in $GLITE_LB_EXPORT_DUMPDIR/*; do if [ -s $file ]; then - $PREFIX/sbin/glite-lb-lb_dump_exporter -d $file -s $GLITE_LB_EXPORT_EXPORTDIR -m $GLITE_LB_EXPORT_JPDUMP_MAILDIR - mv $file $GLITE_LB_EXPORT_DUMPDIR_OLD + $PREFIX/sbin/glite-lb-lb_dump_exporter -d $file -s $GLITE_LB_EXPORT_JOBSDIR -m $GLITE_LB_EXPORT_JPDUMP_MAILDIR + if [ -n "$GLITE_LB_EXPORT_DUMPDIR_KEEP" ]; then + mv $file $GLITE_LB_EXPORT_DUMPDIR_KEEP + else + rm $file + fi else rm $file fi diff --git a/org.glite.jp.client/src/jpimporter.c b/org.glite.jp.client/src/jpimporter.c index 7a30ef8..4b0ea4d 100644 --- a/org.glite.jp.client/src/jpimporter.c +++ b/org.glite.jp.client/src/jpimporter.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "glite/lb/lb_maildir.h" #include "glite/security/glite_gsplugin.h" @@ -66,6 +67,7 @@ static char *name; static char *jpps = GLITE_JPPS; static char reg_mdir[PATH_MAX] = GLITE_REG_IMPORTER_MDIR; static char dump_mdir[PATH_MAX] = GLITE_DUMP_IMPORTER_MDIR; +static char *store = NULL; static struct soap *soap; static time_t cert_mtime; @@ -87,6 +89,7 @@ static struct option opts[] = { { "dump-mdir", 1, NULL, 'd'}, { "pidfile", 1, NULL, 'i'}, { "poll", 1, NULL, 't'}, + { "store", 1, NULL, 's'}, { NULL, 0, NULL, 0} }; @@ -105,6 +108,7 @@ static void usage(char *me) "\t-d, --dump-mdir path to the 'LB maildir' subtree for LB dumps\n" "\t-i, --pidfile file to store master pid\n" "\t-t, --poll maildir polling interval (in seconds)\n", + "\t-s, --store keep uploaded jobs in this directory\n", me); } @@ -157,6 +161,7 @@ int main(int argc, char *argv[]) case 'C': cadir = optarg; break; case 'p': jpps = optarg; break; case 't': poll = atoi(optarg); break; + case 's': store = optarg; break; case 'r': strcpy(reg_mdir, optarg); break; case 'd': strcpy(dump_mdir, optarg); break; case 'i': strcpy(pidfile, optarg); break; @@ -187,6 +192,12 @@ int main(int argc, char *argv[]) edg_wll_MaildirInit(reg_mdir); edg_wll_MaildirInit(dump_mdir); + if (store && *store) { + if (mkdir(store, 0750) != 0 && errno != EEXIST) { + fprintf(stderr, "Can't create directory %s: %s\n", store, strerror(errno)); + store = NULL; + } + } if ( !debug ) { if ( daemon(1,0) == -1 ) { perror("deamon()"); exit(1); } @@ -401,7 +412,9 @@ static int dump_importer(void) static int readnew = 1; char *msg = NULL, *fname = NULL, - *aux; + *aux, + *bname; + char fspec[PATH_MAX]; int ret; int fhnd; msg_pattern_t tab[] = { @@ -469,6 +482,20 @@ static int dump_importer(void) ret = soap_call___jpsrv__CommitUpload(soap, tab[_jpps].val?:jpps, "", &cu_in, &empty); if ( (ret = check_soap_fault(soap, ret)) ) break; dprintf(("[%s] Dump upload succesfull\n", name)); + if (store && *store) { + bname = strdup(tab[_file].val); + snprintf(fspec, sizeof fspec, "%s/%s", store, basename(bname)); + free(bname); + if (rename(tab[_file].val, fspec) != 0) + fprintf(stderr, "moving %s to %s failed: %s\n", tab[_file].val, fspec, strerror(errno)); + else + dprintf(("[%s] moving %s to %s OK\n", name, tab[_file].val, fspec)); + } else { + if (unlink(tab[_file].val) != 0) + fprintf(stderr, "removing %s failed: %s\n", tab[_file].val, strerror(errno)); + else + dprintf(("[%s] %s removed\n", name, tab[_file].val)); + } } while (0); edg_wll_MaildirTransEnd(dump_mdir, fname, ret? LBMD_TRANS_FAILED_RETRY: LBMD_TRANS_OK);