Final cleanup LB/JP interaction:

  - JOBSDIR required
  - added JOBSDIR_KEEP for future using
  - purging on localhost on configured port (so removed bkserver config)
  - removing used dumps and job files as default
  - glite-lb-export.sh to /sbin
  - removed LB/JP part from LB deployment patch

diff --git a/org.glite.jp.client/config/startup b/org.glite.jp.client/config/startup
index a17dd4d..0c2c9a0 100755 (executable)
--- a/org.glite.jp.client/config/startup
+++ b/org.glite.jp.client/config/startup
@@ -41,10 +41,14 @@ start()
        jpdump_maildir="--dump-mdir $GLITE_LB_EXPORT_JPDUMP_MAILDIR "
        [ -d "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" ] || mkdir "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" && chown $GLITE_USER:$GLITE_GROUP "$GLITE_LB_EXPORT_JPDUMP_MAILDIR"
        [ -n "$GLITE_LB_EXPORT_JPPS" ] && jpps="--jpps $GLITE_LB_EXPORT_JPPS "
+       if [ -n "$GLITE_LB_EXPORT_JOBSDIR_KEEP" ]; then
+               keep_jobs="--store ${GLITE_LB_EXPORT_JOBSDIR_KEEP} "
+               [ -d $GLITE_LB_EXPORT_JOBSDIR_KEEP ] || mkdir -p $GLITE_LB_EXPORT_JOBSDIR_KEEP
+       fi
 
        echo -n Starting glite-jp-importer ...
        su - $GLITE_USER -c "$GLITE_LOCATION/bin/glite-jp-importer \
-               -i $pidfile $jpreg_maildir $jpdump_maildir $jpps $creds" \
+               -i $pidfile $jpreg_maildir $jpdump_maildir $jpps $keep_jobs $creds" \
        && echo " done" || echo " FAILED"
 }
 

diff --git a/org.glite.jp.client/examples/glite-jp-importer.sh b/org.glite.jp.client/examples/glite-jp-importer.sh
index 804447e..483caea 100644 (file)
--- a/org.glite.jp.client/examples/glite-jp-importer.sh
+++ b/org.glite.jp.client/examples/glite-jp-importer.sh
@@ -21,11 +21,9 @@ if [ -z "$GLITE_LB_EXPORT_JPPS" ]; then
   echo "Please specify the Job Provanance Primary Storage server."
   exit 1
 fi
-# bookkeeping server
-if [ -z "$GLITE_LB_EXPORT_BKSERVER" ]; then
-  echo "Please specify the Book Keeping server."
-  exit 1
-fi
+# book keeping server
+GLITE_LB_SERVER_PORT=${GLITE_LB_SERVER_PORT:-9000}
+GLITE_LB_EXPORT_BKSERVER=${GLITE_LB_EXPORT_BKSERVER:-localhost:$GLITE_LB_SERVER_PORT}
 # certificates
 if [ -z "$X509_USER_CERT" -o -z "$X509_USER_KEY" ]; then
   echo "Please set X509_USER_CERT and X509_USER_KEY."
@@ -41,25 +39,30 @@ if [ -z "$GLITE_LB_EXPORT_JPREG_MAILDIR" ]; then
   GLITE_LB_EXPORT_JPREG_MAILDIR=/tmp/jpreg
   echo "GLITE_LB_EXPORT_JPREG_MAILDIR not specified (-J arguent of the bkserver), used $GLITE_LB_EXPORT_JPREG_MAILDIR"
 fi
+# pidfile
+[ -n "$GLITE_JP_IMPORTER_PIDFILE" ] && pidfile="-i $GLITE_JP_IMPORTER_PIDFILE "
 
 CERT_ARGS="-c $X509_USER_CERT -k $X509_USER_KEY"
 GLITE_LB_EXPORT_JPDUMP_MAILDIR=${GLITE_LB_EXPORT_JPDUMP_MAILDIR:-/tmp/jpdump}
-GLITE_LB_EXPORT_DUMPDIR_OLD=${GLITE_LB_EXPORT_DUMPDIR_OLD:-$GLITE_LB_EXPORT_DUMPDIR.old}
-GLITE_LB_EXPORT_EXPORTDIR=${GLITE_LB_EXPORT_EXPORTDIR:-/tmp/lbexport}
+GLITE_LB_EXPORT_JOBSDIR=${GLITE_LB_EXPORT_JOBSDIR:-/tmp/lbexport}
 PREFIX=${PREFIX:-`dirname $0`/..}
 LOGDIR=$GLITE_LOCATION_VAR
-GLITE_LB_EXPORT_PURGE_ARGS=${GLITE_LB_EXPORT_PURGE_ARGS:--a 1h -c 1h -n 1h -o 1d}
+GLITE_LB_EXPORT_PURGE_ARGS=${GLITE_LB_EXPORT_PURGE_ARGS:---aborted 1d --cleared 1d --cancelled 1d --other 7d}
 
 
 [ -d $GLITE_LB_EXPORT_JPDUMP_MAILDIR ] || mkdir -p $GLITE_LB_EXPORT_JPDUMP_MAILDIR
 [ -d $GLITE_LB_EXPORT_DUMPDIR ] || mkdir -p $GLITE_LB_EXPORT_DUMPDIR
-[ -d $GLITE_LB_EXPORT_DUMPDIR_OLD ] || mkdir -p $GLITE_LB_EXPORT_DUMPDIR_OLD
-[ -d $GLITE_LB_EXPORT_EXPORTDIR ] || mkdir -p $GLITE_LB_EXPORT_EXPORTDIR
+[ -n "$GLITE_LB_EXPORT_DUMPDIR_KEEP" -a ! -d $GLITE_LB_EXPORT_DUMPDIR_KEEP ] && mkdir -p $GLITE_LB_EXPORT_DUMPDIR_KEEP
+[ -d $GLITE_LB_EXPORT_JOBSDIR ] || mkdir -p $GLITE_LB_EXPORT_JOBSDIR
+if [ -n "$GLITE_LB_EXPORT_JOBSDIR_KEEP" ]; then
+  keep_jobs="--store ${GLITE_LB_EXPORT_JOBSDIR_KEEP} "
+  [ -d $GLITE_LB_EXPORT_JOBSDIR_KEEP ] || mkdir -p $GLITE_LB_EXPORT_JOBSDIR_KEEP
+fi
 [ -d $LOGDIR ] || mkdir -p $LOGDIR
 
 echo "Using cert args $CERT_ARGS"
 
-$PREFIX/bin/glite-jp-importer -r $GLITE_LB_EXPORT_JPREG_MAILDIR -d $GLITE_LB_EXPORT_JPDUMP_MAILDIR $CERT_ARGS -g -p $GLITE_LB_EXPORT_JPPS > $LOGDIR/jp-importer.log 2>&1 &
+$PREFIX/bin/glite-jp-importer --reg-mdir $GLITE_LB_EXPORT_JPREG_MAILDIR --dump-mdir $GLITE_LB_EXPORT_JPDUMP_MAILDIR $CERT_ARGS -g --jpps $GLITE_LB_EXPORT_JPPS $pidfile$keep_jobs> $LOGDIR/jp-importer.log 2>&1 &
 JP_PID=$!
 trap "kill $JP_PID; exit 0" SIGINT
 
@@ -68,8 +71,12 @@ while [ 1 ]; do
 
   for file in $GLITE_LB_EXPORT_DUMPDIR/*; do
     if [ -s $file ]; then
-      $PREFIX/sbin/glite-lb-lb_dump_exporter -d $file -s $GLITE_LB_EXPORT_EXPORTDIR -m $GLITE_LB_EXPORT_JPDUMP_MAILDIR
-      mv $file $GLITE_LB_EXPORT_DUMPDIR_OLD
+      $PREFIX/sbin/glite-lb-lb_dump_exporter -d $file -s $GLITE_LB_EXPORT_JOBSDIR -m $GLITE_LB_EXPORT_JPDUMP_MAILDIR
+      if [ -n "$GLITE_LB_EXPORT_DUMPDIR_KEEP" ]; then
+        mv $file $GLITE_LB_EXPORT_DUMPDIR_KEEP
+      else
+        rm $file
+      fi
     else
       rm $file
     fi

diff --git a/org.glite.jp.client/src/jpimporter.c b/org.glite.jp.client/src/jpimporter.c
index 7a30ef8..4b0ea4d 100644 (file)
--- a/org.glite.jp.client/src/jpimporter.c
+++ b/org.glite.jp.client/src/jpimporter.c
@@ -12,6 +12,7 @@
 #include <errno.h>
 #include <syslog.h>
 #include <fcntl.h>
+#include <libgen.h>
 
 #include "glite/lb/lb_maildir.h"
 #include "glite/security/glite_gsplugin.h"
@@ -66,6 +67,7 @@ static char                      *name;
 static char                       *jpps = GLITE_JPPS;
 static char                            reg_mdir[PATH_MAX] = GLITE_REG_IMPORTER_MDIR;
 static char                            dump_mdir[PATH_MAX] = GLITE_DUMP_IMPORTER_MDIR;
+static char                            *store = NULL;
 static struct soap        *soap;
 
 static time_t                  cert_mtime;
@@ -87,6 +89,7 @@ static struct option opts[] = {
        { "dump-mdir",   1, NULL,    'd'},
        { "pidfile",     1, NULL,    'i'},
        { "poll",        1, NULL,    't'},
+       { "store",       1, NULL,    's'},
        { NULL,          0, NULL,     0}
 };
 
@@ -105,6 +108,7 @@ static void usage(char *me)
                "\t-d, --dump-mdir    path to the 'LB maildir' subtree for LB dumps\n"
                "\t-i, --pidfile      file to store master pid\n"
                "\t-t, --poll         maildir polling interval (in seconds)\n",
+               "\t-s, --store        keep uploaded jobs in this directory\n",
                me);
 }
 
@@ -157,6 +161,7 @@ int main(int argc, char *argv[])
                case 'C': cadir = optarg; break;
                case 'p': jpps = optarg; break;
                case 't': poll = atoi(optarg); break;
+               case 's': store = optarg; break;
                case 'r': strcpy(reg_mdir, optarg); break;
                case 'd': strcpy(dump_mdir, optarg); break;
                case 'i': strcpy(pidfile, optarg); break;
@@ -187,6 +192,12 @@ int main(int argc, char *argv[])
                
        edg_wll_MaildirInit(reg_mdir);
        edg_wll_MaildirInit(dump_mdir);
+       if (store && *store) {
+               if (mkdir(store, 0750) != 0 && errno != EEXIST) {
+                       fprintf(stderr, "Can't create directory %s: %s\n", store, strerror(errno));
+                       store = NULL;
+               }
+       }
 
        if ( !debug ) {
                if ( daemon(1,0) == -1 ) { perror("deamon()"); exit(1); }
@@ -401,7 +412,9 @@ static int dump_importer(void)
        static int              readnew = 1;
        char               *msg = NULL,
                                   *fname = NULL,
-                                  *aux;
+                                  *aux,
+                                  *bname;
+       char                        fspec[PATH_MAX];
        int                             ret;
        int                             fhnd;
        msg_pattern_t   tab[] = {
@@ -469,6 +482,20 @@ static int dump_importer(void)
                ret = soap_call___jpsrv__CommitUpload(soap, tab[_jpps].val?:jpps, "", &cu_in, &empty);
                if ( (ret = check_soap_fault(soap, ret)) ) break;
                dprintf(("[%s] Dump upload succesfull\n", name));
+               if (store && *store) {
+                       bname = strdup(tab[_file].val);
+                       snprintf(fspec, sizeof fspec, "%s/%s", store, basename(bname));
+                       free(bname);
+                       if (rename(tab[_file].val, fspec) != 0) 
+                               fprintf(stderr, "moving %s to %s failed: %s\n", tab[_file].val, fspec, strerror(errno));
+                       else
+                               dprintf(("[%s] moving %s to %s OK\n", name, tab[_file].val, fspec));
+               } else {
+                       if (unlink(tab[_file].val) != 0)
+                               fprintf(stderr, "removing %s failed: %s\n", tab[_file].val, strerror(errno));
+                       else
+                               dprintf(("[%s] %s removed\n", name, tab[_file].val));
+               }
        } while (0);
 
        edg_wll_MaildirTransEnd(dump_mdir, fname, ret? LBMD_TRANS_FAILED_RETRY: LBMD_TRANS_OK);