From: František Dvořák Date: Fri, 28 Mar 2008 17:31:10 +0000 (+0000) Subject: Move the jobs mill from /tmp to /var/glite. X-Git-Tag: merge_313_4~3 X-Git-Url: http://scientific.zcu.cz/git/?a=commitdiff_plain;h=5a780f73d6aab26c24f7717f76b4ba6bcaa9fe4d;p=jra1mw.git Move the jobs mill from /tmp to /var/glite. Refactor all-in-one example mill script - using glite-lb-exporter.sh. --- diff --git a/org.glite.jp.client/config/startup b/org.glite.jp.client/config/startup index b38020d..7b843bb 100755 --- a/org.glite.jp.client/config/startup +++ b/org.glite.jp.client/config/startup @@ -2,7 +2,6 @@ GLITE_LOCATION=${GLITE_LOCATION:-/opt/glite} GLITE_LOCATION_VAR=${GLITE_LOCATION_VAR:-/var/glite} -GLITE_TMP=${GLITE_TMP:-/tmp} [ -f /etc/glite.conf ] && . /etc/glite.conf [ -f $GLITE_LOCATION/etc/glite-wms.conf ] && . $GLITE_LOCATION/etc/glite-wms.conf @@ -37,10 +36,10 @@ start() [ -z "$creds" ] && echo $0: WARNING: No credentials specified. Using default lookup which is dangerous. >&2 - [ -z "$GLITE_LB_EXPORT_JPREG_MAILDIR" ] && GLITE_LB_EXPORT_JPREG_MAILDIR=$GLITE_TMP/jpreg + [ -z "$GLITE_LB_EXPORT_JPREG_MAILDIR" ] && GLITE_LB_EXPORT_JPREG_MAILDIR=$GLITE_LOCATION_VAR/jpreg jpreg_maildir="--reg-mdir $GLITE_LB_EXPORT_JPREG_MAILDIR " [ -d "$GLITE_LB_EXPORT_JPREG_MAILDIR" ] || mkdir "$GLITE_LB_EXPORT_JPREG_MAILDIR" && chown $GLITE_USER:$GLITE_GROUP "$GLITE_LB_EXPORT_JPREG_MAILDIR" - [ -z "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" ] && GLITE_LB_EXPORT_JPDUMP_MAILDIR=$GLITE_TMP/jpdump + [ -z "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" ] && GLITE_LB_EXPORT_JPDUMP_MAILDIR=$GLITE_LOCATION_VAR/jpdump jpdump_maildir="--dump-mdir $GLITE_LB_EXPORT_JPDUMP_MAILDIR " [ -d "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" ] || mkdir "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" && chown $GLITE_USER:$GLITE_GROUP "$GLITE_LB_EXPORT_JPDUMP_MAILDIR" [ -n "$GLITE_LB_EXPORT_JPPS" ] && jpps="--jpps $GLITE_LB_EXPORT_JPPS " diff --git a/org.glite.jp.client/examples/glite-jp-importer.sh b/org.glite.jp.client/examples/glite-jp-importer.sh index eb40277..fbf509c 100644 --- a/org.glite.jp.client/examples/glite-jp-importer.sh +++ b/org.glite.jp.client/examples/glite-jp-importer.sh @@ -3,6 +3,8 @@ # # all-in-one example script for purging LB and importing the dumps to JP # +# uses helper purging script glite-lb-export.sh from glite.lb.client +# GLITE_LOCATION=${GLITE_LOCATION:-/opt/glite} GLITE_LOCATION_VAR=${GLITE_LOCATION_VAR:-${GLITE_LOCATION}/var} @@ -15,53 +17,35 @@ GLITE_LOCATION_VAR=${GLITE_LOCATION_VAR:-${GLITE_LOCATION}/var} [ -f $HOME/.glite.conf ] && . $HOME/.glite.conf +# get default values for purge and export +PREFIX=${GLITE_LOCATION:-`dirname $0`/..} +GLITE_LB_EXPORT_ENABLED="false" GLITE_LB_PURGE_ENABLED="false" . $PREFIX/bin/glite-lb-export.sh # job provenance server if [ -z "$GLITE_LB_EXPORT_JPPS" ]; then echo "Please specify the Job Provanance Primary Storage server." exit 1 fi -# book keeping server -GLITE_LB_SERVER_PORT=${GLITE_LB_SERVER_PORT:-9000} -GLITE_LB_EXPORT_BKSERVER=${GLITE_LB_EXPORT_BKSERVER:-localhost:$GLITE_LB_SERVER_PORT} # certificates if [ -z "$X509_USER_CERT" -o -z "$X509_USER_KEY" ]; then echo "Please set X509_USER_CERT and X509_USER_KEY." exit 1 fi -# dump directory of bkserver -if [ -z "$GLITE_LB_EXPORT_DUMPDIR" ]; then - GLITE_LB_EXPORT_DUMPDIR=/tmp/dump - echo "GLITE_LB_EXPORT_DUMPDIR not specified (-D argument of the bkserver), used $GLITE_LB_EXPORT_DUMPDIR" -fi -if [ -z "$GLITE_LB_EXPORT_PURGEDIR" ]; then - GLITE_LB_EXPORT_PURGEDIR=/tmp/purge - echo "GLITE_LB_EXPORT_PURGEDIR not specified (-S argument of the bkserver), used $GLITE_LB_EXPORT_PURGEDIR" -fi # LB maildir for job registration if [ -z "$GLITE_LB_EXPORT_JPREG_MAILDIR" ]; then - GLITE_LB_EXPORT_JPREG_MAILDIR=/tmp/jpreg + GLITE_LB_EXPORT_JPREG_MAILDIR=$GLITE_LOCATION_VAR/jpreg echo "GLITE_LB_EXPORT_JPREG_MAILDIR not specified (-J arguent of the bkserver), used $GLITE_LB_EXPORT_JPREG_MAILDIR" fi +if [ -n "$GLITE_LB_EXPORT_SANDBOX_MAILDIR" ]; then + sandbox_maildir="--sandbox-mdir $GLITE_LB_EXPORT_SANDBOX_MAILDIR " +fi # pidfile [ -n "$GLITE_JP_IMPORTER_PIDFILE" ] && pidfile="-i $GLITE_JP_IMPORTER_PIDFILE " CERT_ARGS="-c $X509_USER_CERT -k $X509_USER_KEY" -GLITE_LB_EXPORT_JPDUMP_MAILDIR=${GLITE_LB_EXPORT_JPDUMP_MAILDIR:-/tmp/jpdump} -GLITE_LB_EXPORT_SANDBOX_MAILDIR=${GLITE_LB_EXPORT_SANDBOX_MAILDIR:-/tmp/jpsandbox} -GLITE_LB_EXPORT_JOBSDIR=${GLITE_LB_EXPORT_JOBSDIR:-/tmp/lbexport} -PREFIX=${PREFIX:-`dirname $0`/..} LOGDIR=$GLITE_LOCATION_VAR GLITE_LB_EXPORT_PURGE_ARGS=${GLITE_LB_EXPORT_PURGE_ARGS:---cleared 2d --aborted 2w --cancelled 2w --other 2m} - -[ -d $GLITE_LB_EXPORT_JPDUMP_MAILDIR ] || mkdir -p $GLITE_LB_EXPORT_JPDUMP_MAILDIR -[ -d $GLITE_LB_EXPORT_DUMPDIR ] || mkdir -p $GLITE_LB_EXPORT_DUMPDIR - -[ -d $GLITE_LB_EXPORT_SANDBOX_MAILDIR ] || mkdir -p $GLITE_LB_EXPORT_SANDBOX_MAILDIR - -[ -n "$GLITE_LB_EXPORT_DUMPDIR_KEEP" -a ! -d $GLITE_LB_EXPORT_DUMPDIR_KEEP ] && mkdir -p $GLITE_LB_EXPORT_DUMPDIR_KEEP -[ -d $GLITE_LB_EXPORT_JOBSDIR ] || mkdir -p $GLITE_LB_EXPORT_JOBSDIR if [ -n "$GLITE_LB_EXPORT_JOBSDIR_KEEP" ]; then keep_jobs="--store ${GLITE_LB_EXPORT_JOBSDIR_KEEP} " [ -d $GLITE_LB_EXPORT_JOBSDIR_KEEP ] || mkdir -p $GLITE_LB_EXPORT_JOBSDIR_KEEP @@ -71,27 +55,13 @@ fi echo "Using cert args $CERT_ARGS" -$PREFIX/bin/glite-jp-importer --reg-mdir $GLITE_LB_EXPORT_JPREG_MAILDIR --dump-mdir $GLITE_LB_EXPORT_JPDUMP_MAILDIR $CERT_ARGS --sandbox-mdir $GLITE_LB_EXPORT_SANDBOX_MAILDIR -g --jpps $GLITE_LB_EXPORT_JPPS $pidfile$keep_jobs$GLITE_JP_IMPORTER_ARGS > $LOGDIR/jp-importer.log 2>&1 & +$PREFIX/bin/glite-jp-importer --reg-mdir $GLITE_LB_EXPORT_JPREG_MAILDIR --dump-mdir $GLITE_LB_EXPORT_JPDUMP_MAILDIR $CERT_ARGS ${sandbox_maildir}-g --jpps $GLITE_LB_EXPORT_JPPS $pidfile$keep_jobs$GLITE_JP_IMPORTER_ARGS > $LOGDIR/jp-importer.log 2>&1 & JP_PID=$! trap "kill $JP_PID; exit 0" SIGINT while [ 1 ]; do - $PREFIX/bin/glite-lb-purge $GLITE_LB_EXPORT_PURGE_ARGS -l -m $GLITE_LB_EXPORT_BKSERVER -s - - list=`ls $GLITE_LB_EXPORT_PURGEDIR/* 2>/dev/null` - for file in $list; do - if [ -s $file ]; then - $PREFIX/bin/glite-lb-dump_exporter -d $file -s $GLITE_LB_EXPORT_JOBSDIR -m $GLITE_LB_EXPORT_JPDUMP_MAILDIR - if [ -n "$GLITE_LB_EXPORT_DUMPDIR_KEEP" ]; then - mv $file $GLITE_LB_EXPORT_DUMPDIR_KEEP - else - rm -f $file - fi - else - rm -f $file - fi - done + GLITE_LB_EXPORT_ENABLED="true" GLITE_LB_PURGE_ENABLED="true" $PREFIX/bin/glite-lb-export.sh sleep 30 done diff --git a/org.glite.jp.client/interface/jpimporter.h b/org.glite.jp.client/interface/jpimporter.h index 019014f..3e293b7 100644 --- a/org.glite.jp.client/interface/jpimporter.h +++ b/org.glite.jp.client/interface/jpimporter.h @@ -2,15 +2,15 @@ #define __GLITE_JPIMPORTER__ #ifndef GLITE_REG_IMPORTER_MDIR -#define GLITE_REG_IMPORTER_MDIR "/tmp/jpreg" +#define GLITE_REG_IMPORTER_MDIR "/var/glite/jpreg" #endif #ifndef GLITE_DUMP_IMPORTER_MDIR -#define GLITE_DUMP_IMPORTER_MDIR "/tmp/jpdump" +#define GLITE_DUMP_IMPORTER_MDIR "/var/glite/jpdump" #endif #ifndef GLITE_SANDBOX_IMPORTER_MDIR -#define GLITE_SANDBOX_IMPORTER_MDIR "/tmp/jpsandbox" +#define GLITE_SANDBOX_IMPORTER_MDIR "/var/glite/jpsandbox" #endif #define PERF_JOBID_START_PREFIX "https://start.megajob/START-"