From 0a00852d581a459e256effceb53e4316e2d83571 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Ale=C5=A1=20K=C5=99enek?= Date: Fri, 21 Oct 2005 12:54:03 +0000 Subject: [PATCH] - merged in 1.5 version of config/startup - database creation shell script - install and quick check documentation - sample LB job for checking --- org.glite.jp.primary/Makefile | 7 ++- .../config/glite-jp-primary-dbsetup.sh | 14 +++++ org.glite.jp.primary/config/startup | 52 ++++++++++++++----- org.glite.jp.primary/doc/README.install | 59 ++++++++++++++++++++++ org.glite.jp.primary/examples/sample_job | 8 +++ org.glite.jp.primary/project/version.properties | 2 +- 6 files changed, 126 insertions(+), 16 deletions(-) create mode 100644 org.glite.jp.primary/config/glite-jp-primary-dbsetup.sh create mode 100644 org.glite.jp.primary/doc/README.install create mode 100644 org.glite.jp.primary/examples/sample_job diff --git a/org.glite.jp.primary/Makefile b/org.glite.jp.primary/Makefile index dd6cdde..67b01cd 100644 --- a/org.glite.jp.primary/Makefile +++ b/org.glite.jp.primary/Makefile @@ -76,9 +76,9 @@ TRIOLIB:=-lglite_jp_trio ifneq (${mysql_prefix},/usr) ifeq ($(shell echo ${mysql_version} | cut -d. -f1,2),4.1) - MYSQLIB := -L${mysql_prefix}/lib/mysql -lmysqlclient + MYSQLIB := -L${mysql_prefix}/lib/mysql -lmysqlclient -lz else - MYSQLIB := -L${mysql_prefix}/lib -lmysqlclient + MYSQLIB := -L${mysql_prefix}/lib -lmysqlclient -lz endif else MYSQLIB := -lmysqlclient @@ -166,6 +166,9 @@ install: ${INSTALL} -m 755 $$plugin ${PREFIX}/lib; \ done ${INSTALL} -m 755 ${top_srcdir}/config/startup ${PREFIX}/etc/init.d/glite-jp-primary + ${INSTALL} -m 755 ${top_srcdir}/config/glite-jp-primary-dbsetup.sql ${PREFIX}/etc + ${INSTALL} -m 755 ${top_srcdir}/config/glite-jp-primary-dbsetup.sh ${PREFIX}/etc + ${INSTALL} -m 644 ${top_srcdir}/examples/sample_job ${PREFIX}/examples/glite-jp-primary-sample_job.lb clean: diff --git a/org.glite.jp.primary/config/glite-jp-primary-dbsetup.sh b/org.glite.jp.primary/config/glite-jp-primary-dbsetup.sh new file mode 100644 index 0000000..aed8e1a --- /dev/null +++ b/org.glite.jp.primary/config/glite-jp-primary-dbsetup.sh @@ -0,0 +1,14 @@ +#! /bin/sh + +# +# Shell example of preparing the database for JP Index Server +# + +# database +mysqladmin -u root -p create jpps + +# user +mysql -u root -p -e 'GRANT ALL on jpps.* to jpps@localhost' + +# tables +mysql -u jpps jpps < `dirname $0`/glite-jp-primary-dbsetup.sql diff --git a/org.glite.jp.primary/config/startup b/org.glite.jp.primary/config/startup index 6050d99..b3154ef 100644 --- a/org.glite.jp.primary/config/startup +++ b/org.glite.jp.primary/config/startup @@ -16,7 +16,7 @@ GLITE_LOCATION_VAR=${GLITE_LOCATION_VAR:-${GLITE_LOCATION}/var} unset creds -GLITE_JP_PRIMARY_PORT=8901 +test -n "$GLITE_JP_PRIMARY_PORT" || GLITE_JP_PRIMARY_PORT=8901 start() { @@ -30,10 +30,15 @@ start() echo FAILED return 1 fi + if test -z "$GLITE_JP_PRIMARY_PEERS" ;then + echo 'Error: incomplete configuration (GLITE_JP_PRIMARY_PEERS is not set)' + echo FAILED + return 1 + fi if test -z "$GLITE_JP_PRIMARY_FTP_PORT" -o \ -z "$GLITE_JP_PRIMARY_INTERNAL" -o -z "$GLITE_JP_PRIMARY_EXTERNAL" ;then echo 'Error: incomplete configuration (GLITE_JP_PRIMARY_FTP_PORT,' \ - 'GLITE_JP_PRIMARY_INTERNAL, or GLITE_JP_PRIMARY_EXTERNAL is not set' + 'GLITE_JP_PRIMARY_INTERNAL, or GLITE_JP_PRIMARY_EXTERNAL is not set)' echo FAILED return 1 fi @@ -63,16 +68,20 @@ start() echo -n Starting glite-jp-primarystoraged ... su - $GLITE_USER -c " $GLITE_LOCATION/bin/glite-jp-primarystoraged \ - $creds -i '$pidfile' + -P $GLITE_LOCATION/lib/glite-jp-tags.so -P $GLITE_LOCATION/lib/glite_lb_plugin.so \ + $creds -a '$GLITE_JP_PRIMARY_PEERS' \ + -i '$pidfile' -p $GLITE_JP_PRIMARY_PORT $GLITE_JP_PRIMARY_SPECIAL \ -BI,'$GLITE_JP_PRIMARY_INTERNAL' -BE,'$GLITE_JP_PRIMARY_EXTERNAL' \ ${GLITE_JP_PRIMARY_DBCS_OPTION} " && echo " done" || echo " FAILED" echo -n Starting JP gridftp server ... su - $GLITE_USER -c "X509_USER_CERT=\"$X509_USER_CERT\" \ X509_USER_KEY=\"$X509_USER_KEY\" \ + GLITE_USER=\"$GLITE_USER\" \ + FTPBE_INT_PREFIX=\"$GLITE_JP_PRIMARY_INTERNAL\" \ LD_PRELOAD=$GLITE_LOCATION/lib/glite-jp-ftpdauth.so \ - $GLOBUS_LOCATION/sbin/in.ftpd -a -S -p $GLITE_JP_PRIMARY_FTP_PORT & \ - pid=\$!; sleep 2; kill -0 \$pid" && echo " done" || echo " FAILED" + $GLOBUS_LOCATION/sbin/in.ftpd -a -s -p $GLITE_JP_PRIMARY_FTP_PORT & \ + pid=\$!; echo \$pid > $pidfile.ftpd; sleep 2; true kill -0 \$pid" && echo " done" || echo " FAILED" } stop() @@ -96,31 +105,48 @@ stop() echo "$pidfile" does not exist - glite-jp-primarystoraged not running? >&2 return 1 fi -} + if [ -f "$pidfile.ftpd" ]; then + pid=`cat "$pidfile.ftpd"` + kill $pid + echo -n Stopping JP ftp server \($pid\) ... + try=0 + while ps p $pid >/dev/null 2>&1; do + sleep 1; + try=`expr $try + 1` + if [ $try = 20 ]; then + echo " giving up after $try retries" + return 1 + fi + done + echo " done" + rm -f "$pidfile.ftpd" + else + echo "$pidfile.ftpd" does not exist - JP ftp server not running? >&2 + return 1 + fi - # XXX - echo >&2 'stopping JP gridftp server not implemented yet' - return 1 + return 0 } status() { - # XXX pidfile + retval=0 + # XXX pidfile? if netstat -an --inet | grep "^tcp .* 0.0.0.0:${GLITE_JP_PRIMARY_PORT} .*LISTEN" >/dev/null 2>&1 ;then echo glite-jp-primarystoraged running else echo glite-jp-primarystoraged not running - return 1 + retval=1 fi if netstat -an --inet | grep "^tcp .* 0.0.0.0:${GLITE_JP_PRIMARY_FTP_PORT} .*LISTEN" >/dev/null 2>&1 ;then echo JP gridftp server running else echo JP gridftp server not running - return 1 + retval=1 fi - return 0 + return $retval } case x$1 in diff --git a/org.glite.jp.primary/doc/README.install b/org.glite.jp.primary/doc/README.install new file mode 100644 index 0000000..17d0d27 --- /dev/null +++ b/org.glite.jp.primary/doc/README.install @@ -0,0 +1,59 @@ +0. install glite-jp-primary package + +1. create MySQL database of the service, preferably using + ${GLITE_LOCATION}/etc/glite-jp-primary-dbsetup.sh + (the script prompts two times for MySQL root password) + +2. create a directory where JPPS files will be stored and configure + gridftp server so that this directory is accessible via gridftp + +3. create the configuration file $GLITE_LOCATION/etc/jp.conf, it should + contain /bin/sh syntax assignment to the following variables (or they + have to be set in the environment of the startup script by other means: + + GLITE_USER the unix user running the service + + GLOBUS_LOCATION where is Globus installed + + GLITE_JP_PRIMARY_FTP_PORT + port for the ftp JPPS interface (bulk file transfers) + + GLITE_JP_PRIMARY_INTERNAL + directory where the JPPS files are stored (local path) + + GLITE_JP_PRIMARY_EXTERNAL + URL prefix which is mapped by gridftp to the directory, + eg. gsiftp://this.host.name:$GLITE_JP_PRIMARY_FTP_PORT/some/directory + + + GLITE_JP_PRIMARY_PEERS + file with list (one per line) of X509 certificate subjects of "trusted peers", i.e. L&B servers which may upload data to this JPPS + + +4. start the service with + $GLITE_LOCATION/etc/init.d/glite-jp-primary start + + +5. This step taints the database with testing data. Don't do it if it is an issue. +It is not necessary for the service operation, it's just checking its basic functionality. + +$ JOB=https://scientific.civ.zcu.cz:9200/1Yjk3Wlaspl07y45iX8Tng +$ TEST=$GLITE_LOCATION/examples/glite-jp-primary-test +$ export X5O9_USER_PROXY=/some/proxy.pem # must be among $GLITE_JP_PRIMARY_PEERS + +$ $TEST registerjob $JOB '/the/job/owner/x509/cert' + +$ $TEST startupload "urn:org.glite.jp.primary:lb" 1 text/plain + responds with destination gsiftp url + +$ globus-url-copy file:///$GLITE_LOCATION/examples/glite-jp-primary-sample_job.lb $DESTINATION_URL + +$ $TEST commitupload $DESTINATION_URL + +$ $TEST getjobattr $JOB http://egee.cesnet.cz/en/Schema/LB/Attributes:finalStatus + +The command should output: + + Waiting FILE Mon Oct 10 23:48:47 2005 + +-- diff --git a/org.glite.jp.primary/examples/sample_job b/org.glite.jp.primary/examples/sample_job new file mode 100644 index 0000000..f9b515f --- /dev/null +++ b/org.glite.jp.primary/examples/sample_job @@ -0,0 +1,8 @@ +DG.ARRIVED=20051010204845.000000 DATE=20051010204845.409455 HOST="scientific.civ.zcu.cz" PROG=edg-wms LVL=SYSTEM DG.PRIORITY=0 DG.SOURCE="UserInterface" DG.SRC_INSTANCE="" DG.EVNT="RegJob" DG.JOBID="https://scientific.civ.zcu.cz:9200/1Yjk3Wlaspl07y45iX8Tng" DG.SEQCODE="UI=000001:NS=0000000000:WM=000000:BH=0000000000:JSS=000000:LM=000000:LRMS=000000:APP=000000" DG.USER="/O=CESNET/O=Masaryk University/CN=Ales Krenek" DG.REGJOB.JDL="blabla" DG.REGJOB.NS="NNNSSSS" DG.REGJOB.PARENT="" DG.REGJOB.JOBTYPE="SIMPLE" DG.REGJOB.NSUBJOBS="0" DG.REGJOB.SEED="uLU0BArrdV98O41PLThJ5Q" +DG.ARRIVED=20051010204847.000000 DATE=20051010204847.451986 HOST="scientific.civ.zcu.cz" PROG=edg-wms LVL=SYSTEM DG.PRIORITY=0 DG.SOURCE="WorkloadManager" DG.SRC_INSTANCE="" DG.EVNT="DeQueued" DG.JOBID="https://scientific.civ.zcu.cz:9200/1Yjk3Wlaspl07y45iX8Tng" DG.SEQCODE="UI=000002:NS=0000000004:WM=000001:BH=0000000000:JSS=000000:LM=000000:LRMS=000000:APP=000000" DG.USER="/O=CESNET/O=Masaryk University/CN=Ales Krenek" DG.DEQUEUED.QUEUE="queue name" DG.DEQUEUED.LOCAL_JOBID="new jobId assigned by the receiving component" +DG.ARRIVED=20051010204847.000000 DATE=20051010204847.485702 HOST="scientific.civ.zcu.cz" PROG=edg-wms LVL=SYSTEM DG.PRIORITY=0 DG.SOURCE="WorkloadManager" DG.SRC_INSTANCE="" DG.EVNT="HelperCall" DG.JOBID="https://scientific.civ.zcu.cz:9200/1Yjk3Wlaspl07y45iX8Tng" DG.SEQCODE="UI=000002:NS=0000000004:WM=000003:BH=0000000000:JSS=000000:LM=000000:LRMS=000000:APP=000000" DG.USER="/O=CESNET/O=Masaryk University/CN=Ales Krenek" DG.HELPERCALL.HELPER_NAME="name of the called component" DG.HELPERCALL.HELPER_PARAMS="parameters of the call" DG.HELPERCALL.SRC_ROLE="CALLING" +DG.ARRIVED=20051010204847.000000 DATE=20051010204847.520663 HOST="scientific.civ.zcu.cz" PROG=edg-wms LVL=SYSTEM DG.PRIORITY=0 DG.SOURCE="WorkloadManager" DG.SRC_INSTANCE="" DG.EVNT="Match" DG.JOBID="https://scientific.civ.zcu.cz:9200/1Yjk3Wlaspl07y45iX8Tng" DG.SEQCODE="UI=000002:NS=0000000004:WM=000005:BH=0000000000:JSS=000000:LM=000000:LRMS=000000:APP=000000" DG.USER="/O=CESNET/O=Masaryk University/CN=Ales Krenek" DG.MATCH.DEST_ID="destination CE/queue" +DG.ARRIVED=20051010204847.000000 DATE=20051010204847.552015 HOST="scientific.civ.zcu.cz" PROG=edg-wms LVL=SYSTEM DG.PRIORITY=0 DG.SOURCE="WorkloadManager" DG.SRC_INSTANCE="" DG.EVNT="HelperReturn" DG.JOBID="https://scientific.civ.zcu.cz:9200/1Yjk3Wlaspl07y45iX8Tng" DG.SEQCODE="UI=000002:NS=0000000004:WM=000007:BH=0000000000:JSS=000000:LM=000000:LRMS=000000:APP=000000" DG.USER="/O=CESNET/O=Masaryk University/CN=Ales Krenek" DG.HELPERRETURN.HELPER_NAME="name of the called component" DG.HELPERRETURN.RETVAL="returned data" DG.HELPERRETURN.SRC_ROLE="CALLING" +DG.ARRIVED=20051010204847.000000 DATE=20051010204847.584267 HOST="scientific.civ.zcu.cz" PROG=edg-wms LVL=SYSTEM DG.PRIORITY=0 DG.SOURCE="WorkloadManager" DG.SRC_INSTANCE="" DG.EVNT="EnQueued" DG.JOBID="https://scientific.civ.zcu.cz:9200/1Yjk3Wlaspl07y45iX8Tng" DG.SEQCODE="UI=000002:NS=0000000004:WM=000009:BH=0000000000:JSS=000000:LM=000000:LRMS=000000:APP=000000" DG.USER="/O=CESNET/O=Masaryk University/CN=Ales Krenek" DG.ENQUEUED.QUEUE="destination queue" DG.ENQUEUED.JOB="job description in receiver language" DG.ENQUEUED.RESULT="OK" DG.ENQUEUED.REASON="detailed description of transfer" +DG.ARRIVED=20051010204847.000000 DATE=20051010204847.381506 HOST="scientific.civ.zcu.cz" PROG=edg-wms LVL=SYSTEM DG.PRIORITY=0 DG.SOURCE="NetworkServer" DG.SRC_INSTANCE="" DG.EVNT="Accepted" DG.JOBID="https://scientific.civ.zcu.cz:9200/1Yjk3Wlaspl07y45iX8Tng" DG.SEQCODE="UI=000002:NS=0000000001:WM=000000:BH=0000000000:JSS=000000:LM=000000:LRMS=000000:APP=000000" DG.USER="/O=CESNET/O=Masaryk University/CN=Ales Krenek" DG.ACCEPTED.FROM="UserInterface" DG.ACCEPTED.FROM_HOST="sending component hostname" DG.ACCEPTED.FROM_INSTANCE="sending component instance" DG.ACCEPTED.LOCAL_JOBID="new jobId (Condor Globus ...)" +DG.ARRIVED=20051010204847.000000 DATE=20051010204847.409422 HOST="scientific.civ.zcu.cz" PROG=edg-wms LVL=SYSTEM DG.PRIORITY=0 DG.SOURCE="NetworkServer" DG.SRC_INSTANCE="" DG.EVNT="EnQueued" DG.JOBID="https://scientific.civ.zcu.cz:9200/1Yjk3Wlaspl07y45iX8Tng" DG.SEQCODE="UI=000002:NS=0000000003:WM=000000:BH=0000000000:JSS=000000:LM=000000:LRMS=000000:APP=000000" DG.USER="/O=CESNET/O=Masaryk University/CN=Ales Krenek" DG.ENQUEUED.QUEUE="destination queue" DG.ENQUEUED.JOB="job description in receiver language" DG.ENQUEUED.RESULT="OK" DG.ENQUEUED.REASON="detailed description of transfer" diff --git a/org.glite.jp.primary/project/version.properties b/org.glite.jp.primary/project/version.properties index 1ef4b00..3964bdd 100644 --- a/org.glite.jp.primary/project/version.properties +++ b/org.glite.jp.primary/project/version.properties @@ -1,2 +1,2 @@ -module.version=1.1.0 +module.version=1.1.1 module.age=1 -- 1.8.2.3