From: Jan Pospíšil Date: Mon, 4 Sep 2006 19:36:38 +0000 (+0000) Subject: updated README.LB-statistics X-Git-Tag: gridsite-core_R_1_3_2~2 X-Git-Url: http://scientific.zcu.cz/git/?a=commitdiff_plain;h=4f6f75b1bbd3efde82433a71780607335a42fe8c;p=jra1mw.git updated README.LB-statistics --- diff --git a/org.glite.lb.utils/doc/README.LB-statistics b/org.glite.lb.utils/doc/README.LB-statistics index 9cf4fa0..5025459 100644 --- a/org.glite.lb.utils/doc/README.LB-statistics +++ b/org.glite.lb.utils/doc/README.LB-statistics @@ -3,3 +3,39 @@ LB statistics tools GLITE-LB-STATISTICS ------------------- + +General idea: + +- LB server produces dump files (during each purge on regular basis) +see LB server startup script; option -D / --dump-prefix of glite-lb-bkserverd + +- these dumps are exported for the purposes of JP also on regular basis, +see LB/JP deplyment module; option -s/ --store of glite-lb-lb_dump_exporter + +- it depends on the LB server policy if dumps in this directory are used for +the statistics purposes or all files are hardlinked for example to a different +directory + +- general idea is such that data are available for statistics server that downloads +and removes dumps after download! Dump files are then processed on the statistics +server. + + +What needs to be done on the LB server: + +- bkserver + lb_dump_exporter running + +- gridftp running (allowing statistics server to download and remove files from +a given directory + + +What needs to be done on the statistics server: + +- download and remove files from the LB server +see glite-lb-statistics-gsi.sh (shell script in the examples directory) + +- process dump files using the glite-lb-statistics tool +see glite-lb-statistics.sh (shell script in the examples directory) + +all scripts are supposed to be run from a crontab. + diff --git a/org.glite.lb.utils/examples/glite-lb-statistics-gsi.sh b/org.glite.lb.utils/examples/glite-lb-statistics-gsi.sh new file mode 100755 index 0000000..18c2a9d --- /dev/null +++ b/org.glite.lb.utils/examples/glite-lb-statistics-gsi.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +# +# an example script for getting LB dumps by gsiftp +# and removing them at by glite-gridftp-rm +# + +GLITE_LOCATION=${GLITE_LOCATION:-/opt/glite} +GLITE_LOCATION_VAR=${GLITE_LOCATION_VAR:-${GLITE_LOCATION}/var} +GLITE_LB_DUMPDIR=${GLITE_LB_DUMPDIR:-${GLITE_LOCATION_VAR}/dump} + +GSIFTP_SERVER=scientific.civ.zcu.cz +GSIFTP_PORT=8911 +GLITE_LB_SERVER=${GSIFTP_SERVER}:${GSIFTP_PORT} +GLITE_LB_EXPORT_DUMPDIR=/home/glite/LB/export + +X509_USER_KEY=~/.cert/usercert.pem +X509_USER_CERT=~/.cert/userkey.pem + +progname=`basename $0` + +function syslog () { + if [ ! -z "$*" ]; then + echo `date +'%b %d %H:%M:%S'` `hostname` $progname: $* + fi +} + +if [ ! -d ${GLITE_LB_DUMPDIR} ]; then + syslog "Creating directory ${GLITE_LB_DUMPDIR}" + mkdir -vp ${GLITE_LB_DUMPDIR} || exit 1 +fi + +syslog "getting new dumps to ${GLITE_LB_DUMPDIR}" +globus-url-copy gsiftp://${GLITE_LB_SERVER}/${GLITE_LB_EXPORT_DUMPDIR}/ file://${GLITE_LB_DUMPDIR}/ + +syslog "NOT removing dumps on the server" +# glite-gridftp-rm ... + +syslog "done" diff --git a/org.glite.lb.utils/examples/glite-lb-statistics-rsync.sh b/org.glite.lb.utils/examples/glite-lb-statistics-rsync.sh new file mode 100755 index 0000000..232f828 --- /dev/null +++ b/org.glite.lb.utils/examples/glite-lb-statistics-rsync.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +# +# an example script for getting LB dumps by rsync +# (it does not remove files on the LB server!) +# + +GLITE_LOCATION=${GLITE_LOCATION:-/opt/glite} +GLITE_LOCATION_VAR=${GLITE_LOCATION_VAR:-${GLITE_LOCATION}/var} +GLITE_LB_DUMPDIR=${GLITE_LB_DUMPDIR:-${GLITE_LOCATION_VAR}/dump} + +GLITE_LB_SERVER=scientific.civ.zcu.cz +GLITE_LB_EXPORT_DUMPDIR=/home/glite/LB/export + +progname=`basename $0` + +function syslog () { + if [ ! -z "$*" ]; then + echo `date +'%b %d %H:%M:%S'` `hostname` $progname: $* + fi +} + +if [ ! -d ${GLITE_LB_DUMPDIR} ]; then + syslog "Creating directory ${GLITE_LB_DUMPDIR}" + mkdir -vp ${GLITE_LB_DUMPDIR} || exit 1 +fi + +syslog "getting new dumps to ${GLITE_LB_DUMPDIR}" +rsync -rlptoDuv ${GLITE_LB_SERVER}:${GLITE_LB_EXPORT_DUMPDIR}/ ${GLITE_LB_DUMPDIR}/ + +syslog "done" diff --git a/org.glite.lb.utils/examples/glite-lb-statistics-sftp.sh b/org.glite.lb.utils/examples/glite-lb-statistics-sftp.sh new file mode 100755 index 0000000..04f468c --- /dev/null +++ b/org.glite.lb.utils/examples/glite-lb-statistics-sftp.sh @@ -0,0 +1,42 @@ +#!/bin/sh + +# +# an example script for getting LB dumps using sftp +# (it does not remove files on the LB server!) +# + +GLITE_LOCATION=${GLITE_LOCATION:-/opt/glite} +GLITE_LOCATION_VAR=${GLITE_LOCATION_VAR:-${GLITE_LOCATION}/var} +GLITE_LB_DUMPDIR=${GLITE_LB_DUMPDIR:-${GLITE_LOCATION_VAR}/dump} + +GLITE_LB_SFTPBATCH=/tmp/glite-lb-statistics-sftp-batch + +GLITE_LB_SERVER=scientific.civ.zcu.cz +GLITE_LB_EXPORT_DUMPDIR=/home/glite/LB/export + +progname=`basename $0` + +function syslog () { + if [ ! -z "$*" ]; then + echo `date +'%b %d %H:%M:%S'` `hostname` $progname: $* + fi +} + +if [ ! -d ${GLITE_LB_DUMPDIR} ]; then + syslog "Creating directory ${GLITE_LB_DUMPDIR}" + mkdir -vp ${GLITE_LB_DUMPDIR} || exit 1 +fi + +if [ -f ${GLITE_LB_SFTPBATCH} ]; then + syslog "WARNING: File ${GLITE_LB_SFTPBATCH} already exists, will be overwritten" +fi + +syslog "writing ${GLITE_LB_SFTPBATCH}" +echo "lcd ${GLITE_LB_DUMPDIR} +cd ${GLITE_LB_EXPORT_DUMPDIR} +get -P *" > ${GLITE_LB_SFTPBATCH} || exit 1 + +syslog "getting new dumps to ${GLITE_LB_DUMPDIR}" +sftp -b ${GLITE_LB_SFTPBATCH} ${GLITE_LB_SERVER} + +syslog "done" diff --git a/org.glite.lb.utils/examples/glite-lb-statistics.sh b/org.glite.lb.utils/examples/glite-lb-statistics.sh new file mode 100755 index 0000000..388e9ae --- /dev/null +++ b/org.glite.lb.utils/examples/glite-lb-statistics.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +# +# an example script for processing LB dumps for the statistics purposes +# suitable for running from crontab +# + +GLITE_LOCATION=${GLITE_LOCATION:-/opt/glite} +GLITE_LOCATION_VAR=${GLITE_LOCATION_VAR:-${GLITE_LOCATION}/var} +GLITE_LB_DUMPDIR=${GLITE_LB_DUMPDIR:-${GLITE_LOCATION_VAR}/dump} +GLITE_LB_STATISTICS=${GLITE_LOCATION}/bin/glite-lb-statistics + +progname=`basename $0` + +function syslog () { + if [ ! -z "$*" ]; then + echo `date +'%b %d %H:%M:%S'` `hostname` $progname: $* + fi +} + +if [ ! -d ${GLITE_LB_DUMPDIR} ]; then + syslog "Creating directory ${GLITE_LB_DUMPDIR}" + mkdir -vp ${GLITE_LB_DUMPDIR} || exit 1 +fi + +if [ ! -f ${GLITE_LB_STATISTICS} ]; then + syslog "Program ${GLITE_LB_STATISTICS} is missing" + exit 1 +fi + +syslog "processing new dumps in ${GLITE_LB_DUMPDIR}" +for file in ${GLITE_LB_DUMPDIR}/*[^xml,log] ; do + if [ ! -s $file.xml ]; then + if [ -s $file ]; then + ${GLITE_LB_STATISTICS} -v -f $file > $file.xml 2> $file.log + syslog "processed $file" + let num++ + else + syslog `rm -v -f $file*` + fi +# else +# syslog "file $file.xml already exists" + fi +done + +if [ -z $num ]; then + syslog "processed no files" +else + syslog "processed $num files" +fi