updated README.LB-statistics

diff --git a/org.glite.lb.utils/doc/README.LB-statistics b/org.glite.lb.utils/doc/README.LB-statistics
index 9cf4fa0..5025459 100644 (file)
--- a/org.glite.lb.utils/doc/README.LB-statistics
+++ b/org.glite.lb.utils/doc/README.LB-statistics
@@ -3,3 +3,39 @@ LB statistics tools
 
 GLITE-LB-STATISTICS
 -------------------
+
+General idea:
+
+- LB server produces dump files (during each purge on regular basis)
+see LB server startup script; option -D / --dump-prefix of glite-lb-bkserverd
+
+- these dumps are exported for the purposes of JP also on regular basis,
+see LB/JP deplyment module; option -s/ --store of glite-lb-lb_dump_exporter 
+
+- it depends on the LB server policy if dumps in this directory are used for
+the statistics purposes or all files are hardlinked for example to a different
+directory
+
+- general idea is such that data are available for statistics server that downloads
+and removes dumps after download! Dump files are then processed on the statistics
+server.
+
+
+What needs to be done on the LB server:
+
+- bkserver + lb_dump_exporter running
+
+- gridftp running (allowing statistics server to download and remove files from 
+a given directory
+
+
+What needs to be done on the statistics server:
+
+- download and remove files from the LB server
+see glite-lb-statistics-gsi.sh (shell script in the examples directory)
+
+- process dump files using the glite-lb-statistics tool
+see glite-lb-statistics.sh (shell script in the examples directory)
+
+all scripts are supposed to be run from a crontab.
+

diff --git a/org.glite.lb.utils/examples/glite-lb-statistics-gsi.sh b/org.glite.lb.utils/examples/glite-lb-statistics-gsi.sh
new file mode 100755 (executable)
index 0000000..18c2a9d
--- /dev/null
+++ b/org.glite.lb.utils/examples/glite-lb-statistics-gsi.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+#
+# an example script for getting LB dumps by gsiftp
+# and removing them at by glite-gridftp-rm
+#
+
+GLITE_LOCATION=${GLITE_LOCATION:-/opt/glite}
+GLITE_LOCATION_VAR=${GLITE_LOCATION_VAR:-${GLITE_LOCATION}/var}
+GLITE_LB_DUMPDIR=${GLITE_LB_DUMPDIR:-${GLITE_LOCATION_VAR}/dump}
+
+GSIFTP_SERVER=scientific.civ.zcu.cz
+GSIFTP_PORT=8911
+GLITE_LB_SERVER=${GSIFTP_SERVER}:${GSIFTP_PORT}
+GLITE_LB_EXPORT_DUMPDIR=/home/glite/LB/export
+
+X509_USER_KEY=~/.cert/usercert.pem
+X509_USER_CERT=~/.cert/userkey.pem
+
+progname=`basename $0`
+
+function syslog () {
+        if [ ! -z "$*" ]; then
+                echo `date +'%b %d %H:%M:%S'` `hostname` $progname: $*
+        fi
+}
+
+if [ ! -d ${GLITE_LB_DUMPDIR} ]; then
+        syslog "Creating directory ${GLITE_LB_DUMPDIR}"
+        mkdir -vp ${GLITE_LB_DUMPDIR} || exit 1
+fi
+
+syslog "getting new dumps to ${GLITE_LB_DUMPDIR}"
+globus-url-copy gsiftp://${GLITE_LB_SERVER}/${GLITE_LB_EXPORT_DUMPDIR}/ file://${GLITE_LB_DUMPDIR}/
+
+syslog "NOT removing dumps on the server"
+# glite-gridftp-rm ...
+
+syslog "done"

diff --git a/org.glite.lb.utils/examples/glite-lb-statistics-rsync.sh b/org.glite.lb.utils/examples/glite-lb-statistics-rsync.sh
new file mode 100755 (executable)
index 0000000..232f828
--- /dev/null
+++ b/org.glite.lb.utils/examples/glite-lb-statistics-rsync.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+#
+# an example script for getting LB dumps by rsync
+# (it does not remove files on the LB server!)
+#
+
+GLITE_LOCATION=${GLITE_LOCATION:-/opt/glite}
+GLITE_LOCATION_VAR=${GLITE_LOCATION_VAR:-${GLITE_LOCATION}/var}
+GLITE_LB_DUMPDIR=${GLITE_LB_DUMPDIR:-${GLITE_LOCATION_VAR}/dump}
+
+GLITE_LB_SERVER=scientific.civ.zcu.cz
+GLITE_LB_EXPORT_DUMPDIR=/home/glite/LB/export
+
+progname=`basename $0`
+
+function syslog () {
+        if [ ! -z "$*" ]; then
+                echo `date +'%b %d %H:%M:%S'` `hostname` $progname: $*
+        fi
+}
+
+if [ ! -d ${GLITE_LB_DUMPDIR} ]; then
+        syslog "Creating directory ${GLITE_LB_DUMPDIR}"
+        mkdir -vp ${GLITE_LB_DUMPDIR} || exit 1
+fi
+
+syslog "getting new dumps to ${GLITE_LB_DUMPDIR}"
+rsync -rlptoDuv ${GLITE_LB_SERVER}:${GLITE_LB_EXPORT_DUMPDIR}/ ${GLITE_LB_DUMPDIR}/
+
+syslog "done"

diff --git a/org.glite.lb.utils/examples/glite-lb-statistics-sftp.sh b/org.glite.lb.utils/examples/glite-lb-statistics-sftp.sh
new file mode 100755 (executable)
index 0000000..04f468c
--- /dev/null
+++ b/org.glite.lb.utils/examples/glite-lb-statistics-sftp.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+#
+# an example script for getting LB dumps using sftp
+# (it does not remove files on the LB server!)
+#
+
+GLITE_LOCATION=${GLITE_LOCATION:-/opt/glite}
+GLITE_LOCATION_VAR=${GLITE_LOCATION_VAR:-${GLITE_LOCATION}/var}
+GLITE_LB_DUMPDIR=${GLITE_LB_DUMPDIR:-${GLITE_LOCATION_VAR}/dump}
+
+GLITE_LB_SFTPBATCH=/tmp/glite-lb-statistics-sftp-batch
+
+GLITE_LB_SERVER=scientific.civ.zcu.cz
+GLITE_LB_EXPORT_DUMPDIR=/home/glite/LB/export
+
+progname=`basename $0`
+
+function syslog () {
+        if [ ! -z "$*" ]; then
+                echo `date +'%b %d %H:%M:%S'` `hostname` $progname: $*
+        fi
+}
+
+if [ ! -d ${GLITE_LB_DUMPDIR} ]; then
+        syslog "Creating directory ${GLITE_LB_DUMPDIR}"
+        mkdir -vp ${GLITE_LB_DUMPDIR} || exit 1
+fi
+
+if [ -f ${GLITE_LB_SFTPBATCH} ]; then
+        syslog "WARNING: File ${GLITE_LB_SFTPBATCH} already exists, will be overwritten"
+fi
+
+syslog "writing ${GLITE_LB_SFTPBATCH}"
+echo "lcd ${GLITE_LB_DUMPDIR}
+cd ${GLITE_LB_EXPORT_DUMPDIR}
+get -P *" > ${GLITE_LB_SFTPBATCH} || exit 1
+
+syslog "getting new dumps to ${GLITE_LB_DUMPDIR}"
+sftp -b ${GLITE_LB_SFTPBATCH} ${GLITE_LB_SERVER}
+
+syslog "done"

diff --git a/org.glite.lb.utils/examples/glite-lb-statistics.sh b/org.glite.lb.utils/examples/glite-lb-statistics.sh
new file mode 100755 (executable)
index 0000000..388e9ae
--- /dev/null
+++ b/org.glite.lb.utils/examples/glite-lb-statistics.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+
+#
+# an example script for processing LB dumps for the statistics purposes
+# suitable for running from crontab
+#
+
+GLITE_LOCATION=${GLITE_LOCATION:-/opt/glite}
+GLITE_LOCATION_VAR=${GLITE_LOCATION_VAR:-${GLITE_LOCATION}/var}
+GLITE_LB_DUMPDIR=${GLITE_LB_DUMPDIR:-${GLITE_LOCATION_VAR}/dump}
+GLITE_LB_STATISTICS=${GLITE_LOCATION}/bin/glite-lb-statistics
+
+progname=`basename $0`
+
+function syslog () {
+       if [ ! -z "$*" ]; then
+               echo `date +'%b %d %H:%M:%S'` `hostname` $progname: $*
+       fi
+}
+
+if [ ! -d ${GLITE_LB_DUMPDIR} ]; then
+       syslog "Creating directory ${GLITE_LB_DUMPDIR}"
+       mkdir -vp ${GLITE_LB_DUMPDIR} || exit 1
+fi   
+
+if [ ! -f ${GLITE_LB_STATISTICS} ]; then
+       syslog "Program ${GLITE_LB_STATISTICS} is missing"
+       exit 1
+fi
+
+syslog "processing new dumps in ${GLITE_LB_DUMPDIR}"
+for file in ${GLITE_LB_DUMPDIR}/*[^xml,log] ; do
+       if [ ! -s $file.xml ]; then
+               if [ -s $file ]; then
+                       ${GLITE_LB_STATISTICS} -v -f $file > $file.xml 2> $file.log
+                       syslog "processed $file"
+                       let num++
+               else
+                       syslog `rm -v -f $file*`        
+               fi
+#      else
+#              syslog "file $file.xml already exists"
+       fi
+done
+
+if [ -z $num ]; then
+       syslog "processed no files"
+else
+       syslog "processed $num files"
+fi