Finalize jobs statistics and the laste cleanups.

diff --git a/templates/accounting/jobs.py.erb b/files/accounting/jobs.py
similarity index 59%
rename from templates/accounting/jobs.py.erb
rename to files/accounting/jobs.py
index d383310..bd9deb6 100755 (executable)
--- a/templates/accounting/jobs.py.erb
+++ b/files/accounting/jobs.py
@@ -6,6 +6,31 @@ from io import BytesIO
 import time
 import calendar
 import datetime
+import sys
+import getopt
+import socket
+
+base_url = "http://" + socket.getfqdn() + ":19888"
+begin_rel = 24 * 3600
+end_rel = 0
+
+try:
+       opts, args = getopt.getopt(sys.argv[1:], "hm:b:e:", ["help", "mapred-url=", "begin=", "end="])
+except getopt.GetoptError:
+       print 'Args error'
+       sys.exit(2)
+for opt, arg in opts:
+       if opt in ('-h', '--help'):
+               print('jobs.py [-h|--help] [-m|--mapred-url URL] [-b|--begin] [-e|--end]')
+       elif opt in ('-m', '--mapred-url'):
+               base_url = arg
+       elif opt in ('-b', '--begin'):
+               begin_rel = int(arg)
+       elif opt in ('-e', '--end'):
+               end_rel = int(arg)
+       else:
+               print 'Args error'
+               sys.exit(2)
 
 # epoch time of local date
 #now = datetime.date.today().strftime('%s')
@@ -15,9 +40,9 @@ now0 = datetime.date.today()
 now = calendar.timegm(datetime.datetime(now0.year, now0.month, now0.day, 0, 0).timetuple())
 print '# ' + str(now0)
 
-begin = 1000 * (now - 24 * 3600)
-end = 1000 * now
-url = "<%= @_mapred_url -%>/ws/v1/history/mapreduce/jobs?finishedTimeBegin=" + str(begin) + "&finishedTimeEnd=" + str(end)
+begin = now - begin_rel
+end = now - end_rel
+url = base_url + "/ws/v1/history/mapreduce/jobs?finishedTimeBegin=" + str(1000 * begin) + "&finishedTimeEnd=" + str(1000 * end)
 print '# ' + url
 
 b = BytesIO()
@@ -74,6 +99,8 @@ if j["jobs"]:
 
 #              print '#[progress]', username, users[username].total, user.completed, user.wait, user.time
 
-print "INSERT INTO measure (name) VALUES ('jobs')"
+sql_begin = datetime.datetime.fromtimestamp(begin).strftime('%Y-%m-%d %H:%M:%S')
+sql_end = datetime.datetime.fromtimestamp(end).strftime('%Y-%m-%d %H:%M:%S')
+print "INSERT INTO measure (name, start, end) VALUES ('jobs', '%s', '%s');" % (sql_begin, sql_end)
 for username, user in users.iteritems():
-       print "INSERT INTO jobs (id_measure, user, jobs, done, fail, real_wait, real_time, wait_min, wait_max) VALUES (last_insert_id(), '%s', %d, %d, %d, %d, %d, %d, %d)" % (username, user.jobs, user.completed, user.total - user.completed, user.wait, user.time, user.wait_min, user.wait_max)
+       print "INSERT INTO jobs (id_measure, user, jobs, done, fail, real_wait, real_time, wait_min, wait_max) VALUES (last_insert_id(), '%s', %d, %d, %d, %d, %d, %d, %d);" % (username, user.jobs, user.completed, user.total - user.completed, user.wait, user.time, user.wait_min, user.wait_max)

diff --git a/manifests/accounting.pp b/manifests/accounting.pp
index cf95992..33b4307 100644 (file)
--- a/manifests/accounting.pp
+++ b/manifests/accounting.pp
@@ -183,10 +183,17 @@ class site_hadoop::accounting(
     }
   }
   file {'/usr/local/bin/accounting-jobs':
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    content => template('site_hadoop/accounting/jobs.sh.erb'),
+  }
+  file {'/usr/local/share/hadoop/accounting-jobs.py':
     owner  => 'root',
     group  => 'root',
     mode   => '0755',
-    content => template('site_hadoop/accounting/jobs.py.erb'),
+    source => 'puppet:///modules/site_hadoop/accounting/jobs.py',
+    require => File['/usr/local/share/hadoop'],
   }
   if $accounting_jobs {
     file{'/etc/cron.d/accounting-jobs':

diff --git a/templates/accounting/cron-jobs.erb b/templates/accounting/cron-jobs.erb
index b08ef5e..fe1c73f 100644 (file)
--- a/templates/accounting/cron-jobs.erb
+++ b/templates/accounting/cron-jobs.erb
@@ -2,4 +2,4 @@
 MAILTO='<%= @email -%>'
 
 <% end -%>
-#<%= @accounting_jobs -%>      hdfs    /usr/local/bin/accounting-jobs
+<%= @accounting_jobs -%>       hdfs    /usr/local/bin/accounting-jobs

diff --git a/templates/accounting/hdfs.sh.erb b/templates/accounting/hdfs.sh.erb
index a50c78c..3fd4d27 100755 (executable)
--- a/templates/accounting/hdfs.sh.erb
+++ b/templates/accounting/hdfs.sh.erb
@@ -2,7 +2,7 @@
 
 PREFIX='/usr/local'
 DEFAULTDIR='<%= scope.lookupvar('site_hadoop::defaultconfdir') -%>'
-export KRB5CCNAME='FILE:/tmp/krb5cc_hdfs_stat'
+export KRB5CCNAME='FILE:/tmp/krb5cc_accounting_hdfs'
 KEYTAB='FILE:/etc/security/keytab/nn.service.keytab'
 PRINCIPAL="nn/`hostname -f`"
 MYSQL_DB='accounting'

diff --git a/templates/accounting/jobs.sh.erb b/templates/accounting/jobs.sh.erb
new file mode 100755 (executable)
index 0000000..1b79ae5
--- /dev/null
+++ b/templates/accounting/jobs.sh.erb
@@ -0,0 +1,27 @@
+#! /bin/sh -e
+
+PREFIX='/usr/local'
+DEFAULTDIR='<%= scope.lookupvar('site_hadoop::defaultconfdir') -%>'
+export KRB5CCNAME='FILE:/tmp/krb5cc_accounting_jobs'
+KEYTAB='FILE:/etc/security/keytab/nn.service.keytab'
+PRINCIPAL="nn/`hostname -f`"
+MYSQL_DB='accounting'
+MYSQL_USER='accounting'
+MYSQL_PASSWORD=''
+
+if test -f ${DEFAULTDIR}/hadoop-accounting; then
+ . ${DEFAULTDIR}/hadoop-accounting
+fi
+
+if test -n "${PRINCIPAL}"; then
+  kinit -k -t ${KEYTAB} -l 60m ${PRINCIPAL}
+fi
+
+rm -f /tmp/accounting.jobs.sql
+${PREFIX}/share/hadoop/accounting-jobs.py --mapred-url '<%= @_mapred_url -%>' > /tmp/accounting.jobs.sql
+
+if test -n "${PRINCIPAL}"; then
+  kdestroy
+fi
+
+cat /tmp/accounting.jobs.sql | mysql --user ${MYSQL_USER} --password=${MYSQL_PASSWORD} ${MYSQL_DB}

diff --git a/templates/accounting/quota.sh.erb b/templates/accounting/quota.sh.erb
index 3b96e59..6098968 100755 (executable)
--- a/templates/accounting/quota.sh.erb
+++ b/templates/accounting/quota.sh.erb
@@ -2,7 +2,7 @@
 
 PREFIX='/usr/local'
 DEFAULTDIR='<%= scope.lookupvar('site_hadoop::defaultconfdir') -%>'
-export KRB5CCNAME='FILE:/tmp/krb5cc_hdfs_stat'
+export KRB5CCNAME='FILE:/tmp/krb5cc_accounting_quota'
 KEYTAB='FILE:/etc/security/keytab/nn.service.keytab'
 PRINCIPAL="nn/`hostname -f`"
 MYSQL_DB='accounting'
@@ -14,7 +14,7 @@ if test -f ${DEFAULTDIR}/hadoop-accounting; then
 fi
 
 if test -n "${PRINCIPAL}"; then
-  kinit -k -t ${KEYTAB} -l 5m ${PRINCIPAL}
+  kinit -k -t ${KEYTAB} -l 30m ${PRINCIPAL}
 fi
 
 rm -f /tmp/accounting.quota.txt