From 7aa51a53b5e352494fdef4aca2d6fd67a4151f43 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= Date: Sun, 18 Jan 2015 12:29:03 +0100 Subject: [PATCH] Finalize jobs statistics and the laste cleanups. --- .../jobs.py.erb => files/accounting/jobs.py | 37 +++++++++++++++++++--- manifests/accounting.pp | 9 +++++- templates/accounting/cron-jobs.erb | 2 +- templates/accounting/hdfs.sh.erb | 2 +- templates/accounting/jobs.sh.erb | 27 ++++++++++++++++ templates/accounting/quota.sh.erb | 4 +-- 6 files changed, 71 insertions(+), 10 deletions(-) rename templates/accounting/jobs.py.erb => files/accounting/jobs.py (59%) create mode 100755 templates/accounting/jobs.sh.erb diff --git a/templates/accounting/jobs.py.erb b/files/accounting/jobs.py similarity index 59% rename from templates/accounting/jobs.py.erb rename to files/accounting/jobs.py index d383310..bd9deb6 100755 --- a/templates/accounting/jobs.py.erb +++ b/files/accounting/jobs.py @@ -6,6 +6,31 @@ from io import BytesIO import time import calendar import datetime +import sys +import getopt +import socket + +base_url = "http://" + socket.getfqdn() + ":19888" +begin_rel = 24 * 3600 +end_rel = 0 + +try: + opts, args = getopt.getopt(sys.argv[1:], "hm:b:e:", ["help", "mapred-url=", "begin=", "end="]) +except getopt.GetoptError: + print 'Args error' + sys.exit(2) +for opt, arg in opts: + if opt in ('-h', '--help'): + print('jobs.py [-h|--help] [-m|--mapred-url URL] [-b|--begin] [-e|--end]') + elif opt in ('-m', '--mapred-url'): + base_url = arg + elif opt in ('-b', '--begin'): + begin_rel = int(arg) + elif opt in ('-e', '--end'): + end_rel = int(arg) + else: + print 'Args error' + sys.exit(2) # epoch time of local date #now = datetime.date.today().strftime('%s') @@ -15,9 +40,9 @@ now0 = datetime.date.today() now = calendar.timegm(datetime.datetime(now0.year, now0.month, now0.day, 0, 0).timetuple()) print '# ' + str(now0) -begin = 1000 * (now - 24 * 3600) -end = 1000 * now -url = "<%= @_mapred_url -%>/ws/v1/history/mapreduce/jobs?finishedTimeBegin=" + str(begin) + "&finishedTimeEnd=" + str(end) +begin = now - begin_rel +end = now - end_rel +url = base_url + "/ws/v1/history/mapreduce/jobs?finishedTimeBegin=" + str(1000 * begin) + "&finishedTimeEnd=" + str(1000 * end) print '# ' + url b = BytesIO() @@ -74,6 +99,8 @@ if j["jobs"]: # print '#[progress]', username, users[username].total, user.completed, user.wait, user.time -print "INSERT INTO measure (name) VALUES ('jobs')" +sql_begin = datetime.datetime.fromtimestamp(begin).strftime('%Y-%m-%d %H:%M:%S') +sql_end = datetime.datetime.fromtimestamp(end).strftime('%Y-%m-%d %H:%M:%S') +print "INSERT INTO measure (name, start, end) VALUES ('jobs', '%s', '%s');" % (sql_begin, sql_end) for username, user in users.iteritems(): - print "INSERT INTO jobs (id_measure, user, jobs, done, fail, real_wait, real_time, wait_min, wait_max) VALUES (last_insert_id(), '%s', %d, %d, %d, %d, %d, %d, %d)" % (username, user.jobs, user.completed, user.total - user.completed, user.wait, user.time, user.wait_min, user.wait_max) + print "INSERT INTO jobs (id_measure, user, jobs, done, fail, real_wait, real_time, wait_min, wait_max) VALUES (last_insert_id(), '%s', %d, %d, %d, %d, %d, %d, %d);" % (username, user.jobs, user.completed, user.total - user.completed, user.wait, user.time, user.wait_min, user.wait_max) diff --git a/manifests/accounting.pp b/manifests/accounting.pp index cf95992..33b4307 100644 --- a/manifests/accounting.pp +++ b/manifests/accounting.pp @@ -183,10 +183,17 @@ class site_hadoop::accounting( } } file {'/usr/local/bin/accounting-jobs': + owner => 'root', + group => 'root', + mode => '0755', + content => template('site_hadoop/accounting/jobs.sh.erb'), + } + file {'/usr/local/share/hadoop/accounting-jobs.py': owner => 'root', group => 'root', mode => '0755', - content => template('site_hadoop/accounting/jobs.py.erb'), + source => 'puppet:///modules/site_hadoop/accounting/jobs.py', + require => File['/usr/local/share/hadoop'], } if $accounting_jobs { file{'/etc/cron.d/accounting-jobs': diff --git a/templates/accounting/cron-jobs.erb b/templates/accounting/cron-jobs.erb index b08ef5e..fe1c73f 100644 --- a/templates/accounting/cron-jobs.erb +++ b/templates/accounting/cron-jobs.erb @@ -2,4 +2,4 @@ MAILTO='<%= @email -%>' <% end -%> -#<%= @accounting_jobs -%> hdfs /usr/local/bin/accounting-jobs +<%= @accounting_jobs -%> hdfs /usr/local/bin/accounting-jobs diff --git a/templates/accounting/hdfs.sh.erb b/templates/accounting/hdfs.sh.erb index a50c78c..3fd4d27 100755 --- a/templates/accounting/hdfs.sh.erb +++ b/templates/accounting/hdfs.sh.erb @@ -2,7 +2,7 @@ PREFIX='/usr/local' DEFAULTDIR='<%= scope.lookupvar('site_hadoop::defaultconfdir') -%>' -export KRB5CCNAME='FILE:/tmp/krb5cc_hdfs_stat' +export KRB5CCNAME='FILE:/tmp/krb5cc_accounting_hdfs' KEYTAB='FILE:/etc/security/keytab/nn.service.keytab' PRINCIPAL="nn/`hostname -f`" MYSQL_DB='accounting' diff --git a/templates/accounting/jobs.sh.erb b/templates/accounting/jobs.sh.erb new file mode 100755 index 0000000..1b79ae5 --- /dev/null +++ b/templates/accounting/jobs.sh.erb @@ -0,0 +1,27 @@ +#! /bin/sh -e + +PREFIX='/usr/local' +DEFAULTDIR='<%= scope.lookupvar('site_hadoop::defaultconfdir') -%>' +export KRB5CCNAME='FILE:/tmp/krb5cc_accounting_jobs' +KEYTAB='FILE:/etc/security/keytab/nn.service.keytab' +PRINCIPAL="nn/`hostname -f`" +MYSQL_DB='accounting' +MYSQL_USER='accounting' +MYSQL_PASSWORD='' + +if test -f ${DEFAULTDIR}/hadoop-accounting; then + . ${DEFAULTDIR}/hadoop-accounting +fi + +if test -n "${PRINCIPAL}"; then + kinit -k -t ${KEYTAB} -l 60m ${PRINCIPAL} +fi + +rm -f /tmp/accounting.jobs.sql +${PREFIX}/share/hadoop/accounting-jobs.py --mapred-url '<%= @_mapred_url -%>' > /tmp/accounting.jobs.sql + +if test -n "${PRINCIPAL}"; then + kdestroy +fi + +cat /tmp/accounting.jobs.sql | mysql --user ${MYSQL_USER} --password=${MYSQL_PASSWORD} ${MYSQL_DB} diff --git a/templates/accounting/quota.sh.erb b/templates/accounting/quota.sh.erb index 3b96e59..6098968 100755 --- a/templates/accounting/quota.sh.erb +++ b/templates/accounting/quota.sh.erb @@ -2,7 +2,7 @@ PREFIX='/usr/local' DEFAULTDIR='<%= scope.lookupvar('site_hadoop::defaultconfdir') -%>' -export KRB5CCNAME='FILE:/tmp/krb5cc_hdfs_stat' +export KRB5CCNAME='FILE:/tmp/krb5cc_accounting_quota' KEYTAB='FILE:/etc/security/keytab/nn.service.keytab' PRINCIPAL="nn/`hostname -f`" MYSQL_DB='accounting' @@ -14,7 +14,7 @@ if test -f ${DEFAULTDIR}/hadoop-accounting; then fi if test -n "${PRINCIPAL}"; then - kinit -k -t ${KEYTAB} -l 5m ${PRINCIPAL} + kinit -k -t ${KEYTAB} -l 30m ${PRINCIPAL} fi rm -f /tmp/accounting.quota.txt -- 1.8.2.3