USer quota statistics.

diff --git a/files/accounting/create.sql b/files/accounting/create.sql
index 6631f5b..157fc5a 100644 (file)
--- a/files/accounting/create.sql
+++ b/files/accounting/create.sql
@@ -122,5 +122,5 @@ DELIMITER ;
 
 CREATE VIEW view_measures AS SELECT m.* FROM measure m, statistic s WHERE s.last_id_measure = m.id_measure;
 CREATE VIEW view_hdfs AS SELECT m.seq, m.time, h.hostname, h.full, h.disk, h.disk_used, h.disk_free, h.block_under, h.block_corrupt, h.block_missing FROM hdfs h, measure m WHERE h.id_measure=m.id_measure;
-CREATE VIEW view_quotas AS SELECT m.seq, m.time, q.user, q.used FROM quota q, measure m WHERE q.id_measure=m.id_measure;
+CREATE VIEW view_quota AS SELECT m.seq, m.time, q.user, q.used FROM quota q, measure m WHERE q.id_measure=m.id_measure;
 CREATE VIEW view_jobstat AS SELECT m.seq, m.time, m.start, m.end, j.user, j.total, j.fails, j.wait_min, j.wait_avg, j.wait_max FROM jobstat j, measure m WHERE j.id_measure=m.id_measure;

diff --git a/files/accounting/quota.awk b/files/accounting/quota.awk
new file mode 100644 (file)
index 0000000..b3b0a44
--- /dev/null
+++ b/files/accounting/quota.awk
@@ -0,0 +1,26 @@
+#
+# Parsing output of:
+#
+# hdfs dfs -du -s '/user/*'
+#
+
+function dbstr(s) {
+       if (s) { return "'" s "'" }
+       else { return "NULL" }
+}
+
+function dbi(i) {
+       if (i >= 0) { return i }
+       else { return "NULL" }
+}
+
+BEGIN {
+       FS="[ \t/]+"
+       print "INSERT INTO measure (name) VALUES ('quota');";
+}
+
+{
+       used=$1
+       user=$4
+       print "INSERT INTO quota (id_measure, user, used) VALUES (last_insert_id(), " dbstr(user) ", " dbi(used) ");"
+}

diff --git a/manifests/accounting.pp b/manifests/accounting.pp
index c1ec1e2..f492095 100644 (file)
--- a/manifests/accounting.pp
+++ b/manifests/accounting.pp
@@ -66,29 +66,22 @@ class site_hadoop::accounting(
   $db_password = undef,
   $email = undef,
   $hdfs  = undef,
+  $quota  = undef,
   $principal = undef,
 ) {
-  file {'/usr/local/bin/accounting-hdfs':
-    owner  => 'root',
-    group  => 'root',
-    mode   => '0755',
-    content => template('site_hadoop/accounting/hdfs.sh.erb'),
-  }
-
+  # common
   file{'/usr/local/share/hadoop':
     ensure  => 'directory',
     owner => 'root',
     group => 'root',
     mode  => '0755',
   }
-  ->
-  file {'/usr/local/share/hadoop/accounting-hdfs.awk':
-    owner => 'root',
-    group => 'root',
-    mode  => '0644',
-    source => 'puppet:///modules/site_hadoop/accounting/hdfs.awk',
+  file{"${site_hadoop::defaultconfdir}/hadoop-accounting":
+    owner  => 'hdfs',
+    group  => 'hdfs',
+    mode   => '0400',
+    content => template('site_hadoop/accounting/hadoop-accounting.erb'),
   }
-
   file{'/usr/local/share/hadoop/accounting.sql':
     owner => 'root',
     group => 'root',
@@ -96,13 +89,20 @@ class site_hadoop::accounting(
     source => 'puppet:///modules/site_hadoop/accounting/create.sql',
   }
 
-  file{"${site_hadoop::defaultconfdir}/hadoop-accounting":
-    owner  => 'hdfs',
-    group  => 'hdfs',
-    mode   => '0400',
-    content => template('site_hadoop/accounting/hadoop-accounting.erb'),
+  # hdfs data
+  file {'/usr/local/bin/accounting-hdfs':
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    content => template('site_hadoop/accounting/hdfs.sh.erb'),
+  }
+  file {'/usr/local/share/hadoop/accounting-hdfs.awk':
+    owner => 'root',
+    group => 'root',
+    mode  => '0644',
+    source => 'puppet:///modules/site_hadoop/accounting/hdfs.awk',
+    require => File['/usr/local/share/hadoop'],
   }
-
   if $hdfs {
     file{'/etc/cron.d/accounting-hdfs':
       owner => 'root',
@@ -115,4 +115,31 @@ class site_hadoop::accounting(
       ensure => 'absent',
     }
   }
+
+  # user quota
+  file {'/usr/local/bin/accounting-quota':
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    content => template('site_hadoop/accounting/quota.sh.erb'),
+  }
+  file {'/usr/local/share/hadoop/accounting-quota.awk':
+    owner => 'root',
+    group => 'root',
+    mode  => '0644',
+    source => 'puppet:///modules/site_hadoop/accounting/quota.awk',
+    require => File['/usr/local/share/hadoop'],
+  }
+  if $quota {
+    file{'/etc/cron.d/accounting-quota':
+      owner => 'root',
+      group => 'root',
+      mode  => '0644',
+      content => template('site_hadoop/accounting/cron-quota.erb'),
+    }
+  } else {
+    file{'/etc/cron.d/accounting-quota':
+      ensure => 'absent',
+    }
+  }
 }

diff --git a/templates/accounting/cron-quota.erb b/templates/accounting/cron-quota.erb
new file mode 100644 (file)
index 0000000..ef9f4f8
--- /dev/null
+++ b/templates/accounting/cron-quota.erb
@@ -0,0 +1,5 @@
+<% if @email -%>
+MAILTO='<%= @email -%>'
+
+<% end -%>
+<%= @quota -%> hdfs    /usr/local/bin/accounting-quota

diff --git a/templates/accounting/quota.sh.erb b/templates/accounting/quota.sh.erb
new file mode 100755 (executable)
index 0000000..3b96e59
--- /dev/null
+++ b/templates/accounting/quota.sh.erb
@@ -0,0 +1,27 @@
+#! /bin/sh -e
+
+PREFIX='/usr/local'
+DEFAULTDIR='<%= scope.lookupvar('site_hadoop::defaultconfdir') -%>'
+export KRB5CCNAME='FILE:/tmp/krb5cc_hdfs_stat'
+KEYTAB='FILE:/etc/security/keytab/nn.service.keytab'
+PRINCIPAL="nn/`hostname -f`"
+MYSQL_DB='accounting'
+MYSQL_USER='accounting'
+MYSQL_PASSWORD=''
+
+if test -f ${DEFAULTDIR}/hadoop-accounting; then
+ . ${DEFAULTDIR}/hadoop-accounting
+fi
+
+if test -n "${PRINCIPAL}"; then
+  kinit -k -t ${KEYTAB} -l 5m ${PRINCIPAL}
+fi
+
+rm -f /tmp/accounting.quota.txt
+hdfs dfs -du -s '/user/*' >/tmp/accounting.quota.txt
+
+if test -n "${PRINCIPAL}"; then
+  kdestroy
+fi
+
+cat /tmp/accounting.quota.txt | awk -f ${PREFIX}/share/hadoop/accounting-quota.awk | mysql --user ${MYSQL_USER} --password=${MYSQL_PASSWORD} ${MYSQL_DB}