From a448300e12d61dd285344b5a406aaba69b026777 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= Date: Thu, 7 Jan 2016 01:15:15 +0100 Subject: [PATCH] More details in the tasts sumary: map/reduce. --- chroust-nodes-opt.py | 14 ++++++++++++-- nodes-sum-simple.dem | 11 +++++++++++ nodes-sum.dem | 8 +++++--- 3 files changed, 28 insertions(+), 5 deletions(-) create mode 100755 nodes-sum-simple.dem diff --git a/chroust-nodes-opt.py b/chroust-nodes-opt.py index 0bfefa7..49d322f 100755 --- a/chroust-nodes-opt.py +++ b/chroust-nodes-opt.py @@ -165,7 +165,9 @@ if debug: if header: usages = list() usages.append('time') - usages.append('mapred') + usages.append('n') + usages.append('map') + usages.append('reduce') for nodeid in sorted(nodes.keys()): usages.append(nodes[nodeid]) print "\t".join(usages) @@ -175,6 +177,8 @@ nodes_subjobs = dict() for nodeid in sorted(nodes.keys()): nodes_usage[nodeid] = list(0 for i in range(0, imax - imin)) nodes_subjobs[nodeid] = list(0 for i in range(0, imax - imin)) +map_usage = list(0 for i in range(0, imax - imin)) +reduce_usage = list(0 for i in range(0, imax - imin)) # SELECT * FROM subjobs WHERE (finish >= %s OR finish = 0 OR finish IS NULL) AND start <= %s', (start, end) st = db.cursor(MySQLdb.cursors.SSDictCursor) @@ -213,6 +217,12 @@ while data is not None: nodes_usage[data['nodeid']][i - imin] += usage; nodes_subjobs[data['nodeid']][i - imin] += 1; + if data['type'] == 'MAP': + map_usage[i - imin] += usage + elif data['type'] == 'REDUCE': + reduce_usage[i - imin] += usage + + data = st.fetchone() st.close() @@ -225,4 +235,4 @@ for i in range(0, imax - imin): usages.append("%s" % nodes_usage[nodeid][i]) subjobs += nodes_subjobs[nodeid][i] #print '#%s' % (int(i * 1000.0 * interval + min)) - print '%s\t%d\t%s' % (strp(i * 1000 * interval + min), subjobs, "\t".join(usages)) + print '%s\t%d\t%s\t%s\t%s' % (strp(i * 1000 * interval + min), subjobs, map_usage[i], reduce_usage[i], "\t".join(usages)) diff --git a/nodes-sum-simple.dem b/nodes-sum-simple.dem new file mode 100755 index 0000000..08b95c4 --- /dev/null +++ b/nodes-sum-simple.dem @@ -0,0 +1,11 @@ +#! /usr/bin/gnuplot -p + +set title "Hadoop: Map/Reduce Jobs Summary" font ',16' +set xdata time +set timefmt "%Y-%m-%d %H:%M:%S" +set format x "%Y-%m-%d" + +set xtics rotate by -20 + +#plot for [i=3:25] 'nodes.dat' using 1:(sum [col=4:i+1] column(col)) with lines notitle +plot for [i=26:26] 'nodes.dat' using 1:(sum [col=6:i+1] column(col)) with lines title 'jobs (max 384)' diff --git a/nodes-sum.dem b/nodes-sum.dem index bf234a4..0ef925e 100755 --- a/nodes-sum.dem +++ b/nodes-sum.dem @@ -1,11 +1,13 @@ #! /usr/bin/gnuplot -p -set title "Hadoop: Map/Reduce Jobs Summary" font ',16' +set title "Hadoop: Map/Reduce Tasks Summary" font ',16' set xdata time set timefmt "%Y-%m-%d %H:%M:%S" set format x "%Y-%m-%d" +set yrange [0:384] set xtics rotate by -20 -#plot for [i=3:25] 'nodes.dat' using 1:(sum [col=4:i+1] column(col)) with lines notitle -plot for [i=26:26] 'nodes.dat' using 1:(sum [col=4:i+1] column(col)) with lines title 'jobs (max 384)' +#plot for [i=3:4] 'nodes.dat' using 1:(sum [col=4:i+1] column(col)) with lines title 'jobs (max 384)' + +plot for [i=3:4] 'nodes.dat' using 1:(sum [col=4:i+1] column(col)) with lines title column(i) -- 1.8.2.3