From 348e44a7b77b689a091793f48c255f8fc5c50a31 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= Date: Fri, 6 Mar 2015 21:54:19 +0100 Subject: [PATCH] Sync with MetaCenterCloudPuppet/cesnet-site_hadoop. --- jobs.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/jobs.py b/jobs.py index df2856b..08bfcb3 100755 --- a/jobs.py +++ b/jobs.py @@ -46,13 +46,14 @@ node_ids = dict() jobs = dict() c = pycurl.Curl() -c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE) -c.setopt(pycurl.USERPWD, ":") def get_rest(base_url, url): + if debug >= 3: + print '# %s%s' % (base_url, url) + b = BytesIO() - c.setopt(pycurl.URL, base_url + url) + c.setopt(pycurl.URL, str(base_url + url)) #c.setopt(pycurl.WRITEDATA, b) c.setopt(pycurl.WRITEFUNCTION, b.write) c.perform() @@ -67,8 +68,6 @@ def get_rest(base_url, url): j = json.loads(s) - if debug >= 3: - print '# %s%s' % (base_url, url) if debug >= 4: print json.dumps(j, indent=4) @@ -142,7 +141,7 @@ OPTIONS are:\n\ elif cfg[0] == 'yarn': base_yarn_url = cfg[1] elif cfg[0] == 'ssl': - ssl = cfg[1] + ssl = int(cfg[1]) elif opt in ('-d', '--debug'): debug = int(arg) elif opt in ('--db'): @@ -169,6 +168,10 @@ OPTIONS are:\n\ print 'Args error' sys.exit(2) +if ssl: + c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE) + c.setopt(pycurl.USERPWD, ":") + base_yarn_url = gen_url(base_yarn_url, 8088, 8090) base_mapred_url = gen_url(base_mapred_url, 19888, 19890) @@ -308,7 +311,7 @@ for id, job in jobs.iteritems(): print ' elapsed: %.3f s' % ((job.mapred['finishTime'] - job.mapred['startTime']) / 1000.0) print ' finished: %.3f s' % ((job.mapred['finishTime'] - job.mapred['submitTime']) / 1000.0) - if job.yarn and debug >= 1: + if job.yarn and 'memorySeconds' in job.yarn.keys() and debug >= 1: print ' MB x s: %d' % job.yarn['memorySeconds'] print ' CPU x s: %d' % job.yarn['vcoreSeconds'] @@ -347,7 +350,7 @@ for id, job in jobs.iteritems(): st.execute("UPDATE jobs SET submit=%s, map=%s, reduce=%s WHERE id=%s", (job.mapred['submitTime'], job.mapred['mapsTotal'], job.mapred['reducesTotal'], id)) if debug >= 3: print '[db] job %s mapred updated' % id changed = 1 - if job.yarn: + if job.yarn and 'memorySeconds' in job.yarn.keys(): if data and data[MEMORY] == job.yarn['memorySeconds'] and data[CPU] == job.yarn['vcoreSeconds']: if debug >= 3: print '[db] job %s yarn is actual' % id else: -- 1.8.2.3