From 43bfa281efecc2e46c6b56f69b3c60891c0dd51f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ole=20Sch=C3=BCtt?= Date: Fri, 20 Apr 2018 14:28:50 +0000 Subject: [PATCH] Dashboard: Add git support svn-origin-rev: 18382 --- tools/dashboard/download_reports.sh | 2 +- tools/dashboard/generate_dashboard.py | 465 +++++++++++++------------- 2 files changed, 239 insertions(+), 228 deletions(-) diff --git a/tools/dashboard/download_reports.sh b/tools/dashboard/download_reports.sh index f35eb1974e..d5c2c0bf33 100755 --- a/tools/dashboard/download_reports.sh +++ b/tools/dashboard/download_reports.sh @@ -1,5 +1,5 @@ #!/bin/bash -wget -nH -Nxi http://dashboard.cp2k.org/archive/list_recent.txt +wget --no-verbose -nH -Nxi http://dashboard.cp2k.org/archive/list_recent.txt #EOF diff --git a/tools/dashboard/generate_dashboard.py b/tools/dashboard/generate_dashboard.py index baec7f6ca9..c39520f0e3 100755 --- a/tools/dashboard/generate_dashboard.py +++ b/tools/dashboard/generate_dashboard.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # Generates the CP2K Dashboard html page @@ -13,60 +13,61 @@ from email.mime.text import MIMEText import re import gzip from datetime import datetime, timedelta -import subprocess +from subprocess import check_output import traceback from os import path from pprint import pformat -from xml.dom import minidom from glob import glob import itertools - -try: - from urllib.parse import urlencode -except ImportError: - from urllib import urlencode - -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen - -try: - import configparser -except ImportError: - import ConfigParser as configparser +from urllib.parse import urlencode +from urllib.request import urlopen +import configparser import matplotlib as mpl mpl.use('Agg') # change backend, to run without X11 import matplotlib.pyplot as plt from matplotlib.ticker import AutoMinorLocator +#=============================================================================== +class GitLog(list): + def __init__(self): + cmd = ["git", "log", "--pretty=format:%H%n%ct%n%an%n%s%n%b<--seperator-->"] + outbytes = check_output(cmd) + output = outbytes.decode("utf-8") + for entry in output.split("<--seperator-->")[:-1]: + lines = entry.strip().split("\n") + commit = dict() + commit['git-sha'] = lines[0] + commit['date'] = datetime.fromtimestamp(float(lines[1])) + commit['author'] = lines[2] + commit['msg'] = lines[3] + m = re.match("git-svn-id: svn://svn.code.sf.net/p/cp2k/code/trunk@(\d+) .+", lines[-1]) + if(m): + commit['svn-rev'] = int(m.group(1)) + self.append(commit) + + # git-log outputs entries from new to old. + self.index = { c['git-sha']: i for i, c in enumerate(self) } + self.svn2git = { c['svn-rev']: c['git-sha'] for c in self if 'svn-rev' in c } + print("done.") + #=============================================================================== def main(): - if(len(sys.argv) not in (5, 6)): - print("Usage update_dashboard.py [--full-archive]") + if(len(sys.argv) != 5): + print("Usage update_dashboard.py ") sys.exit(1) - config_fn, abook_fn, status_fn, outdir = sys.argv[1:5] + config_fn, abook_fn, status_fn, outdir = sys.argv[1:] assert(outdir.endswith("/")) assert(path.exists(config_fn)) - full_archive = False - if(len(sys.argv) == 6): - assert(sys.argv[5] == "--full-archive") - full_archive = True - config = configparser.ConfigParser() config.read(config_fn) + log = GitLog() # Reads history from local git repo. - if(full_archive): - log = svn_log() # fetch entire history - else: - log = svn_log(limit=100) - gen_frontpage(config, log, abook_fn, status_fn, outdir) - - gen_archive(config, log, outdir, full_archive) - gen_url_list(config, outdir, full_archive) + gen_frontpage(config, log, abook_fn, status_fn, outdir) + gen_archive(config, log, outdir) + gen_url_list(config, outdir) #=============================================================================== def gen_frontpage(config, log, abook_fn, status_fn, outdir): @@ -77,22 +78,20 @@ def gen_frontpage(config, log, abook_fn, status_fn, outdir): else: status = dict() - trunk_revision = log[0]['num'] - log_index = dict([(r['num'], r) for r in log]) - now = datetime.utcnow().replace(microsecond=0) - - output = html_header(title="CP2K Dashboard", rev=trunk_revision) + output = html_header(title="CP2K Dashboard") output += '
\n' - output += html_svnbox(log) + output += html_gitbox(log) output += html_linkbox() output += '
\n' output += '\n' output += '' - output += '\n\n' + output += '\n\n' def get_sortkey(s): return config.getint(s, "sortkey") + now = datetime.utcnow().replace(microsecond=0) + for s in sorted(config.sections(), key=get_sortkey): print("Working on summary entry of: "+s) name = config.get(s,"name") @@ -102,48 +101,48 @@ def gen_frontpage(config, log, abook_fn, status_fn, outdir): do_notify = config.getboolean(s,"notify") if(config.has_option(s,"notify")) else False timeout = config.getint(s,"timeout") if(config.has_option(s,"timeout")) else 24 - # find latest revision that should have been tested by now + # find latest commit that should have been tested by now freshness_threshold = now - timedelta(hours=timeout) - revs_beyond_threshold = [ r['num'] for r in log if r['date'] < freshness_threshold ] - threshold_rev = revs_beyond_threshold[0] + ages_beyond_threshold = [ i for i, c in enumerate(log) if c['date'] < freshness_threshold ] + threshold_age = ages_beyond_threshold[0] # get and parse report report_txt = retrieve_report(report_url) - report = parse_report(report_txt, report_type) + report = parse_report(report_txt, report_type, log) - if(s not in status.keys()): + if(s not in status): status[s] = {'last_ok': None, 'notified': False} if(report['status'] == "OK"): - status[s]['last_ok'] = report['revision'] + status[s]['last_ok'] = report['git-sha'] status[s]['notified'] = False elif(do_notify and not status[s]['notified']): - send_notification(report, addressbook, status[s]['last_ok'], log_index, name, s) + send_notification(report, addressbook, status[s]['last_ok'], log, name, s) status[s]['notified'] = True - uptodate = report['revision']==trunk_revision # report from latest commit? - if(report['revision'] != None): - if(report['revision'] threshold_age): report['status'] = "OUTDATED" elif(report['status'] in ("OK", "FAILED")): # store only useful and fresh reports, prevents overwriting archive - fn = outdir+"archive/%s/rev_%d.txt.gz"%(s,report['revision']) - write_file(fn, report_txt, gz=True) + store_report(report, report_txt, s, outdir) + uptodate = report['git-sha'] == log[0]['git-sha'] # report from latest commit? output += '' output += ''%(s, name) output += ''%host output += status_cell(report['status'], report_url, uptodate) - #Revision - output += revision_cell(report['revision'], trunk_revision) + #Commit + output += commit_cell(report['git-sha'], log) #Summary output += ''%report['summary'] #Last OK if(report['status'] != "OK"): - output += revision_cell(status[s]['last_ok'], trunk_revision) + output += commit_cell(status[s]['last_ok'], log) else: output += '' @@ -152,104 +151,107 @@ def gen_frontpage(config, log, abook_fn, status_fn, outdir): output += '
NameHostStatusRevisionSummaryLast OK
CommitSummaryLast OK
%s%s%s
\n' output += '
\n' # complete flex-container output += html_footer() - write_file(outdir+"index.html", output.encode("utf8")) + write_file(outdir+"index.html", output) write_file(status_fn, pformat(status)) #=============================================================================== -def gen_archive(config, log, outdir, full_archive): - log_index = dict([(r['num'], r) for r in log]) - - if(full_archive): - print("Doing the full archive index pages") - trunk_revision = None # trunk_version changes quickly, leave it out. - html_out_postfix = "index_full.html" - urls_out_postfix = "list_full.txt" - other_index_link = '

View recent archive

' - else: - print("Doing recent archive index pages") - trunk_revision = log[0]['num'] - html_out_postfix = "index.html" - urls_out_postfix = "list_recent.txt" - other_index_link = '

View full archive

' +def gen_archive(config, log, outdir): for s in config.sections(): print("Working on archive page of: "+s) name = config.get(s,"name") report_type = config.get(s,"report_type") info_url = config.get(s,"info_url") if(config.has_option(s,"info_url")) else None - html_out_fn = outdir+"archive/%s/%s"%(s,html_out_postfix) - urls_out_fn = outdir+"archive/%s/%s"%(s,urls_out_postfix) - archive_files = glob(outdir+"archive/%s/rev_*.txt.gz"%s) - - # check if anything has changed - if(path.exists(html_out_fn)): - last_change = max([path.getmtime(fn) for fn in archive_files]) - if(last_change < path.getmtime(html_out_fn)): - print("Nothing has changed, skipping: "+html_out_fn) - continue + archive_files = glob(outdir+"archive/%s/rev_*.txt.gz"%s) + \ + glob(outdir+"archive/%s/commit_*.txt.gz"%s) # read all archived reports archive_reports = dict() - for fn in sorted(archive_files, reverse=True): - report_txt = gzip.open(fn, 'rb').read() - report = parse_report(report_txt, report_type) + for fn in archive_files: + report_txt = gzip.open(fn, 'rb').read().decode("utf-8") + report = parse_report(report_txt, report_type, log) report['url'] = path.basename(fn)[:-3] - archive_reports[report['revision']] = report + sha = report['git-sha'] + assert sha not in archive_reports + archive_reports[sha] = report - # generate archive index - archive_output = html_header(title=name, rev=trunk_revision) - archive_output += '

Go back to main page

\n' - if(info_url): - archive_output += '

Get more information

\n'%info_url - if(report_type == "generic"): - archive_output += gen_plots(archive_reports, log, outdir+"archive/"+s+"/", full_archive) - archive_output += other_index_link - archive_output += '\n' - archive_output += '\n\n' - - # loop over all relevant revisions - url_list = "" - rev_start = max(min(archive_reports.keys()), min(log_index.keys())) - rev_end = max(log_index.keys()) - for r in range(rev_end, rev_start-1, -1): - archive_output += '' - archive_output += revision_cell(r, trunk_revision) - if(archive_reports.has_key(r)): - report = archive_reports[r] - archive_output += status_cell(report['status'], report['url']) - archive_output += ''%report['summary'] - url_list += "https://dashboard.cp2k.org/archive/%s/%s.gz\n"%(s, report['url']) + # loop over all relevant commits + all_url_rows = [] + all_html_rows = [] + max_age = max([log.index[sha] for sha in archive_reports.keys()]) + for commit in log[:max_age]: + sha = commit['git-sha'] + html_row = '' + html_row += commit_cell(sha, log) + if(sha in archive_reports): + report = archive_reports[sha] + html_row += status_cell(report['status'], report['url']) + html_row += ''%report['summary'] + url_row = "https://dashboard.cp2k.org/archive/%s/%s.gz\n"%(s, report['url']) else: - archive_output += 2*'' - svn_rev = log_index[r] - archive_output += ''%svn_rev['author'] - archive_output += ''%svn_rev['msg'].split("\n")[0] - archive_output += '\n\n' + html_row += 2*'' + url_row = "" + html_row += ''%commit['author'] + html_row += ''%commit['msg'] + html_row += '\n\n' + all_html_rows.append(html_row) + all_url_rows.append(url_row) - archive_output += '
RevisionStatusSummaryAuthorCommit Message
%s
%s%s%s
%s%s
\n' - archive_output += other_index_link - archive_output += html_footer() - write_file(html_out_fn, archive_output.encode("utf8")) - write_file(urls_out_fn, url_list) + # generate html pages + for full_archive in (False, True): + if(full_archive): + html_out_postfix = "index_full.html" + urls_out_postfix = "list_full.txt" + other_index_link = '

View recent archive

' + max_age = None # output all + else: + html_out_postfix = "index.html" + urls_out_postfix = "list_recent.txt" + other_index_link = '

View full archive

' + max_age = 100 + + # generate archive index + output = html_header(title=name) + output += '

Go back to main page

\n' + if(info_url): + output += '

Get more information

\n'%info_url + if(report_type == "generic"): + output += gen_plots(archive_reports, log, outdir+"archive/"+s+"/", full_archive) + output += other_index_link + output += '\n' + output += '\n\n' + output += "".join(all_html_rows[:max_age]) + output += '
CommitStatusSummaryAuthorCommit Message
\n' + output += other_index_link + output += html_footer() + html_out_fn = outdir+"archive/%s/%s"%(s,html_out_postfix) + write_file(html_out_fn, output) + + url_list = "".join(all_url_rows[:max_age]) + urls_out_fn = outdir+"archive/%s/%s"%(s,urls_out_postfix) + write_file(urls_out_fn, url_list) #=============================================================================== -def gen_url_list(config, outdir, full_archive): - print("Working url list:") - urls_out_postfix = "list_full.txt" if full_archive else "list_recent.txt" - url_list = "" - for s in config.sections(): - fn = outdir+"archive/%s/%s"%(s,urls_out_postfix) - if(not path.exists(fn)): - continue - url_list += open(fn).read() - write_file(outdir+"archive/"+urls_out_postfix, url_list) +def gen_url_list(config, outdir): + print("Working on url lists.") + for postfix in ("list_full.txt", "list_recent.txt"): + url_list = "" + for s in config.sections(): + fn = outdir+"archive/%s/%s"%(s, postfix) + if(not path.exists(fn)): + continue + url_list += open(fn).read() + write_file(outdir+"archive/" + postfix, url_list) #=============================================================================== -def gen_plots(all_reports, log, outdir, full_archive): +def gen_plots(archive_reports, log, outdir, full_archive): + + ordered_shas = [commit['git-sha'] for commit in log] + ordered_reports = [archive_reports[sha] for sha in ordered_shas if sha in archive_reports] + # collect plot data plots = {} - for revision in sorted(all_reports.keys()): - report = all_reports[revision] + for report in ordered_reports: for p in report['plots']: if(p['name'] not in plots.keys()): plots[p['name']] = {'curves':{}} @@ -260,7 +262,8 @@ def gen_plots(all_reports, log, outdir, full_archive): if(pp['name'] not in p['curves'].keys()): p['curves'][pp['name']] = {'x':[], 'y':[], 'yerr':[]} c = p['curves'][pp['name']] - c['x'].append(report['revision']) + age = log.index[report['git-sha']] + c['x'].append(-age) c['y'].append(pp['y']) c['yerr'].append(pp['yerr']) c['label'] = pp['label'] # update label @@ -268,13 +271,13 @@ def gen_plots(all_reports, log, outdir, full_archive): # write raw data tags = sorted([(pname, cname) for pname, p in plots.items() for cname in p['curves'].keys()]) if(tags): - raw_output = "#%9s"%"revision" + raw_output = "# %6s %40s" % ("age", "commit") for pname, cname in tags: raw_output += " %18s %22s"%(pname+"/"+cname,pname+"/"+cname+"_err") raw_output += "\n" - for revision in sorted(all_reports.keys(), reverse=True): - report = all_reports[revision] - raw_output += "%10d"%revision + for report in reversed(ordered_reports): + age = log.index[report['git-sha']] + raw_output += "%8d %40s"%(-age, report['git-sha']) for pname, cname in tags: pp = [pp for pp in report['plotpoints'] if(pp['plot']==pname and pp['name']==cname)] assert(len(pp)<=1) @@ -286,16 +289,20 @@ def gen_plots(all_reports, log, outdir, full_archive): write_file(outdir+"plot_data.txt", raw_output) # create png images - fig_ext = "_full.png" if(full_archive) else ".png" - rev_end = log[0]['num'] - rev_start = min(all_reports.keys()) if(full_archive) else rev_end-100 + if (full_archive): + fig_ext = "_full.png" + max_age = max([log.index[sha] for sha in archive_reports.keys()]) + else: + fig_ext = ".png" + max_age = 100 + for pname, p in plots.items(): print("Working on plot: "+pname) fig = plt.figure(figsize=(12,4)) fig.subplots_adjust(bottom=0.18, left=0.06, right=0.70) fig.suptitle(p['title'], fontsize=14, fontweight='bold', x=0.4) ax = fig.add_subplot(111) - ax.set_xlabel('SVN Revision') + ax.set_xlabel('Commit Age') ax.set_ylabel(p['ylabel']) markers = itertools.cycle('os>^*') for cname in sorted(p['curves'].keys()): @@ -304,13 +311,13 @@ def gen_plots(all_reports, log, outdir, full_archive): ax.plot(c['x'], c['y'], label=c['label'], linewidth=2) # less crowded else: ax.errorbar(c['x'], c['y'], yerr=c['yerr'], label=c['label'], - marker=markers.next(), linewidth=2, markersize=6) - ax.set_xlim(rev_start-1, rev_end+1) + marker=next(markers), linewidth=2, markersize=6) + ax.set_xlim(-max_age-1, 0) ax.xaxis.set_minor_locator(AutoMinorLocator()) ax.legend(bbox_to_anchor=(1.01, 1), loc='upper left', numpoints=1, fancybox=True, shadow=True, borderaxespad=0.0) - visibles = [[y for x,y in zip(c['x'],c['y']) if x>=rev_start] for c in p['curves'].values()] # visible y-values - ymin = min([min(ys) for ys in visibles if ys]) # lowest point from lowest curve + visibles = [[y for x,y in zip(c['x'],c['y']) if x>=-max_age] for c in p['curves'].values()] # visible y-values + ymin = min([min(ys) for ys in visibles if ys]) # lowest point from lowest curve ymax = max([max(ys) for ys in visibles if ys]) # highest point from highest curve if(full_archive): ax.set_ylim(0.98*ymin, 1.02*ymax) @@ -318,6 +325,7 @@ def gen_plots(all_reports, log, outdir, full_archive): ymax2 = max([min(ys) for ys in visibles if ys]) # lowest point from highest curve ax.set_ylim(0.98*ymin, min(1.02*ymax, 1.3*ymax2)) # protect against outlayers fig.savefig(outdir+pname+fig_ext) + plt.close(fig) # write html output html_output = "" @@ -326,33 +334,11 @@ def gen_plots(all_reports, log, outdir, full_archive): return(html_output) #=============================================================================== -def retrieve_report(report_url): - try: - return urlopen(report_url, timeout=5).read() - except: - print(traceback.print_exc()) - return None - -#=============================================================================== -def parse_report(report_txt, report_type): - if(report_txt==None): - return( {'status':'UNKNOWN', 'summary':'Error while retrieving report.', 'revision':None} ) - try: - if(report_type == "regtest"): - return parse_regtest_report(report_txt) - elif(report_type == "generic"): - return parse_generic_report(report_txt) - else: - raise(Exception("Unknown report_type")) - except: - print(traceback.print_exc()) - return( {'status':'UNKNOWN', 'summary':'Error while parsing report.', 'revision':None} ) - -#=============================================================================== -def send_notification(report, addressbook, last_ok, svn_log, name, s): - rev_end = report['revision'] if(report['revision']) else max(svn_log.keys()) - if(rev_end==last_ok): return # probably a flapping tester - authors = set([svn_log[r]['author'] for r in range(last_ok+1, rev_end+1)]) +def send_notification(report, addressbook, last_ok, log, name, s): + idx_end = log.index[report['git-sha']] if(report['git-sha']) else 0 + idx_last_ok = log.index[last_ok] + if(idx_end == idx_last_ok): return # probably a flapping tester + authors = set([log[i]['author'] for i in range(idx_end, last_ok)]) emails = [addressbook[a] for a in authors] print("Sending email to: "+", ".join(emails)) @@ -361,7 +347,7 @@ def send_notification(report, addressbook, last_ok, svn_log, name, s): msg_txt += " test name: %s\n"%name msg_txt += " report state: %s\n"%report['status'] msg_txt += " report summary: %s\n"%report['summary'] - msg_txt += " last OK rev: %d\n\n"%last_ok + msg_txt += " last OK commit: %d\n\n"%last_ok msg_txt += "For more information visit:\n" msg_txt += " https://dashboard.cp2k.org/archive/%s/index.html \n\n"%s msg_txt += "Sincerely,\n" @@ -377,7 +363,7 @@ def send_notification(report, addressbook, last_ok, svn_log, name, s): smtp_conn.quit() #=============================================================================== -def html_header(title, rev=None): +def html_header(title): output = '\n' output += '\n' output += '\n' @@ -437,7 +423,7 @@ def html_header(title, rev=None): output += ' width: 15em;\n' output += '}\n' output += '\n' - output += '%s%s\n'%(title, (" (%d)"%rev if rev else "")) + output += '%s\n'%title output += '\n' output += '
Need Help?
\n' output += '

%s

\n'%title.upper() @@ -454,19 +440,22 @@ def html_linkbox(): return(output) #=============================================================================== -def html_svnbox(log): +def html_gitbox(log): now = datetime.utcnow() output = '
\n' output += '

Recent Commits

\n' - for r in log[0:10]: - url = "https://sourceforge.net/p/cp2k/code/%d/"%r['num'] - msg = r['msg'].split("\n")[0] + for commit in log[0:10]: + url = "https://github.com/cp2k/cp2k/commit/" + commit['git-sha'] + msg = commit['msg'] if(len(msg) > 27): msg = msg[:26] + "..." - output += '

%s
\n'%(r['msg'], url, msg) - delta = now - r['date'] + output += '

%s
\n'%(commit['msg'], url, msg) + delta = now - commit['date'] age = delta.days*24.0 + delta.seconds/3600.0 - output += '%s %s %.1fh ago.

\n'%(r['num'], r['author'], age) + output += 'git:' + commit['git-sha'][:7] + if 'svn-rev' in commit: + output += ' / svn:%d'%commit['svn-rev'] + output += '
\n%s %.1fh ago.

\n'%(commit['author'], age) output += '
\n' return(output) @@ -488,38 +477,11 @@ def write_file(fn, content, gz=False): if(old_content == content): print("File did not change: "+fn) return - f = gzip.open(fn, 'wb') if(gz) else open(fn, "w") - f.write(content) + f = gzip.open(fn, 'wb') if(gz) else open(fn, "wb") + f.write(content.encode("utf-8")) f.close() print("Wrote: "+fn) -#=============================================================================== -def svn_log(limit=None): - sys.stdout.write("Fetching svn log... ") - sys.stdout.flush() - # xml version contains nice UTC timestamp - if(limit): - cmd = "svn log --limit %d svn://svn.code.sf.net/p/cp2k/code --xml"%limit - else: - # our server is much faster, but might be 5 minutes old - cmd = "svn log https://svn.cp2k.org/cp2k --xml" - log_xml = check_output(cmd.split()) - dom = minidom.parseString(log_xml) - revisions = [] - for entry in dom.getElementsByTagName("logentry"): - rev = dict() - rev['num'] = int(entry.attributes['revision'].value) - rev_date_str = entry.getElementsByTagName('date')[0].firstChild.nodeValue - rev['date'] = datetime.strptime(rev_date_str[:19], '%Y-%m-%dT%H:%M:%S') - # some revisions lack an author - author = entry.getElementsByTagName('author') - rev['author'] = author[0].firstChild.nodeValue if (len(author)==1) else "" - msg = entry.getElementsByTagName('msg')[0].firstChild - rev['msg'] = msg.nodeValue.strip() if(msg) else "" - revisions.append(rev) - print("done.") - return(revisions) - #=============================================================================== def status_cell(status, report_url, uptodate=True): if(status == "OK"): @@ -531,22 +493,71 @@ def status_cell(status, report_url, uptodate=True): return('%s'%(bgcolor, report_url, status)) #=============================================================================== -def revision_cell(rev, trunk_rev): - if(rev == None): +def commit_cell(git_sha, log): + if(git_sha is None): return('N/A') - rev_url = "https://sourceforge.net/p/cp2k/code/%d/"%rev - rev_delta = "(%d)"%(rev - trunk_rev) if(trunk_rev) else "" - output = '%s %s'%(rev_url, rev, rev_delta) + idx = log.index[git_sha] + commit = log[idx] + git_url = "https://github.com/cp2k/cp2k/commit/" + git_sha + output = '%s'%(git_url, git_sha[:7]) + if 'svn-rev' in commit: + svn_rev = commit['svn-rev'] + svn_url = "https://sourceforge.net/p/cp2k/code/%d/"%svn_rev + output += ' / %d'%(svn_url, svn_rev) + output += ' (%d)'%(-idx) return(output) +#=============================================================================== +def retrieve_report(report_url): + try: + return urlopen(report_url, timeout=5).read().decode("utf-8") + except: + print(traceback.print_exc()) + return None + +#=============================================================================== +def store_report(report, report_txt, section, outdir): + if ('svn-rev' in report): + fn = outdir+"archive/%s/rev_%d.txt.gz"%(section, report['svn-rev']) + else: + fn = outdir+"archive/%s/commit_%s.txt.gz"%(section, report['git-sha']) + write_file(fn, report_txt, gz=True) + +#=============================================================================== +def parse_report(report_txt, report_type, log): + if(report_txt is None): + return( {'status':'UNKNOWN', 'summary':'Error while retrieving report.', 'git-sha':None} ) + try: + if(report_type == "regtest"): + report = parse_regtest_report(report_txt) + elif(report_type == "generic"): + report = parse_generic_report(report_txt) + else: + raise(Exception("Unknown report_type")) + if ('svn-rev' in report): + assert 'git-sha' not in report + rev = report['svn-rev'] + if rev not in log.svn2git: + return( {'status':'UNKNOWN', 'summary':'Could not convert svn revision to git commit.', 'git-sha':None} ) + report['git-sha'] = log.svn2git[rev] + return(report) + except: + print(traceback.print_exc()) + return( {'status':'UNKNOWN', 'summary':'Error while parsing report.', 'git-sha':None} ) + #=============================================================================== def parse_regtest_report(report_txt): m = re.search("svn: .*(Can't connect .*):", report_txt) if(m): - return({'revision':None, 'status':'UNKNOWN', 'summary':m.group(1)}) + return({'status':'UNKNOWN', 'summary':m.group(1)}) report = dict() - report['revision'] = int(re.search("(revision|Revision:) (\d+)\.?\n", report_txt).group(2)) + m = re.search("(revision|Revision:) (\d+)\.?\n", report_txt) + if (m): + report['svn-rev'] = int(m.group(2)) + m = re.search("(commit|Commit:) (\w{40})\n", report_txt) + if (m): + report['git-sha'] = int(m.group(2)) if("LOCKFILE" in report_txt): report['status'] = "UNKNOWN" @@ -590,22 +601,22 @@ def parse_regtest_report(report_txt): def parse_generic_report(report_txt): m = re.search("svn: .*(Can't connect .*):", report_txt) if(m): - return({'revision':None, 'status':'UNKNOWN', 'summary':m.group(1)}) + return({'status':'UNKNOWN', 'summary':m.group(1)}) report = dict() - report['revision'] = int(re.search("(^|\n)Revision: (\d+)\n", report_txt).group(2)) + + m = re.search("(^|\n)Revision: (\d+)\n", report_txt) + if (m): + report['svn-rev'] = int(m.group(2)) + m = re.search("(^|\n)Commit: (\w{40})\n", report_txt) + if (m): + report['git-sha'] = int(m.group(2)) + report['summary'] = re.search("(^|\n)Summary: (.+)\n", report_txt).group(2) report['status'] = re.search("(^|\n)Status: (.+)\n", report_txt).group(2) report['plots'] = [eval("dict(%s)"%m[1]) for m in re.findall("(^|\n)Plot: (.+)(?=\n)", report_txt)] report['plotpoints'] = [eval("dict(%s)"%m[1]) for m in re.findall("(^|\n)PlotPoint: (.+)(?=\n)", report_txt)] return(report) -#=============================================================================== -def check_output(command, **kwargs): - p = subprocess.Popen(command, stdout=subprocess.PIPE, **kwargs) - output = p.communicate()[0] - assert(p.wait() == 0) - return(output) - #=============================================================================== if(len(sys.argv)==2 and sys.argv[-1]=="--selftest"): pass #TODO implement selftest