mirror of
https://github.com/cp2k/cp2k.git
synced 2026-07-28 14:15:19 -04:00
Dashboard: Add git support
svn-origin-rev: 18382
This commit is contained in:
Ole Schütt
parent
02196a15bb
commit
43bfa281ef
2 changed files with 239 additions and 228 deletions
|
|
@ -1,5 +1,5 @@
|
|||
#!/bin/bash
|
||||
|
||||
wget -nH -Nxi http://dashboard.cp2k.org/archive/list_recent.txt
|
||||
wget --no-verbose -nH -Nxi http://dashboard.cp2k.org/archive/list_recent.txt
|
||||
|
||||
#EOF
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/python
|
||||
#!/usr/bin/python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Generates the CP2K Dashboard html page
|
||||
|
|
@ -13,60 +13,61 @@ from email.mime.text import MIMEText
|
|||
import re
|
||||
import gzip
|
||||
from datetime import datetime, timedelta
|
||||
import subprocess
|
||||
from subprocess import check_output
|
||||
import traceback
|
||||
from os import path
|
||||
from pprint import pformat
|
||||
from xml.dom import minidom
|
||||
from glob import glob
|
||||
import itertools
|
||||
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
except ImportError:
|
||||
from urllib import urlencode
|
||||
|
||||
try:
|
||||
from urllib.request import urlopen
|
||||
except ImportError:
|
||||
from urllib2 import urlopen
|
||||
|
||||
try:
|
||||
import configparser
|
||||
except ImportError:
|
||||
import ConfigParser as configparser
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import urlopen
|
||||
import configparser
|
||||
|
||||
import matplotlib as mpl
|
||||
mpl.use('Agg') # change backend, to run without X11
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.ticker import AutoMinorLocator
|
||||
|
||||
#===============================================================================
|
||||
class GitLog(list):
|
||||
def __init__(self):
|
||||
cmd = ["git", "log", "--pretty=format:%H%n%ct%n%an%n%s%n%b<--seperator-->"]
|
||||
outbytes = check_output(cmd)
|
||||
output = outbytes.decode("utf-8")
|
||||
for entry in output.split("<--seperator-->")[:-1]:
|
||||
lines = entry.strip().split("\n")
|
||||
commit = dict()
|
||||
commit['git-sha'] = lines[0]
|
||||
commit['date'] = datetime.fromtimestamp(float(lines[1]))
|
||||
commit['author'] = lines[2]
|
||||
commit['msg'] = lines[3]
|
||||
m = re.match("git-svn-id: svn://svn.code.sf.net/p/cp2k/code/trunk@(\d+) .+", lines[-1])
|
||||
if(m):
|
||||
commit['svn-rev'] = int(m.group(1))
|
||||
self.append(commit)
|
||||
|
||||
# git-log outputs entries from new to old.
|
||||
self.index = { c['git-sha']: i for i, c in enumerate(self) }
|
||||
self.svn2git = { c['svn-rev']: c['git-sha'] for c in self if 'svn-rev' in c }
|
||||
print("done.")
|
||||
|
||||
#===============================================================================
|
||||
def main():
|
||||
if(len(sys.argv) not in (5, 6)):
|
||||
print("Usage update_dashboard.py <config-file> <addressbook> <status-file> <output-dir> [--full-archive]")
|
||||
if(len(sys.argv) != 5):
|
||||
print("Usage update_dashboard.py <config-file> <addressbook> <status-file> <output-dir>")
|
||||
sys.exit(1)
|
||||
|
||||
config_fn, abook_fn, status_fn, outdir = sys.argv[1:5]
|
||||
config_fn, abook_fn, status_fn, outdir = sys.argv[1:]
|
||||
assert(outdir.endswith("/"))
|
||||
assert(path.exists(config_fn))
|
||||
|
||||
full_archive = False
|
||||
if(len(sys.argv) == 6):
|
||||
assert(sys.argv[5] == "--full-archive")
|
||||
full_archive = True
|
||||
|
||||
config = configparser.ConfigParser()
|
||||
config.read(config_fn)
|
||||
log = GitLog() # Reads history from local git repo.
|
||||
|
||||
if(full_archive):
|
||||
log = svn_log() # fetch entire history
|
||||
else:
|
||||
log = svn_log(limit=100)
|
||||
gen_frontpage(config, log, abook_fn, status_fn, outdir)
|
||||
|
||||
gen_archive(config, log, outdir, full_archive)
|
||||
gen_url_list(config, outdir, full_archive)
|
||||
gen_frontpage(config, log, abook_fn, status_fn, outdir)
|
||||
gen_archive(config, log, outdir)
|
||||
gen_url_list(config, outdir)
|
||||
|
||||
#===============================================================================
|
||||
def gen_frontpage(config, log, abook_fn, status_fn, outdir):
|
||||
|
|
@ -77,22 +78,20 @@ def gen_frontpage(config, log, abook_fn, status_fn, outdir):
|
|||
else:
|
||||
status = dict()
|
||||
|
||||
trunk_revision = log[0]['num']
|
||||
log_index = dict([(r['num'], r) for r in log])
|
||||
now = datetime.utcnow().replace(microsecond=0)
|
||||
|
||||
output = html_header(title="CP2K Dashboard", rev=trunk_revision)
|
||||
output = html_header(title="CP2K Dashboard")
|
||||
output += '<div id="flex-container"><div>\n'
|
||||
output += html_svnbox(log)
|
||||
output += html_gitbox(log)
|
||||
output += html_linkbox()
|
||||
output += '</div>\n'
|
||||
output += '<table border="1" cellspacing="3" cellpadding="5">\n'
|
||||
output += '<tr><th>Name</th><th>Host</th><th>Status</th>'
|
||||
output += '<th>Revision</th><th>Summary</th><th>Last OK</th></tr>\n\n'
|
||||
output += '<th>Commit</th><th>Summary</th><th>Last OK</th></tr>\n\n'
|
||||
|
||||
def get_sortkey(s):
|
||||
return config.getint(s, "sortkey")
|
||||
|
||||
now = datetime.utcnow().replace(microsecond=0)
|
||||
|
||||
for s in sorted(config.sections(), key=get_sortkey):
|
||||
print("Working on summary entry of: "+s)
|
||||
name = config.get(s,"name")
|
||||
|
|
@ -102,48 +101,48 @@ def gen_frontpage(config, log, abook_fn, status_fn, outdir):
|
|||
do_notify = config.getboolean(s,"notify") if(config.has_option(s,"notify")) else False
|
||||
timeout = config.getint(s,"timeout") if(config.has_option(s,"timeout")) else 24
|
||||
|
||||
# find latest revision that should have been tested by now
|
||||
# find latest commit that should have been tested by now
|
||||
freshness_threshold = now - timedelta(hours=timeout)
|
||||
revs_beyond_threshold = [ r['num'] for r in log if r['date'] < freshness_threshold ]
|
||||
threshold_rev = revs_beyond_threshold[0]
|
||||
ages_beyond_threshold = [ i for i, c in enumerate(log) if c['date'] < freshness_threshold ]
|
||||
threshold_age = ages_beyond_threshold[0]
|
||||
|
||||
# get and parse report
|
||||
report_txt = retrieve_report(report_url)
|
||||
report = parse_report(report_txt, report_type)
|
||||
report = parse_report(report_txt, report_type, log)
|
||||
|
||||
if(s not in status.keys()):
|
||||
if(s not in status):
|
||||
status[s] = {'last_ok': None, 'notified': False}
|
||||
|
||||
if(report['status'] == "OK"):
|
||||
status[s]['last_ok'] = report['revision']
|
||||
status[s]['last_ok'] = report['git-sha']
|
||||
status[s]['notified'] = False
|
||||
elif(do_notify and not status[s]['notified']):
|
||||
send_notification(report, addressbook, status[s]['last_ok'], log_index, name, s)
|
||||
send_notification(report, addressbook, status[s]['last_ok'], log, name, s)
|
||||
status[s]['notified'] = True
|
||||
|
||||
uptodate = report['revision']==trunk_revision # report from latest commit?
|
||||
if(report['revision'] != None):
|
||||
if(report['revision']<threshold_rev):
|
||||
if(report['git-sha']):
|
||||
age = log.index[report['git-sha']]
|
||||
if(age > threshold_age):
|
||||
report['status'] = "OUTDATED"
|
||||
elif(report['status'] in ("OK", "FAILED")):
|
||||
# store only useful and fresh reports, prevents overwriting archive
|
||||
fn = outdir+"archive/%s/rev_%d.txt.gz"%(s,report['revision'])
|
||||
write_file(fn, report_txt, gz=True)
|
||||
store_report(report, report_txt, s, outdir)
|
||||
|
||||
uptodate = report['git-sha'] == log[0]['git-sha'] # report from latest commit?
|
||||
output += '<tr align="center">'
|
||||
output += '<td align="left"><a href="archive/%s/index.html">%s</a></td>'%(s, name)
|
||||
output += '<td align="left">%s</td>'%host
|
||||
output += status_cell(report['status'], report_url, uptodate)
|
||||
|
||||
#Revision
|
||||
output += revision_cell(report['revision'], trunk_revision)
|
||||
#Commit
|
||||
output += commit_cell(report['git-sha'], log)
|
||||
|
||||
#Summary
|
||||
output += '<td align="left">%s</td>'%report['summary']
|
||||
|
||||
#Last OK
|
||||
if(report['status'] != "OK"):
|
||||
output += revision_cell(status[s]['last_ok'], trunk_revision)
|
||||
output += commit_cell(status[s]['last_ok'], log)
|
||||
else:
|
||||
output += '<td></td>'
|
||||
|
||||
|
|
@ -152,104 +151,107 @@ def gen_frontpage(config, log, abook_fn, status_fn, outdir):
|
|||
output += '</table>\n'
|
||||
output += '<div id="dummybox"></div></div>\n' # complete flex-container
|
||||
output += html_footer()
|
||||
write_file(outdir+"index.html", output.encode("utf8"))
|
||||
write_file(outdir+"index.html", output)
|
||||
write_file(status_fn, pformat(status))
|
||||
|
||||
#===============================================================================
|
||||
def gen_archive(config, log, outdir, full_archive):
|
||||
log_index = dict([(r['num'], r) for r in log])
|
||||
|
||||
if(full_archive):
|
||||
print("Doing the full archive index pages")
|
||||
trunk_revision = None # trunk_version changes quickly, leave it out.
|
||||
html_out_postfix = "index_full.html"
|
||||
urls_out_postfix = "list_full.txt"
|
||||
other_index_link = '<p>View <a href="index.html">recent archive</a></p>'
|
||||
else:
|
||||
print("Doing recent archive index pages")
|
||||
trunk_revision = log[0]['num']
|
||||
html_out_postfix = "index.html"
|
||||
urls_out_postfix = "list_recent.txt"
|
||||
other_index_link = '<p>View <a href="index_full.html">full archive</a></p>'
|
||||
def gen_archive(config, log, outdir):
|
||||
|
||||
for s in config.sections():
|
||||
print("Working on archive page of: "+s)
|
||||
name = config.get(s,"name")
|
||||
report_type = config.get(s,"report_type")
|
||||
info_url = config.get(s,"info_url") if(config.has_option(s,"info_url")) else None
|
||||
html_out_fn = outdir+"archive/%s/%s"%(s,html_out_postfix)
|
||||
urls_out_fn = outdir+"archive/%s/%s"%(s,urls_out_postfix)
|
||||
archive_files = glob(outdir+"archive/%s/rev_*.txt.gz"%s)
|
||||
|
||||
# check if anything has changed
|
||||
if(path.exists(html_out_fn)):
|
||||
last_change = max([path.getmtime(fn) for fn in archive_files])
|
||||
if(last_change < path.getmtime(html_out_fn)):
|
||||
print("Nothing has changed, skipping: "+html_out_fn)
|
||||
continue
|
||||
archive_files = glob(outdir+"archive/%s/rev_*.txt.gz"%s) + \
|
||||
glob(outdir+"archive/%s/commit_*.txt.gz"%s)
|
||||
|
||||
# read all archived reports
|
||||
archive_reports = dict()
|
||||
for fn in sorted(archive_files, reverse=True):
|
||||
report_txt = gzip.open(fn, 'rb').read()
|
||||
report = parse_report(report_txt, report_type)
|
||||
for fn in archive_files:
|
||||
report_txt = gzip.open(fn, 'rb').read().decode("utf-8")
|
||||
report = parse_report(report_txt, report_type, log)
|
||||
report['url'] = path.basename(fn)[:-3]
|
||||
archive_reports[report['revision']] = report
|
||||
sha = report['git-sha']
|
||||
assert sha not in archive_reports
|
||||
archive_reports[sha] = report
|
||||
|
||||
# generate archive index
|
||||
archive_output = html_header(title=name, rev=trunk_revision)
|
||||
archive_output += '<p>Go back to <a href="../../index.html">main page</a></p>\n'
|
||||
if(info_url):
|
||||
archive_output += '<p>Get <a href="%s">more information</a></p>\n'%info_url
|
||||
if(report_type == "generic"):
|
||||
archive_output += gen_plots(archive_reports, log, outdir+"archive/"+s+"/", full_archive)
|
||||
archive_output += other_index_link
|
||||
archive_output += '<table border="1" cellspacing="3" cellpadding="5">\n'
|
||||
archive_output += '<tr><th>Revision</th><th>Status</th><th>Summary</th><th>Author</th><th>Commit Message</th></tr>\n\n'
|
||||
|
||||
# loop over all relevant revisions
|
||||
url_list = ""
|
||||
rev_start = max(min(archive_reports.keys()), min(log_index.keys()))
|
||||
rev_end = max(log_index.keys())
|
||||
for r in range(rev_end, rev_start-1, -1):
|
||||
archive_output += '<tr>'
|
||||
archive_output += revision_cell(r, trunk_revision)
|
||||
if(archive_reports.has_key(r)):
|
||||
report = archive_reports[r]
|
||||
archive_output += status_cell(report['status'], report['url'])
|
||||
archive_output += '<td align="left">%s</td>'%report['summary']
|
||||
url_list += "https://dashboard.cp2k.org/archive/%s/%s.gz\n"%(s, report['url'])
|
||||
# loop over all relevant commits
|
||||
all_url_rows = []
|
||||
all_html_rows = []
|
||||
max_age = max([log.index[sha] for sha in archive_reports.keys()])
|
||||
for commit in log[:max_age]:
|
||||
sha = commit['git-sha']
|
||||
html_row = '<tr>'
|
||||
html_row += commit_cell(sha, log)
|
||||
if(sha in archive_reports):
|
||||
report = archive_reports[sha]
|
||||
html_row += status_cell(report['status'], report['url'])
|
||||
html_row += '<td align="left">%s</td>'%report['summary']
|
||||
url_row = "https://dashboard.cp2k.org/archive/%s/%s.gz\n"%(s, report['url'])
|
||||
else:
|
||||
archive_output += 2*'<td></td>'
|
||||
svn_rev = log_index[r]
|
||||
archive_output += '<td align="left">%s</td>'%svn_rev['author']
|
||||
archive_output += '<td align="left">%s</td>'%svn_rev['msg'].split("\n")[0]
|
||||
archive_output += '</tr>\n\n'
|
||||
html_row += 2*'<td></td>'
|
||||
url_row = ""
|
||||
html_row += '<td align="left">%s</td>'%commit['author']
|
||||
html_row += '<td align="left">%s</td>'%commit['msg']
|
||||
html_row += '</tr>\n\n'
|
||||
all_html_rows.append(html_row)
|
||||
all_url_rows.append(url_row)
|
||||
|
||||
archive_output += '</table>\n'
|
||||
archive_output += other_index_link
|
||||
archive_output += html_footer()
|
||||
write_file(html_out_fn, archive_output.encode("utf8"))
|
||||
write_file(urls_out_fn, url_list)
|
||||
# generate html pages
|
||||
for full_archive in (False, True):
|
||||
if(full_archive):
|
||||
html_out_postfix = "index_full.html"
|
||||
urls_out_postfix = "list_full.txt"
|
||||
other_index_link = '<p>View <a href="index.html">recent archive</a></p>'
|
||||
max_age = None # output all
|
||||
else:
|
||||
html_out_postfix = "index.html"
|
||||
urls_out_postfix = "list_recent.txt"
|
||||
other_index_link = '<p>View <a href="index_full.html">full archive</a></p>'
|
||||
max_age = 100
|
||||
|
||||
# generate archive index
|
||||
output = html_header(title=name)
|
||||
output += '<p>Go back to <a href="../../index.html">main page</a></p>\n'
|
||||
if(info_url):
|
||||
output += '<p>Get <a href="%s">more information</a></p>\n'%info_url
|
||||
if(report_type == "generic"):
|
||||
output += gen_plots(archive_reports, log, outdir+"archive/"+s+"/", full_archive)
|
||||
output += other_index_link
|
||||
output += '<table border="1" cellspacing="3" cellpadding="5">\n'
|
||||
output += '<tr><th>Commit</th><th>Status</th><th>Summary</th><th>Author</th><th>Commit Message</th></tr>\n\n'
|
||||
output += "".join(all_html_rows[:max_age])
|
||||
output += '</table>\n'
|
||||
output += other_index_link
|
||||
output += html_footer()
|
||||
html_out_fn = outdir+"archive/%s/%s"%(s,html_out_postfix)
|
||||
write_file(html_out_fn, output)
|
||||
|
||||
url_list = "".join(all_url_rows[:max_age])
|
||||
urls_out_fn = outdir+"archive/%s/%s"%(s,urls_out_postfix)
|
||||
write_file(urls_out_fn, url_list)
|
||||
|
||||
#===============================================================================
|
||||
def gen_url_list(config, outdir, full_archive):
|
||||
print("Working url list:")
|
||||
urls_out_postfix = "list_full.txt" if full_archive else "list_recent.txt"
|
||||
url_list = ""
|
||||
for s in config.sections():
|
||||
fn = outdir+"archive/%s/%s"%(s,urls_out_postfix)
|
||||
if(not path.exists(fn)):
|
||||
continue
|
||||
url_list += open(fn).read()
|
||||
write_file(outdir+"archive/"+urls_out_postfix, url_list)
|
||||
def gen_url_list(config, outdir):
|
||||
print("Working on url lists.")
|
||||
for postfix in ("list_full.txt", "list_recent.txt"):
|
||||
url_list = ""
|
||||
for s in config.sections():
|
||||
fn = outdir+"archive/%s/%s"%(s, postfix)
|
||||
if(not path.exists(fn)):
|
||||
continue
|
||||
url_list += open(fn).read()
|
||||
write_file(outdir+"archive/" + postfix, url_list)
|
||||
|
||||
#===============================================================================
|
||||
def gen_plots(all_reports, log, outdir, full_archive):
|
||||
def gen_plots(archive_reports, log, outdir, full_archive):
|
||||
|
||||
ordered_shas = [commit['git-sha'] for commit in log]
|
||||
ordered_reports = [archive_reports[sha] for sha in ordered_shas if sha in archive_reports]
|
||||
|
||||
# collect plot data
|
||||
plots = {}
|
||||
for revision in sorted(all_reports.keys()):
|
||||
report = all_reports[revision]
|
||||
for report in ordered_reports:
|
||||
for p in report['plots']:
|
||||
if(p['name'] not in plots.keys()):
|
||||
plots[p['name']] = {'curves':{}}
|
||||
|
|
@ -260,7 +262,8 @@ def gen_plots(all_reports, log, outdir, full_archive):
|
|||
if(pp['name'] not in p['curves'].keys()):
|
||||
p['curves'][pp['name']] = {'x':[], 'y':[], 'yerr':[]}
|
||||
c = p['curves'][pp['name']]
|
||||
c['x'].append(report['revision'])
|
||||
age = log.index[report['git-sha']]
|
||||
c['x'].append(-age)
|
||||
c['y'].append(pp['y'])
|
||||
c['yerr'].append(pp['yerr'])
|
||||
c['label'] = pp['label'] # update label
|
||||
|
|
@ -268,13 +271,13 @@ def gen_plots(all_reports, log, outdir, full_archive):
|
|||
# write raw data
|
||||
tags = sorted([(pname, cname) for pname, p in plots.items() for cname in p['curves'].keys()])
|
||||
if(tags):
|
||||
raw_output = "#%9s"%"revision"
|
||||
raw_output = "# %6s %40s" % ("age", "commit")
|
||||
for pname, cname in tags:
|
||||
raw_output += " %18s %22s"%(pname+"/"+cname,pname+"/"+cname+"_err")
|
||||
raw_output += "\n"
|
||||
for revision in sorted(all_reports.keys(), reverse=True):
|
||||
report = all_reports[revision]
|
||||
raw_output += "%10d"%revision
|
||||
for report in reversed(ordered_reports):
|
||||
age = log.index[report['git-sha']]
|
||||
raw_output += "%8d %40s"%(-age, report['git-sha'])
|
||||
for pname, cname in tags:
|
||||
pp = [pp for pp in report['plotpoints'] if(pp['plot']==pname and pp['name']==cname)]
|
||||
assert(len(pp)<=1)
|
||||
|
|
@ -286,16 +289,20 @@ def gen_plots(all_reports, log, outdir, full_archive):
|
|||
write_file(outdir+"plot_data.txt", raw_output)
|
||||
|
||||
# create png images
|
||||
fig_ext = "_full.png" if(full_archive) else ".png"
|
||||
rev_end = log[0]['num']
|
||||
rev_start = min(all_reports.keys()) if(full_archive) else rev_end-100
|
||||
if (full_archive):
|
||||
fig_ext = "_full.png"
|
||||
max_age = max([log.index[sha] for sha in archive_reports.keys()])
|
||||
else:
|
||||
fig_ext = ".png"
|
||||
max_age = 100
|
||||
|
||||
for pname, p in plots.items():
|
||||
print("Working on plot: "+pname)
|
||||
fig = plt.figure(figsize=(12,4))
|
||||
fig.subplots_adjust(bottom=0.18, left=0.06, right=0.70)
|
||||
fig.suptitle(p['title'], fontsize=14, fontweight='bold', x=0.4)
|
||||
ax = fig.add_subplot(111)
|
||||
ax.set_xlabel('SVN Revision')
|
||||
ax.set_xlabel('Commit Age')
|
||||
ax.set_ylabel(p['ylabel'])
|
||||
markers = itertools.cycle('os>^*')
|
||||
for cname in sorted(p['curves'].keys()):
|
||||
|
|
@ -304,13 +311,13 @@ def gen_plots(all_reports, log, outdir, full_archive):
|
|||
ax.plot(c['x'], c['y'], label=c['label'], linewidth=2) # less crowded
|
||||
else:
|
||||
ax.errorbar(c['x'], c['y'], yerr=c['yerr'], label=c['label'],
|
||||
marker=markers.next(), linewidth=2, markersize=6)
|
||||
ax.set_xlim(rev_start-1, rev_end+1)
|
||||
marker=next(markers), linewidth=2, markersize=6)
|
||||
ax.set_xlim(-max_age-1, 0)
|
||||
ax.xaxis.set_minor_locator(AutoMinorLocator())
|
||||
ax.legend(bbox_to_anchor=(1.01, 1), loc='upper left',
|
||||
numpoints=1, fancybox=True, shadow=True, borderaxespad=0.0)
|
||||
visibles = [[y for x,y in zip(c['x'],c['y']) if x>=rev_start] for c in p['curves'].values()] # visible y-values
|
||||
ymin = min([min(ys) for ys in visibles if ys]) # lowest point from lowest curve
|
||||
visibles = [[y for x,y in zip(c['x'],c['y']) if x>=-max_age] for c in p['curves'].values()] # visible y-values
|
||||
ymin = min([min(ys) for ys in visibles if ys]) # lowest point from lowest curve
|
||||
ymax = max([max(ys) for ys in visibles if ys]) # highest point from highest curve
|
||||
if(full_archive):
|
||||
ax.set_ylim(0.98*ymin, 1.02*ymax)
|
||||
|
|
@ -318,6 +325,7 @@ def gen_plots(all_reports, log, outdir, full_archive):
|
|||
ymax2 = max([min(ys) for ys in visibles if ys]) # lowest point from highest curve
|
||||
ax.set_ylim(0.98*ymin, min(1.02*ymax, 1.3*ymax2)) # protect against outlayers
|
||||
fig.savefig(outdir+pname+fig_ext)
|
||||
plt.close(fig)
|
||||
|
||||
# write html output
|
||||
html_output = ""
|
||||
|
|
@ -326,33 +334,11 @@ def gen_plots(all_reports, log, outdir, full_archive):
|
|||
return(html_output)
|
||||
|
||||
#===============================================================================
|
||||
def retrieve_report(report_url):
|
||||
try:
|
||||
return urlopen(report_url, timeout=5).read()
|
||||
except:
|
||||
print(traceback.print_exc())
|
||||
return None
|
||||
|
||||
#===============================================================================
|
||||
def parse_report(report_txt, report_type):
|
||||
if(report_txt==None):
|
||||
return( {'status':'UNKNOWN', 'summary':'Error while retrieving report.', 'revision':None} )
|
||||
try:
|
||||
if(report_type == "regtest"):
|
||||
return parse_regtest_report(report_txt)
|
||||
elif(report_type == "generic"):
|
||||
return parse_generic_report(report_txt)
|
||||
else:
|
||||
raise(Exception("Unknown report_type"))
|
||||
except:
|
||||
print(traceback.print_exc())
|
||||
return( {'status':'UNKNOWN', 'summary':'Error while parsing report.', 'revision':None} )
|
||||
|
||||
#===============================================================================
|
||||
def send_notification(report, addressbook, last_ok, svn_log, name, s):
|
||||
rev_end = report['revision'] if(report['revision']) else max(svn_log.keys())
|
||||
if(rev_end==last_ok): return # probably a flapping tester
|
||||
authors = set([svn_log[r]['author'] for r in range(last_ok+1, rev_end+1)])
|
||||
def send_notification(report, addressbook, last_ok, log, name, s):
|
||||
idx_end = log.index[report['git-sha']] if(report['git-sha']) else 0
|
||||
idx_last_ok = log.index[last_ok]
|
||||
if(idx_end == idx_last_ok): return # probably a flapping tester
|
||||
authors = set([log[i]['author'] for i in range(idx_end, last_ok)])
|
||||
emails = [addressbook[a] for a in authors]
|
||||
print("Sending email to: "+", ".join(emails))
|
||||
|
||||
|
|
@ -361,7 +347,7 @@ def send_notification(report, addressbook, last_ok, svn_log, name, s):
|
|||
msg_txt += " test name: %s\n"%name
|
||||
msg_txt += " report state: %s\n"%report['status']
|
||||
msg_txt += " report summary: %s\n"%report['summary']
|
||||
msg_txt += " last OK rev: %d\n\n"%last_ok
|
||||
msg_txt += " last OK commit: %d\n\n"%last_ok
|
||||
msg_txt += "For more information visit:\n"
|
||||
msg_txt += " https://dashboard.cp2k.org/archive/%s/index.html \n\n"%s
|
||||
msg_txt += "Sincerely,\n"
|
||||
|
|
@ -377,7 +363,7 @@ def send_notification(report, addressbook, last_ok, svn_log, name, s):
|
|||
smtp_conn.quit()
|
||||
|
||||
#===============================================================================
|
||||
def html_header(title, rev=None):
|
||||
def html_header(title):
|
||||
output = '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">\n'
|
||||
output += '<html><head>\n'
|
||||
output += '<meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n'
|
||||
|
|
@ -437,7 +423,7 @@ def html_header(title, rev=None):
|
|||
output += ' width: 15em;\n'
|
||||
output += '}\n'
|
||||
output += '</style>\n'
|
||||
output += '<title>%s%s</title>\n'%(title, (" (%d)"%rev if rev else ""))
|
||||
output += '<title>%s</title>\n'%title
|
||||
output += '</head><body>\n'
|
||||
output += '<div class="ribbon"><a href="https://cp2k.org/dev:dashboard">Need Help?</a></div>\n'
|
||||
output += '<center><h1>%s</h1></center>\n'%title.upper()
|
||||
|
|
@ -454,19 +440,22 @@ def html_linkbox():
|
|||
return(output)
|
||||
|
||||
#===============================================================================
|
||||
def html_svnbox(log):
|
||||
def html_gitbox(log):
|
||||
now = datetime.utcnow()
|
||||
output = '<div class="sidebox">\n'
|
||||
output += '<h2>Recent Commits</h2>\n'
|
||||
for r in log[0:10]:
|
||||
url = "https://sourceforge.net/p/cp2k/code/%d/"%r['num']
|
||||
msg = r['msg'].split("\n")[0]
|
||||
for commit in log[0:10]:
|
||||
url = "https://github.com/cp2k/cp2k/commit/" + commit['git-sha']
|
||||
msg = commit['msg']
|
||||
if(len(msg) > 27):
|
||||
msg = msg[:26] + "..."
|
||||
output += '<p><a title="%s" href="%s">%s</a><br>\n'%(r['msg'], url, msg)
|
||||
delta = now - r['date']
|
||||
output += '<p><a title="%s" href="%s">%s</a><br>\n'%(commit['msg'], url, msg)
|
||||
delta = now - commit['date']
|
||||
age = delta.days*24.0 + delta.seconds/3600.0
|
||||
output += '<small>%s %s %.1fh ago.</small></p>\n'%(r['num'], r['author'], age)
|
||||
output += '<small>git:' + commit['git-sha'][:7]
|
||||
if 'svn-rev' in commit:
|
||||
output += ' / svn:%d'%commit['svn-rev']
|
||||
output += '<br>\n%s %.1fh ago.</small></p>\n'%(commit['author'], age)
|
||||
output += '</div>\n'
|
||||
return(output)
|
||||
|
||||
|
|
@ -488,38 +477,11 @@ def write_file(fn, content, gz=False):
|
|||
if(old_content == content):
|
||||
print("File did not change: "+fn)
|
||||
return
|
||||
f = gzip.open(fn, 'wb') if(gz) else open(fn, "w")
|
||||
f.write(content)
|
||||
f = gzip.open(fn, 'wb') if(gz) else open(fn, "wb")
|
||||
f.write(content.encode("utf-8"))
|
||||
f.close()
|
||||
print("Wrote: "+fn)
|
||||
|
||||
#===============================================================================
|
||||
def svn_log(limit=None):
|
||||
sys.stdout.write("Fetching svn log... ")
|
||||
sys.stdout.flush()
|
||||
# xml version contains nice UTC timestamp
|
||||
if(limit):
|
||||
cmd = "svn log --limit %d svn://svn.code.sf.net/p/cp2k/code --xml"%limit
|
||||
else:
|
||||
# our server is much faster, but might be 5 minutes old
|
||||
cmd = "svn log https://svn.cp2k.org/cp2k --xml"
|
||||
log_xml = check_output(cmd.split())
|
||||
dom = minidom.parseString(log_xml)
|
||||
revisions = []
|
||||
for entry in dom.getElementsByTagName("logentry"):
|
||||
rev = dict()
|
||||
rev['num'] = int(entry.attributes['revision'].value)
|
||||
rev_date_str = entry.getElementsByTagName('date')[0].firstChild.nodeValue
|
||||
rev['date'] = datetime.strptime(rev_date_str[:19], '%Y-%m-%dT%H:%M:%S')
|
||||
# some revisions lack an author
|
||||
author = entry.getElementsByTagName('author')
|
||||
rev['author'] = author[0].firstChild.nodeValue if (len(author)==1) else ""
|
||||
msg = entry.getElementsByTagName('msg')[0].firstChild
|
||||
rev['msg'] = msg.nodeValue.strip() if(msg) else ""
|
||||
revisions.append(rev)
|
||||
print("done.")
|
||||
return(revisions)
|
||||
|
||||
#===============================================================================
|
||||
def status_cell(status, report_url, uptodate=True):
|
||||
if(status == "OK"):
|
||||
|
|
@ -531,22 +493,71 @@ def status_cell(status, report_url, uptodate=True):
|
|||
return('<td bgcolor="%s"><a href="%s">%s</a></td>'%(bgcolor, report_url, status))
|
||||
|
||||
#===============================================================================
|
||||
def revision_cell(rev, trunk_rev):
|
||||
if(rev == None):
|
||||
def commit_cell(git_sha, log):
|
||||
if(git_sha is None):
|
||||
return('<td>N/A</td>')
|
||||
rev_url = "https://sourceforge.net/p/cp2k/code/%d/"%rev
|
||||
rev_delta = "(%d)"%(rev - trunk_rev) if(trunk_rev) else ""
|
||||
output = '<td align="left"><a href="%s">%s</a> %s</td>'%(rev_url, rev, rev_delta)
|
||||
idx = log.index[git_sha]
|
||||
commit = log[idx]
|
||||
git_url = "https://github.com/cp2k/cp2k/commit/" + git_sha
|
||||
output = '<td align="left"><a href="%s">%s</a>'%(git_url, git_sha[:7])
|
||||
if 'svn-rev' in commit:
|
||||
svn_rev = commit['svn-rev']
|
||||
svn_url = "https://sourceforge.net/p/cp2k/code/%d/"%svn_rev
|
||||
output += ' / <a href="%s">%d</a>'%(svn_url, svn_rev)
|
||||
output += ' (%d)</td>'%(-idx)
|
||||
return(output)
|
||||
|
||||
#===============================================================================
|
||||
def retrieve_report(report_url):
|
||||
try:
|
||||
return urlopen(report_url, timeout=5).read().decode("utf-8")
|
||||
except:
|
||||
print(traceback.print_exc())
|
||||
return None
|
||||
|
||||
#===============================================================================
|
||||
def store_report(report, report_txt, section, outdir):
|
||||
if ('svn-rev' in report):
|
||||
fn = outdir+"archive/%s/rev_%d.txt.gz"%(section, report['svn-rev'])
|
||||
else:
|
||||
fn = outdir+"archive/%s/commit_%s.txt.gz"%(section, report['git-sha'])
|
||||
write_file(fn, report_txt, gz=True)
|
||||
|
||||
#===============================================================================
|
||||
def parse_report(report_txt, report_type, log):
|
||||
if(report_txt is None):
|
||||
return( {'status':'UNKNOWN', 'summary':'Error while retrieving report.', 'git-sha':None} )
|
||||
try:
|
||||
if(report_type == "regtest"):
|
||||
report = parse_regtest_report(report_txt)
|
||||
elif(report_type == "generic"):
|
||||
report = parse_generic_report(report_txt)
|
||||
else:
|
||||
raise(Exception("Unknown report_type"))
|
||||
if ('svn-rev' in report):
|
||||
assert 'git-sha' not in report
|
||||
rev = report['svn-rev']
|
||||
if rev not in log.svn2git:
|
||||
return( {'status':'UNKNOWN', 'summary':'Could not convert svn revision to git commit.', 'git-sha':None} )
|
||||
report['git-sha'] = log.svn2git[rev]
|
||||
return(report)
|
||||
except:
|
||||
print(traceback.print_exc())
|
||||
return( {'status':'UNKNOWN', 'summary':'Error while parsing report.', 'git-sha':None} )
|
||||
|
||||
#===============================================================================
|
||||
def parse_regtest_report(report_txt):
|
||||
m = re.search("svn: .*(Can't connect .*):", report_txt)
|
||||
if(m):
|
||||
return({'revision':None, 'status':'UNKNOWN', 'summary':m.group(1)})
|
||||
return({'status':'UNKNOWN', 'summary':m.group(1)})
|
||||
|
||||
report = dict()
|
||||
report['revision'] = int(re.search("(revision|Revision:) (\d+)\.?\n", report_txt).group(2))
|
||||
m = re.search("(revision|Revision:) (\d+)\.?\n", report_txt)
|
||||
if (m):
|
||||
report['svn-rev'] = int(m.group(2))
|
||||
m = re.search("(commit|Commit:) (\w{40})\n", report_txt)
|
||||
if (m):
|
||||
report['git-sha'] = int(m.group(2))
|
||||
|
||||
if("LOCKFILE" in report_txt):
|
||||
report['status'] = "UNKNOWN"
|
||||
|
|
@ -590,22 +601,22 @@ def parse_regtest_report(report_txt):
|
|||
def parse_generic_report(report_txt):
|
||||
m = re.search("svn: .*(Can't connect .*):", report_txt)
|
||||
if(m):
|
||||
return({'revision':None, 'status':'UNKNOWN', 'summary':m.group(1)})
|
||||
return({'status':'UNKNOWN', 'summary':m.group(1)})
|
||||
report = dict()
|
||||
report['revision'] = int(re.search("(^|\n)Revision: (\d+)\n", report_txt).group(2))
|
||||
|
||||
m = re.search("(^|\n)Revision: (\d+)\n", report_txt)
|
||||
if (m):
|
||||
report['svn-rev'] = int(m.group(2))
|
||||
m = re.search("(^|\n)Commit: (\w{40})\n", report_txt)
|
||||
if (m):
|
||||
report['git-sha'] = int(m.group(2))
|
||||
|
||||
report['summary'] = re.search("(^|\n)Summary: (.+)\n", report_txt).group(2)
|
||||
report['status'] = re.search("(^|\n)Status: (.+)\n", report_txt).group(2)
|
||||
report['plots'] = [eval("dict(%s)"%m[1]) for m in re.findall("(^|\n)Plot: (.+)(?=\n)", report_txt)]
|
||||
report['plotpoints'] = [eval("dict(%s)"%m[1]) for m in re.findall("(^|\n)PlotPoint: (.+)(?=\n)", report_txt)]
|
||||
return(report)
|
||||
|
||||
#===============================================================================
|
||||
def check_output(command, **kwargs):
|
||||
p = subprocess.Popen(command, stdout=subprocess.PIPE, **kwargs)
|
||||
output = p.communicate()[0]
|
||||
assert(p.wait() == 0)
|
||||
return(output)
|
||||
|
||||
#===============================================================================
|
||||
if(len(sys.argv)==2 and sys.argv[-1]=="--selftest"):
|
||||
pass #TODO implement selftest
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue