diff --git a/tools/dashboard/generate_dashboard.py b/tools/dashboard/generate_dashboard.py index f6a3bd9e33..632a5d3122 100755 --- a/tools/dashboard/generate_dashboard.py +++ b/tools/dashboard/generate_dashboard.py @@ -581,7 +581,6 @@ def html_header(title: str) -> str: def html_linkbox() -> str: output = '
\n' output += "

More...

\n" - output += 'Regtest Survey
\n' output += 'Test Coverage
\n' output += 'Discontinued Tests
\n' output += 'Supported compilers
\n' diff --git a/tools/dashboard/generate_regtest_survey.py b/tools/dashboard/generate_regtest_survey.py deleted file mode 100755 index 1eb417034c..0000000000 --- a/tools/dashboard/generate_regtest_survey.py +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/env python3 - -# author: Ole Schuett - -from typing import Any, Dict, Set, List, Optional, NewType, cast -from urllib.request import urlopen -from datetime import datetime -from glob import glob -import configparser -import dataclasses -import numpy as np -import gzip -import sys -import re - -# ====================================================================================== -Report = NewType("Report", Dict[str, str]) - - -# ====================================================================================== -@dataclasses.dataclass -class TestDef: - test_type: int - flags: List[str] - tolerance: float - ref_value: str - - -# ====================================================================================== -def main() -> None: - if len(sys.argv) != 3: - print("Usage generate_regtest_survey.py ") - sys.exit(1) - - config_fn, outdir = sys.argv[1:] - assert outdir.endswith("/") - - # parse ../../tests/*/*/TEST_FILES - test_defs = parse_test_files() - - # parse ../../tests/TEST_TYPES - test_types = parse_test_types() - - # find eligible testers by parsing latest reports - tester_names: List[str] = list() - tester_values: Dict[str, Report] = dict() - inp_names: Set[str] = set() - config = configparser.ConfigParser() - config.read(config_fn) - - def get_sortkey(s: str) -> int: - return config.getint(s, "sortkey") - - for tname in sorted(config.sections(), key=get_sortkey): - fn = outdir + "archive/%s/list_recent.txt" % tname - list_recent = open(fn, encoding="utf8").readlines() - if not list_recent: - continue # list_recent is empty - latest_report_url = list_recent[0].strip() - report = parse_report(latest_report_url) - if report: - inp_names.update(report.keys()) - tester_values[tname] = report - tester_names.append(tname) - - # remove outdated inp-names - inp_names = inp_names.intersection(test_defs.keys()) - - # html-header - output = '\n' - output += "\n" - output += '\n' - output += '\n' - output += '\n' - output += '\n" - output += '\n" - output += "CP2K Regtest Survey\n" - output += "\n" - output += "

CP2K REGTEST SURVEY

\n" - - # fun facts - output += '
\n' - ntests = len(test_defs) - output += "" - output += '\n' % ntests - n = sum(len(t.flags) != 0 for t in test_defs.values() if t.flags) - pc = n / (0.01 * ntests) - output += "" - output += '\n' % (n, pc) - n = sum(t.test_type != 0 for t in test_defs.values()) - pc = n / (0.01 * ntests) - output += "" - output += '\n' % (n, pc) - n = sum(t.ref_value != "" for t in test_defs.values()) - pc = n / (0.01 * ntests) - output += "" - output += '\n' % (n, pc) - for i in range(14, 5, -1): - tol = float("1.e-%d" % i) - n = sum(t.test_type != 0 and t.tolerance <= tol for t in test_defs.values()) - pc = n / (0.01 * ntests) - output += "" % i - output += '' % (n, pc) - output += "\n" - output += "
Total number of test-cases%d100.0%%
Tests which require flags%d%.1f%%
Numeric tests, ie. type ≠ 0%d%.1f%%
Numeric tests with fixed reference%d%.1f%%
Numeric tests with tolerance ≤ 10-%d%d%.1f%%
\n" - - # table-body - tester_nskipped = dict([(n, 0) for n in tester_names]) - tester_nfailed = dict([(n, 0) for n in tester_names]) - tbody = "" - for inp in sorted(inp_names): - # calculate median and MAD - value_list = list() - for tname in tester_names: - val = tester_values[tname].get(inp, None) - if val: - value_list.append(float(val)) - values: Any = np.array(value_list) - median_iterp = np.median(values) # takes midpoint if len(values)%2==0 - median_idx = (np.abs(values - median_iterp)).argmin() # find closest point - median = values[median_idx] - norm = median + 1.0 * (median == 0.0) - rel_diff = abs((values - median) / norm) - mad = np.amax(rel_diff) # Maximum Absolute Deviation - outlier = list(rel_diff > test_defs[inp].tolerance) - - # output table-row - tbody += '\n' - style = 'bgcolor="#FF9900"' if any(outlier) else "" - tbody += '%s\n' % (style, inp) - ttype_num = test_defs[inp].test_type - ttype_re = test_types[test_defs[inp].test_type].split("!")[0].strip() - tbody += '%s\n' % (ttype_re, ttype_num) - tbody += "%.1e\n" % test_defs[inp].tolerance - tbody += "%.1e\n" % mad - tol_mad = test_defs[inp].tolerance / max(1e-14, mad) - if tol_mad < 1.0: - tbody += '%.1e\n' % tol_mad - elif tol_mad > 10.0: - tbody += '%.1e\n' % tol_mad - else: - tbody += "%.1e\n" % tol_mad - tbody += "%s\n" % test_defs[inp].ref_value - tbody += "%.17g\n" % median - tbody += "%i\n" % np.sum(outlier) - for tname in tester_names: - val = tester_values[tname].get(inp, None) - if not val: - tbody += "\n" - tester_nskipped[tname] += 1 - elif outlier.pop(0): - tbody += '%s' % val - tester_nfailed[tname] += 1 - else: - tbody += "%s" % val - tbody += '%s\n' % (style, inp) - tbody += "\n" - - # table-header - theader = 'NameTypeTolerance' - theader += 'MAD' - theader += "Tol. / MADReferenceMedian" - theader += '#failed' - for tname in tester_names: - theader += '%s' % config.get(tname, "name") - theader += "
#failed: %d" % tester_nfailed[tname] - theader += "
#skipped: %d" % tester_nskipped[tname] - theader += "\n" - theader += "Name" - theader += "\n" - - # assemble table - output += '
Sorting, please wait...
\n' - output += "

Click on table header to sort by column.

\n" - output += '\n' - output += "" + theader + "" - output += "" + theader + "" - output += "" + tbody + "" - output += "
\n" - - # html-footer - now = datetime.utcnow().replace(microsecond=0) - output += "

Page last updated: %s

\n" % now.isoformat() - output += "" - - # write output file - fn = outdir + "regtest_survey.html" - f = open(fn, "w", encoding="utf8") - f.write(output) - f.close() - print("Wrote: " + fn) - - -# ====================================================================================== -def parse_test_files() -> Dict[str, TestDef]: - test_defs = dict() - - tests_root = "../../tests/" - test_dir_lines = open(tests_root + "TEST_DIRS", encoding="utf8").readlines() - for dline in test_dir_lines: - dline = dline.strip() - if len(dline) == 0: - continue - if dline.startswith("#"): - continue - d = dline.split()[0] - flags = dline.split()[1:] # flags requiremented by this test_dir - fn = tests_root + d + "/TEST_FILES" - content = open(fn, encoding="utf8").read() - for line in content.strip().split("\n"): - if not line or line.startswith("#"): - continue - parts = line.split() - name = d + "/" + parts[0] - test_type = int(parts[1]) - if len(parts) == 2: - tolerance = 1.0e-14 # default - ref_value = "" - elif len(parts) == 3: - tolerance = float(parts[2]) - ref_value = "" - elif len(parts) == 4: - tolerance = float(parts[2]) - ref_value = parts[3] # do not parse float - else: - raise (Exception("Found strange line in: " + fn)) - test_defs[name] = TestDef(test_type, flags, tolerance, ref_value) - - return test_defs - - -# ====================================================================================== -def parse_test_types() -> List[str]: - test_types = [""] - lines = open("../../tests/TEST_TYPES", encoding="utf8").readlines() - ntypes = int(lines[0]) - for i in range(1, ntypes + 1): - test_types.append(lines[i]) - return test_types - - -# ====================================================================================== -def parse_report(report_url: str) -> Optional[Report]: - print("Parsing: " + report_url) - data = urlopen(report_url, timeout=5).read() - report_txt = gzip.decompress(data).decode("utf-8", errors="replace") - - if "Keepalive:" not in report_txt: - print("Could not recognize as regtests report - skipping.") - return None - - base_dir = None - curr_dir = None - values: Report = cast(Report, {}) - for line in report_txt.split("\n"): - if line.startswith("Work base dir:"): - base_dir = line.split(":", 1)[1].strip() - elif "/UNIT/" in line: - continue # ignore unit-tests - elif line.startswith(">>>"): - prefix = f">>> {base_dir}/" - assert line.startswith(prefix) - curr_dir = line[len(prefix) :] - elif line.startswith("<<<"): - curr_dir = None - elif curr_dir: - # Found an actual result line. - if "OK" not in line and "WRONG RESULT" not in line: - continue # ignore failed tests - parts = line.split() - if parts[0].rsplit(".", 1)[1] not in ("inp", "restart"): - print("Found strange line:\n" + line) - continue - test_name = curr_dir + "/" + parts[0] - if parts[1] == "-": - continue # test without numeric check - try: - float(parts[1]) # try parsing float... - values[test_name] = parts[1] # ... but pass on the original string - except ValueError: - pass # ignore values which can not be parsed - - return values - - -# ====================================================================================== -main() - -# EOF diff --git a/tools/docker/scripts/test_misc.sh b/tools/docker/scripts/test_misc.sh index f63aa07485..ee5c196f73 100755 --- a/tools/docker/scripts/test_misc.sh +++ b/tools/docker/scripts/test_misc.sh @@ -46,7 +46,6 @@ run_test ./tools/vibronic_spec/main.py ./tools/vibronic_spec/example/example_con run_test mypy --strict ./tools/pao-ml/ run_test mypy --strict ./tools/minimax_tools/minimax_to_fortran_source.py run_test mypy --strict ./tools/dashboard/generate_dashboard.py -run_test mypy --strict ./tools/dashboard/generate_regtest_survey.py run_test mypy --strict ./tools/regtesting/optimize_test_dirs.py run_test mypy --strict ./tools/precommit/precommit.py run_test mypy --strict ./tools/precommit/check_file_properties.py