Move to Python3

.pre-commit-config.yaml

@@ -7,15 +7,6 @@ repos:
     -   id: check-yaml
     -   id: check-added-large-files
     -   id: check-ast
-    -   id: check-ast
-        name: Check python ast (with Python 2.7)
-        language_version: python2.7
-        stages: [manual]
-        files: '^tools/build_utils/.*\.py'
-    -   id: check-ast
-        name: Check python ast (with Python 3)
-        language_version: python3
-        stages: [manual]
 -   repo: https://github.com/ambv/black
     rev: stable
     hooks:

tools/benchmark_plots/plot_benchmark.py

@@ -1,8 +1,9 @@
+#!/usr/bin/env python3
+
+
 """
 Make a plot of CP2K benchmark data.
 """
-from __future__ import absolute_import
-from __future__ import print_function
 import numpy as np
 import matplotlib.pyplot as plt
 import matplotlib.ticker

tools/benchmark_plots/plot_comparison.py

@@ -1,8 +1,9 @@
+#!/usr/bin/env python3
+
+
 """
 Make a plot of CP2K benchmark data.
 """
-from __future__ import absolute_import
-from __future__ import print_function
 import numpy as np
 import matplotlib.pyplot as plt
 import matplotlib.ticker

tools/build_utils/check_archives.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 # author: Ole Schuett
 

tools/build_utils/discover_programs.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 import re, sys, os
 from os import path

tools/build_utils/fypp

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 ################################################################################
 #

tools/build_utils/makedep.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 import re
 import sys

tools/conventions/analyze_gfortran_ast.py

@@ -1,10 +1,7 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 # author: Ole Schuett
 
-from __future__ import print_function
-
 import argparse
 import re
 from os import path

tools/conventions/analyze_gfortran_warnings.py

@@ -1,10 +1,7 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 # author: Ole Schuett
 
-from __future__ import print_function
-
 import argparse
 import re
 import ast

tools/conventions/analyze_src.py

@@ -1,10 +1,7 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 # author: Ole Schuett
 
-from __future__ import print_function
-
 import argparse
 import re
 import sys
@@ -187,19 +184,20 @@ def validate(cp2k_dir, filelist=None, excluded_dirs=DEFAULT_EXCLUDED_DIRS):
                 warnings += ["Text file %s contains DOS linebreaks" % shortfn]
 
             # check for non-ascii chars
-            if b"# -*- coding: utf-8 -*-" in content:
-                continue
+            if re.search(b"[\x80-\xFF]", content):
+                if absfn.endswith(".py"):
+                    continue  # python files are utf8 encoded by default
 
-            if not re.search(b"[\x80-\xFF]", content):
-                continue
+                if b"# -*- coding: utf-8 -*-" in content:
+                    continue
 
-            for lineno, line in enumerate(content.splitlines()):
-                m = re.search(b"[\x80-\xFF]", line)
-                if m:
-                    warnings += [
-                        "Found non-ascii char in %s line %d at position %d"
-                        % (shortfn, lineno + 1, m.start(0) + 1)
-                    ]
+                for lineno, line in enumerate(content.splitlines()):
+                    m = re.search(b"[\x80-\xFF]", line)
+                    if m:
+                        warnings += [
+                            "Found non-ascii char in %s line %d at position %d"
+                            % (shortfn, lineno + 1, m.start(0) + 1)
+                        ]
 
     return warnings
 

tools/conventions/summarize_issues.py

@@ -1,9 +1,7 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 # author: Ole Schuett
 
-from __future__ import print_function
 
 import argparse
 

tools/coverage/test_coverage.py

@@ -1,9 +1,6 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-#
-# author: Ole Schuett
+#!/usr/bin/env python3
 
-from __future__ import print_function
+# author: Ole Schuett
 
 import sys
 import subprocess

tools/dashboard/generate_dashboard.py

@@ -1,5 +1,4 @@
-#!/usr/bin/python3
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 # Generates the CP2K Dashboard html page
 # Inspired by Iain's cp2k_page_update.sh

tools/dashboard/generate_regtest_survey.py

@@ -1,5 +1,4 @@
-#!/usr/bin/python3
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 # author: Ole Schuett
 

tools/diff_cp2k.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Compare CP2K outputs
 # Author: Alfio Lazzaro
@@ -160,7 +160,7 @@ def main():
         metavar="field",
         type=int,
         dest="field",
-        choices=xrange(1, 5),
+        choices=range(1, 5),
         default=2,
         help="which field to show (default is 2)",
     )

tools/docker/scripts/install_python.sh

@@ -7,21 +7,15 @@ apt-get update -qq
 apt-get install -qq --no-install-recommends \
     python                \
     python3               \
-    libpython-stdlib      \
     libpython3-stdlib     \
-    python-pip            \
     python3-pip           \
-    python-wheel          \
     python3-wheel         \
-    python-setuptools     \
     python3-setuptools    \
-    python-dev            \
     python3-dev           \
     build-essential
 rm -rf /var/lib/apt/lists/*
 
 # install python packages
-pip  install --quiet numpy matplotlib requests
 pip3 install --quiet numpy matplotlib requests pre-commit
 
 # register the pre-commit hooks

tools/docker/scripts/test_python.sh

@@ -12,10 +12,6 @@ function run_tests {
     #find ./src/ ./tools/ -name "*.pyc" -exec rm {} \;
     for i in $SCRIPTS ; do
         set +e #disable error trapping
-        if [[ $(head -n1 "$i") =~ "python3" && "$PYTHON" != "python3" ]]; then
-          echo "Skipping $i - it's Python3 only."
-          continue
-        fi
         echo "Running $i"
         cd "$(dirname "$i")"
         SCRIPT=$(basename "$i")
@@ -31,7 +27,7 @@ function run_tests {
 function run_pre_commit {
     set +e #disable error trapping
     if ! pre-commit run --all-files --hook-stage manual check-ast ; then
-        echo "Syntax check failed for either Python 2 or 3"
+        echo "Syntax check failed."
         ERRORS=$((ERRORS+1))
     fi
     set -e #re-enable error trapping
@@ -46,9 +42,6 @@ cd /workspace/cp2k
 ALL_TEST_SCRIPTS=$(find ./src/ ./tools/ -name "*_test.py"  -executable)
 ESSENTIAL_TEST_SCRIPTS=$(find ./tools/build_utils -name "*_test.py"  -executable)
 
-# python 2.7
-run_tests python2.7 "${ALL_TEST_SCRIPTS}"
-
 # python 3.x
 run_tests python3 "${ALL_TEST_SCRIPTS}"
 

tools/doxify/is_fypp.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, re
 

tools/find_bitrot_code.py

@@ -1,64 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
-import sys
-from os.path import basename
-
-threshold = 5
-total_matches = 0
-
-# ===============================================================================
-def main():
-    if len(sys.argv) < 2:
-        print(
-            "Tool for finding large commented regions of code. Uses very simply heuristics."
-        )
-        print("Usage: find_bitrot_code <file1.F> ... <fileN.F>")
-        sys.exit(1)
-
-    files = sys.argv[1:]
-    for fn in files:
-        check(fn)
-
-    print("Found %d spots in %d files." % (total_matches, len(files)))
-
-
-# ===============================================================================
-def check(fn):
-    f = open(fn)
-    all_lines = f.read().split("\n")
-
-    counter = 0
-    start = 0
-
-    def report():
-        global total_matches
-        total_matches += 1
-        print(" +" + ("-" * 87) + "+")
-        msg = "%s: line %d ... line %d" % (basename(fn), start + 1, lineno + 1)
-        print(" | " + msg.ljust(85) + " |")
-        print(" +" + ("-" * 87) + "+")
-        for i in range(start, lineno):
-            print(" | %4d " % (i) + all_lines[i].ljust(80) + " |")
-        print(" +" + ("-" * 87) + "+")
-        print("\n")
-
-    for lineno, line in enumerate(all_lines):
-        s = line.strip().lower()
-
-        if s.startswith("!>") or s.startswith("!$omp"):
-            if counter > threshold:
-                report()
-            counter = 0
-        elif s.startswith("!"):
-            if counter == 0:
-                start = lineno
-            counter += 1
-        elif len(s) > 0:
-            if counter > threshold:
-                report()
-            counter = 0
-
-
-# ===============================================================================
-main()

tools/find_openmp_mistakes.py

@@ -1,59 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
-import sys
-import re
-from os.path import basename
-
-total_matches = 0
-
-# ===============================================================================
-def main():
-    if len(sys.argv) < 2:
-        print("Tool for finding common mistakes with OMP pragmas.")
-        print("Hint: It expects DEFAULT(NONE) in the first line of a PARALLEL pragma.")
-        print("Usage: find_openmp_mistakes.py <file1.F> ... <fileN.F>")
-        sys.exit(1)
-
-    files = sys.argv[1:]
-    for fn in files:
-        check(fn)
-
-    print("Found %d spots in %d files." % (total_matches, len(files)))
-
-
-# ===============================================================================
-def check(fn):
-    global total_matches
-
-    f = open(fn)
-    all_lines = f.read().split("\n")
-
-    for lineno, line in enumerate(all_lines):
-        m = re.search("^\s*!(.*)OMP\s", line, re.IGNORECASE)
-        if m and m.group(1) != "$":
-            total_matches += 1
-            print(
-                'Found strange OMP stuff "%s" in %s:%d'
-                % (m.group(0), basename(fn), lineno + 1)
-            )
-
-        m = re.search("!\$OMP\s+CRITICAL\s*$", line, re.IGNORECASE)
-        if m:
-            total_matches += 1
-            print("Found unnamed OMP CRITICAL in %s:%d" % (basename(fn), lineno + 1))
-
-        m = re.search("!\$OMP\s+PARALLEL\s+(.*)$", line, re.IGNORECASE)
-        if m:
-            m2 = re.search("default\s*\(none\)", m.group(1), re.IGNORECASE)
-            if not m2:
-                total_matches += 1
-                print(
-                    "Found OMP PARALLEL without DEFAULT(NONE) %s:%d"
-                    % (basename(fn), lineno + 1)
-                )
-                # print line
-
-
-# ===============================================================================
-main()

tools/fix_unused_public.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 # author: Ole Schuett
 

tools/logo/gen_cp2k_logo.py

@@ -1,5 +1,4 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 # Generates the CP2K Logo
 #

tools/manual/generate_sitemap.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 import os
 import sys

tools/maple2f90/maple2f90.py

@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/usr/bin/env python3
 import re, sys
 
 """Script to help the conversion of maple generated fortran code to f90"""

tools/minimax_tools/minimax_to_fortran_source.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import re
 import sys

tools/package_planner/plan_packages.py

@@ -1,10 +1,8 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# A tool to help planning CP2K packages, listing currently violated dependencies if any
-#
+#!/usr/bin/env python3
 
-from __future__ import print_function
+# author: Ole Schuett
+
+# A tool to help planning CP2K packages, listing currently violated dependencies if any
 
 import re, sys, os
 from os import path
@@ -52,7 +50,7 @@ def main():
     mod2fn = dict()
     for fn in src_files:
         for m in parsed_files[fn]["module"]:
-            if mod2fn.has_key(m):
+            if m in mod2fn:
                 error('Multiple declarations of module "%s"' % m)
             mod2fn[m] = fn
     print("Created mod2fn table, found %d modules." % len(mod2fn))
@@ -77,15 +75,15 @@ def main():
     # update with manifest with planned packages
     for pp in planned_pkgs:
         p = abspath(path.join(srcdir, pp["dirname"]))
-        if not packages.has_key(p):
+        if p not in packages:
             packages[p] = {"problems": []}
         packages[p].update(pp)
-        if pp.has_key("files"):
+        if "files" in pp:
             for fn in pp["files"]:
                 fn2pkg[fn] = p
-        if pp.has_key("requires+"):
+        if "requires+" in pp:
             packages[p]["requires"] += pp["requires+"]
-        if pp.has_key("requires-"):
+        if "requires-" in pp:
             for i in pp["requires-"]:
                 while i in packages[p]["requires"]:
                     packages[p]["requires"].remove(i)
@@ -101,9 +99,7 @@ def main():
     for fn in src_files:
         p = fn2pkg[basename(fn)]
         deps = collect_include_deps(parsed_files, fn)
-        deps += [
-            mod2fn[m] for m in collect_use_deps(parsed_files, fn) if mod2fn.has_key(m)
-        ]
+        deps += [mod2fn[m] for m in collect_use_deps(parsed_files, fn) if m in mod2fn]
         n_deps += len(deps)
         for d in deps:
             dp = fn2pkg[basename(d)]
@@ -114,7 +110,7 @@ def main():
             )
             if dp not in packages[p]["allowed_deps"]:
                 packages[p]["problems"].append(msg + "(requirement not listed)")
-            if dp != p and packages[dp].has_key("public"):
+            if dp != p and "public" in packages[dp]:
                 if basename(d) not in packages[dp]["public"]:
                     packages[p]["problems"].append(msg + "(file not public)")
 
@@ -170,7 +166,7 @@ def parse_file(parsed_files, fn):
 
 # =============================================================================
 def read_manifest(packages, p):
-    if packages.has_key(p):
+    if p in packages:
         return
 
     fn = p + "/PACKAGE"
@@ -188,7 +184,7 @@ def read_manifest(packages, p):
 
 # =============================================================================
 def process_manifest(packages, p):
-    if not packages[p].has_key("archive"):
+    if "archive" not in packages[p]:
         packages[p]["archive"] = "libcp2k" + basename(p)
     packages[p]["allowed_deps"] = [normpath(p)]
     packages[p]["allowed_deps"] += [
@@ -197,7 +193,7 @@ def process_manifest(packages, p):
 
     for r in packages[p]["requires"]:
         rp = normpath(path.join(p, r))
-        if not packages.has_key(rp):
+        if rp not in packages:
             error(
                 "Unexpected package requirement: "
                 + r
@@ -224,7 +220,7 @@ def collect_include_deps(parsed_files, fn):
 
     for i in pf["include"]:
         fn_inc = normpath(path.join(dirname(fn), i))
-        if parsed_files.has_key(fn_inc):
+        if fn_inc in parsed_files:
             incs.append(fn_inc)
             incs += collect_include_deps(parsed_files, fn_inc)
 
@@ -238,7 +234,7 @@ def collect_use_deps(parsed_files, fn):
 
     for i in pf["include"]:
         fn_inc = normpath(path.join(dirname(fn), i))
-        if parsed_files.has_key(fn_inc):
+        if fn_inc in parsed_files:
             uses += collect_use_deps(parsed_files, fn_inc)
 
     return list(set(uses))
@@ -247,7 +243,7 @@ def collect_use_deps(parsed_files, fn):
 # =============================================================================
 def find_cycles(parsed_files, mod2fn, fn, S=None):
     pf = parsed_files[fn]
-    if pf.has_key("visited"):
+    if "visited" in pf:
         return
 
     if S == None:
@@ -260,7 +256,7 @@ def find_cycles(parsed_files, mod2fn, fn, S=None):
         S.append(m)
 
     for m in collect_use_deps(parsed_files, fn):
-        if mod2fn.has_key(m):
+        if m in mod2fn:
             find_cycles(parsed_files, mod2fn, mod2fn[m], S)
 
     for m in pf["module"]:

tools/plot_mem_from_trace.py

@@ -1,45 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
-import sys
-import re
-import matplotlib.pyplot as plt
-
-# ===============================================================================
-def main():
-    if len(sys.argv) != 2:
-        print("Usage: plot_mem_from_trace.py <trace.out>")
-        sys.exit(1)
-
-    re_mem = re.compile("Hostmem: (\d+) MB GPUmem: (\d+) MB")
-    trace_fn = sys.argv[1]
-    f = open(trace_fn)
-    hostmem_stats = []
-    gpumem_stats = []
-    for line in f.readlines():
-        m = re_mem.search(line)
-        if not m:
-            continue
-        hostmem_stats.append(m.group(1))
-        gpumem_stats.append(m.group(2))
-
-    fig = plt.figure()
-    ax1 = fig.add_subplot(111)
-
-    ax1.plot(hostmem_stats, color="red", label="Host")
-    ax1.set_ylabel("Host Memory Usage [MB]")
-    ax1.set_xlabel("time")
-
-    ax2 = ax1.twinx()
-    ax2.plot(gpumem_stats, color="green", label="GPU")
-    ax2.set_ylabel("GPU Memory Usage [MB]")
-
-    handles1, labels1 = ax1.get_legend_handles_labels()
-    handles2, labels2 = ax2.get_legend_handles_labels()
-    plt.legend(handles1 + handles2, labels1 + labels2)
-    plt.show()
-
-
-# ===============================================================================
-main()
-# EOF

tools/regtesting/check_inputs.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # author: Ole Schuett
 

tools/regtesting/prettify_TEST_FILES.py

@@ -1,5 +1,4 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 # author: Ole Schuett
 

tools/regtesting/test_scaling.py

@@ -1,5 +1,4 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 # author: Ole Schuett
 

tools/regtesting/timings.py

@@ -1,9 +1,7 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 
 # author: Ole Schuett
 
-from __future__ import print_function
 import sys
 import math
 

tools/toolchain/scripts/parse_if.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-# -*- coding: utf8 -*-
+#!/usr/bin/env python3
 
 
 import sys