diff --git a/tools/precommit/Dockerfile b/tools/precommit/Dockerfile new file mode 100644 index 0000000000..1747d3a877 --- /dev/null +++ b/tools/precommit/Dockerfile @@ -0,0 +1,40 @@ +FROM ubuntu:20.04 + +# author: Ole Schuett + +# Install Ubuntu packages. +RUN export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true && \ + apt-get update && apt-get install -y --no-install-recommends \ + clang-format \ + git \ + less \ + python3 \ + python3-pip \ + python3-wheel \ + python3-setuptools \ + rubygems \ + shellcheck \ + vim \ + && rm -rf /var/lib/apt/lists/* + +# Install Markdownlint. +RUN gem install mdl + +# Install Black Python Formatter. +RUN pip3 install black + +# Clone cp2k repository (needed for CI mode). +RUN git clone --quiet --recursive --single-branch -b master https://github.com/cp2k/cp2k.git /workspace/cp2k + +# Install Flask app. +RUN pip3 install flask gunicorn +WORKDIR /opt/cp2k-precommit +COPY precommit_server.py . + +ARG REVISION +ENV REVISION=${REVISION} + +COPY entrypoint.sh . +CMD ["./entrypoint.sh"] + +#EOF diff --git a/tools/precommit/README.md b/tools/precommit/README.md new file mode 100644 index 0000000000..46514b8fc0 --- /dev/null +++ b/tools/precommit/README.md @@ -0,0 +1,41 @@ +# CP2K Precommit + +The precommit system consists of the following tools that analyze and format the source code: + + - [doxify](../doxify/) to add Doxygen templates. + - [prettify](../prettify/) to format Fortran files. + - [analyze_src](../conventions/analyze_src.py) to check copyright banners and a few other things. + - [ast.parse](https://docs.python.org/3/library/ast.html) to check Python syntax. + - [clang-format](https://clang.llvm.org/docs/ClangFormat.html) to format C and Cuda files. + - [black](https://github.com/psf/black) to format Python scripts. + - [shellcheck](https://github.com/koalaman/shellcheck) to analyze Shell scripts. + - [markdownlint](https://github.com/markdownlint/markdownlint) to analyze Markdown files. + +In contrast to the [CP2K-CI](https://github.com/cp2k/cp2k-ci) these tools process each file individually, which makes them much more lightweight. + +## Install Git Hook + +The [precommit.py](./precommit.py) script can be readily installed as git hook: +``` +ln -fs ../../tools/precommit/precommit.py .git/hooks/pre-commit +``` + +## Server + +Many of the tools listed above require more than just Python. To avoid their tedious installation a remote server is used by default. +It is hosted at https://precommit.cp2k.org via [Cloud Run](https://cloud.google.com/run). + +The same server can also be started locally when Docker is available: +``` +./start_local_server.sh +``` +The server can also be installed without Docker by following the steps in the [Dockerfile](./Dockerfile). + +Once the server is up and running it can be used like this: +``` +$ export CP2K_PRECOMMIT_SERVER="http://127.0.0.1:8080" +$ ./precommit.py +Running precommit checks using 8 workers and server: http://127.0.0.1:8080 +Searching for files... +``` + diff --git a/tools/precommit/cloudbuild.yaml b/tools/precommit/cloudbuild.yaml new file mode 100644 index 0000000000..02a18f314a --- /dev/null +++ b/tools/precommit/cloudbuild.yaml @@ -0,0 +1,28 @@ +# author: Ole Schuett + +steps: +- name: 'gcr.io/cloud-builders/docker' + args: ["build", "--build-arg ", "REVISION=$SHORT_SHA", "-t", "img_cp2kprecommit", "."] + +- name: 'gcr.io/cloud-builders/docker' + args: ["tag", "img_cp2kprecommit", "gcr.io/$PROJECT_ID/img_cp2kprecommit:$SHORT_SHA"] + +- name: 'gcr.io/cloud-builders/docker' + args: ["push", "gcr.io/$PROJECT_ID/img_cp2kprecommit:$SHORT_SHA"] + +- name: 'gcr.io/cloud-builders/docker' + args: ["tag", "img_cp2kprecommit", "gcr.io/$PROJECT_ID/img_cp2kprecommit:latest"] + +- name: 'gcr.io/cloud-builders/docker' + args: ["push", "gcr.io/$PROJECT_ID/img_cp2kprecommit:latest"] + +- name: "gcr.io/cloud-builders/gcloud" + args: + - "run" + - "deploy" + - "cp2k-precommit" + - "--platform=managed" + - "--region=us-central1" + - "--image=gcr.io/$PROJECT_ID/img_cp2kprecommit:$SHORT_SHA" + +#EOF diff --git a/tools/precommit/deploy.sh b/tools/precommit/deploy.sh new file mode 100755 index 0000000000..e4dc10c6a5 --- /dev/null +++ b/tools/precommit/deploy.sh @@ -0,0 +1,18 @@ +#!/bin/bash -e + +# author: Ole Schuett + +set -x + +SHORT_SHA=$(git rev-parse --short HEAD) +docker build --build-arg "REVISION=${SHORT_SHA}" -t cp2k-precommit . + +docker tag cp2k-precommit gcr.io/cp2k-org-project/img_cp2kprecommit:${SHORT_SHA} +docker tag cp2k-precommit gcr.io/cp2k-org-project/img_cp2kprecommit:latest +docker push gcr.io/cp2k-org-project/img_cp2kprecommit:${SHORT_SHA} +docker push gcr.io/cp2k-org-project/img_cp2kprecommit:latest + +gcloud run deploy cp2k-precommit --platform=managed --region=us-central1 \ + --image=gcr.io/cp2k-org-project/img_cp2kprecommit:${SHORT_SHA} + +#EOF diff --git a/tools/precommit/entrypoint.sh b/tools/precommit/entrypoint.sh new file mode 100755 index 0000000000..e40a1c65aa --- /dev/null +++ b/tools/precommit/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/bash -e + +# author: Ole Schuett + +trap "exit" INT + +if [ -z "${GIT_REF}" ]; then + # Server mode. + gunicorn --bind=:8080 --workers=1 --threads=8 --timeout=0 precommit_server:app & + wait # Can be interrupted with ctrl-C. +else + # CI mode. + echo -e "\n========== Starting Precommit Server ==========" + gunicorn --bind=:8080 --workers=1 --threads=8 --timeout=0 precommit_server:app &> server.logs & + sleep 3 + cat server.logs + echo -e "\n========== Fetching Git Commit ==========" + cd /workspace/cp2k + git fetch --quiet origin "${GIT_BRANCH}" + git reset --quiet --hard "${GIT_REF}" + git submodule update --init --recursive + git --no-pager log -1 --pretty='%nCommitSHA: %H%nCommitTime: %ci%nCommitAuthor: %an%nCommitSubject: %s%n' + echo -e "\n========== Running Precommit Checks ==========" + export CP2K_PRECOMMIT_SERVER="http://127.0.0.1:8080" + ./tools/precommit/precommit.py --no-cache --progressbar-wait=10 || true +fi + +#EOF diff --git a/tools/precommit/precommit.py b/tools/precommit/precommit.py new file mode 100755 index 0000000000..ac1a70b3e4 --- /dev/null +++ b/tools/precommit/precommit.py @@ -0,0 +1,275 @@ +#!/usr/bin/python3 + +# author: Ole Schuett + +import ast +import argparse +import json +from time import time, sleep +import re +import os +from os import path +import sys +import concurrent.futures +from subprocess import PIPE, STDOUT +import subprocess +import traceback +from urllib.request import urlopen, Request +from urllib.error import HTTPError +import uuid +from difflib import unified_diff + +SCRATCH_DIR = "./obj/precommit" +CACHE_FILE = path.join(SCRATCH_DIR, "cache.json") +SERVER = os.environ.get("CP2K_PRECOMMIT_SERVER", "https://precommit.cp2k.org") + + +# ====================================================================================== +def main(): + # Parse command line arguments. + parser = argparse.ArgumentParser( + description="Check source code for formatting and linter problems." + ) + parser.add_argument( + "-a", + "--no-cache", + action="store_true", + help="ignore the cache and check all files", + ) + parser.add_argument( + "-m", + "--allow-modifications", + action="store_true", + help="allow the tools to modify files", + ) + parser.add_argument( + "-j", + "--num_workers", + type=int, + default=min(32, os.cpu_count() + 4), # copied from ThreadPoolExecutor + help="number of parallel workers", + ) + parser.add_argument( + "--progressbar-wait", + type=int, + default=1, + help="number seconds in between progressbar updates", + ) + args = parser.parse_args() + + # Say hello to the server. + print( + f"Running precommit checks using {args.num_workers} workers and server: {SERVER}" + ) + server_hello = urlopen(Request(SERVER + "/"), timeout=10).read().decode("utf8") + assert server_hello.startswith("cp2k precommit server") + + # Change into base dir and create scratch dir. + base_dir = path.realpath(path.join(path.dirname(__file__), "../../")) + os.chdir(base_dir) + os.makedirs(SCRATCH_DIR, exist_ok=True) + + # Collect eligible files. + sys.stdout.write("Searching for files...\r") + sys.stdout.flush() + filename_pattern = re.compile(r".*\.(F|fypp|c|cu|h|py)$") + file_list = [] + for root, dirs, files in os.walk("."): + if root.startswith("./tools/toolchain/build"): + continue + if root.startswith("./tools/toolchain/install"): + continue + if root.startswith("./tools/prettify/fprettify"): + continue + if root.startswith("./tools/build_utils/fypp"): + continue + if root.startswith("./tools/autotune_grid"): + continue + if root.startswith("./exts"): + continue + if root.startswith("./obj"): + continue + if root.startswith("./lib"): + continue + if root.startswith("./exe"): + continue + if root.startswith("./regtesting"): + continue + if root.startswith("./.git"): + continue + file_list += [path.join(root, fn) for fn in files if filename_pattern.match(fn)] + + # Load cache. + should_load_cache = path.exists(CACHE_FILE) and not args.no_cache + cache = json.load(open(CACHE_FILE)) if should_load_cache else {} + + # Launch async processing of files. + futures = {} + executor = concurrent.futures.ThreadPoolExecutor(max_workers=args.num_workers) + for fn in file_list: + if path.getmtime(fn) != cache.get(fn, -1): + futures[fn] = executor.submit(process_file, fn, args.allow_modifications) + num_skipped = len(file_list) - len(futures) + + # Continuously update progressbar, save cache file, and print errors. + failed_files = set() + while True: + num_done = num_skipped + for fn, f in futures.items(): + if f.done(): + num_done += 1 + if not f.exception(): + cache[fn] = path.getmtime(fn) + elif fn not in failed_files: + failed_files.add(fn) + print_box(fn, str(f.exception())) + json.dump(cache, open(CACHE_FILE, "w")) + progressbar = "=" * int(60 * num_done / len(file_list)) + sys.stdout.write( + f"[{progressbar:60s}] {num_done} / {len(file_list)} files processed\r" + ) + sys.stdout.flush() + if num_done == len(file_list) or len(failed_files) >= 10: + executor.shutdown(wait=False) + break + sleep(args.progressbar_wait) + + # Print final message. + print( + f"Summary: Found {len(file_list)}, " + f"skipped {num_skipped}, " + f"checked {num_done - num_skipped}, " + f"and failed {len(failed_files)} files." + (" " * 50) + ) + print("Status: " + ("FAILED" if failed_files else "OK")) + sys.exit(len(failed_files)) + + +# ====================================================================================== +def print_box(fn, message): + print("+" + "-" * 160 + "+") + print(f"| {fn:^158s} |") + print("+" + "-" * 160 + "+") + for line in message.strip().split("\n"): + print(f"| {line:<158s} |") + print("+" + "-" * 160 + "+\n\n") + + +# ====================================================================================== +def process_file(fn, allow_modifications): + orig_content = open(fn, "rb").read() + + if re.match(r".*\.(F|fypp)$", fn): + run_local_tool("./tools/doxify/doxify.sh", fn) + run_prettify(fn) + run_analyze_src(fn) + + elif re.match(r".*\.(c|cu|cpp|h|hpp)$", fn): + run_remote_tool("clangformat", fn) + run_analyze_src(fn) + + elif re.match(r".*\.py$", fn): + ast.parse(orig_content, filename=fn) + run_remote_tool("black", fn) + run_analyze_src(fn) + + elif re.match(r".*\.sh$", fn): + run_remote_tool("shellcheck", fn) + + elif re.match(r".*\.md$", fn): + run_remote_tool("markdownlint", fn) + + else: + raise Exception("Unknown file extension: " + fn) + + new_content = open(fn, "rb").read() + if new_content == orig_content: + return + + # Deal with a modified file. + if allow_modifications: + bak_fn = path.join(SCRATCH_DIR, f"{path.basename(fn)}_{int(time())}.bak") + open(bak_fn, "wb").write(orig_content) # make a backup copy + else: + open(fn, "wb").write(orig_content) # restore origin content + try: + orig_lines = orig_content.decode("utf8").split("\n") + new_lines = new_content.decode("utf8").split("\n") + diff = unified_diff(orig_lines, new_lines, "before", "after", lineterm="") + except: + diff = [] # + raise Exception(f"File modified:\n" + "\n".join(diff)) + + +# ====================================================================================== +def run_prettify(fn): + if fn in ("./src/base/base_uses.f90", "./src/common/util.F"): + return # Skipping because of prettify bugs. + + # The prettify tool processes only about 1k lines of code per second. + # Hence, setting a generous timeout as our largest file has 100k lines. + run_local_tool( + "./tools/prettify/prettify.py", "--no-report-errors", fn, timeout=300 + ) + + +# ====================================================================================== +def run_analyze_src(fn): + run_local_tool( + "./tools/conventions/analyze_src.py", + "--fail", + "--suppressions", + "./tools/conventions/conventions.supp", + fn, + ) + + +# ====================================================================================== +def run_local_tool(*cmd, timeout=10): + p = subprocess.run(cmd, timeout=timeout, stdout=PIPE, stderr=STDOUT) + if p.returncode != 0: + raise Exception(p.stdout.decode("utf8")) + + +# ====================================================================================== +def run_remote_tool(tool, fn): + url = f"{SERVER}/{tool}" + r = http_post(url, fn) + if r.status == 304: + pass # file not modified + elif r.status == 200: + open(fn, "wb").write(r.read()) + else: + raise Exception(r.read().decode("utf8")) # something went wrong + + +# ====================================================================================== +def http_post(url, fn): + # This would be so much easier with the Requests library where it'd be a one-liner: + # return requests.post(url, files={path.basename(fn): open(fn, "rb").read()}) + + boundary = uuid.uuid1().hex + name = path.basename(fn) + data = b"".join( + [ + f"--{boundary}\r\nContent-Disposition: ".encode("utf8"), + f'form-data; name="{name}"; filename="{name}"\r\n\r\n'.encode("utf8"), + open(fn, "rb").read(), + f"\r\n--{boundary}--\r\n".encode("utf8"), + ] + ) + headers = { + "Content-Length": len(data), + "Content-Type": f"multipart/form-data; boundary={boundary}", + } + try: + return urlopen(Request(url, data=data, headers=headers), timeout=60) + except HTTPError as err: + return err # HTTPError has .status and .read() just like a normal HTTPResponse. + + +# ====================================================================================== +if __name__ == "__main__": + main() + +# EOF diff --git a/tools/precommit/precommit_server.py b/tools/precommit/precommit_server.py new file mode 100755 index 0000000000..0bec5e629c --- /dev/null +++ b/tools/precommit/precommit_server.py @@ -0,0 +1,83 @@ +#!/usr/bin/python3 + +# author: Ole Schuett + +import os +import logging +from os import path +from time import time +import tempfile +import subprocess +from subprocess import PIPE, STDOUT +from flask import Flask, request, abort + +app = Flask(__name__) +app.config["MAX_CONTENT_LENGTH"] = 1024 * 1024 # 1MB +app.logger.setLevel(logging.INFO) +app.logger.info("CP2K Precommit Server is up and running :-)") + +# ====================================================================================== +@app.route("/") +def hello(): + return "cp2k precommit server revision: " + os.environ["REVISION"] + + +# ====================================================================================== +@app.route("/black", methods=["POST"]) +def black(): + return run_tool(["black"]) + + +# ====================================================================================== +@app.route("/shellcheck", methods=["POST"]) +def shellcheck(): + return run_tool(["shellcheck"]) + + +# ====================================================================================== +@app.route("/markdownlint", methods=["POST"]) +def markdownlint(): + return run_tool(["mdl"]) + + +# ====================================================================================== +@app.route("/clangformat", methods=["POST"]) +def clangformat(): + return run_tool(["clang-format", "--style=llvm", "-i"]) + + +# ====================================================================================== +def run_tool(cmd, timeout=30): + assert len(request.files) == 1 + orig_fn = list(request.files.keys())[0] + data_before = request.files[orig_fn].read() + data_kb = len(data_before) / 1024.0 + fn = path.basename(orig_fn) + workdir = tempfile.TemporaryDirectory() + abs_fn = path.join(workdir.name, fn) + open(abs_fn, "wb").write(data_before) + + t1 = time() + try: + p = subprocess.run( + cmd + [fn], cwd=workdir.name, timeout=timeout, stdout=PIPE, stderr=STDOUT + ) + except subprocess.TimeoutExpired: + app.logger.info(f"Timeout of {cmd[0]} on {data_kb:.1f}KB after {timeout}s.") + return f"Timeout while running {cmd[0]} - please try again.", 504 + t2 = time() + app.logger.info(f"Ran {cmd[0]} on {data_kb:.1f}KB in {t2-t1:.1f}s.") + + if p.returncode != 0: + return p.stdout, 422 # Unprocessable Entity + data_after = open(abs_fn, "rb").read() + if data_after == data_before: + return "Not Modified", 304 + return data_after, 200 + + +# ====================================================================================== +if __name__ == "__main__": + app.run() + +# EOF diff --git a/tools/precommit/start_local_server.sh b/tools/precommit/start_local_server.sh new file mode 100755 index 0000000000..19869094bf --- /dev/null +++ b/tools/precommit/start_local_server.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e + +# author: Ole Schuett + +set -x + +SHORT_SHA=$(git rev-parse --short HEAD) +docker build --build-arg "REVISION=${SHORT_SHA}" -t cp2k-precommit . +docker run --rm -p127.0.0.1:8080:8080 cp2k-precommit + +#EOF