From 43a662790fb0bf009ea9d75e40b3d1276cb34c9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ole=20Sch=C3=BCtt?= <ole.schuett@cp2k.org>
Date: Thu, 18 Feb 2021 12:47:34 +0100
Subject: [PATCH] Docker: Add performance test

---
 tools/docker/Dockerfile.test_performance    | 21 ++++++++
 tools/docker/run_test.sh                    |  2 +-
 tools/docker/run_test_master.sh             |  2 +-
 tools/docker/scripts/install_performance.sh | 23 ++++++++
 tools/docker/scripts/plot_performance.py    | 60 +++++++++++++++++++++
 tools/docker/scripts/test_performance.sh    | 59 ++++++++++++++++++++
 6 files changed, 165 insertions(+), 2 deletions(-)
 create mode 100644 tools/docker/Dockerfile.test_performance
 create mode 100755 tools/docker/scripts/install_performance.sh
 create mode 100755 tools/docker/scripts/plot_performance.py
 create mode 100755 tools/docker/scripts/test_performance.sh

diff --git a/tools/docker/Dockerfile.test_performance b/tools/docker/Dockerfile.test_performance
new file mode 100644
index 0000000000..848ce7b168
--- /dev/null
+++ b/tools/docker/Dockerfile.test_performance
@@ -0,0 +1,21 @@
+ARG TOOLCHAIN=cp2k/toolchain:latest
+FROM ${TOOLCHAIN}
+
+# author: Ole Schuett
+
+WORKDIR /workspace
+
+COPY ./scripts/install_basics.sh .
+RUN ./install_basics.sh
+
+COPY ./scripts/install_performance.sh .
+RUN ./install_performance.sh
+
+COPY ./scripts/ci_entrypoint.sh \
+     ./scripts/test_performance.sh  \
+     ./scripts/plot_performance.py  \
+     ./
+
+CMD ["./ci_entrypoint.sh", "./test_performance.sh"]
+
+#EOF
diff --git a/tools/docker/run_test.sh b/tools/docker/run_test.sh
index cb06067a9a..a42ae9c53a 100755
--- a/tools/docker/run_test.sh
+++ b/tools/docker/run_test.sh
@@ -17,7 +17,7 @@ CP2K_LOCAL=$(realpath ../../)
 set -x
 
 # SYS_PTRACE needed by LeakSanitizer.
-${DOCKER:-docker} run -i --init --rm --cap-add=SYS_PTRACE \
+${DOCKER:-docker} run -i --init --rm --cap-add=SYS_PTRACE --shm-size=1g \
   --volume "${CP2K_LOCAL}:/mnt/cp2k/:ro" \
   "$@" "img_cp2k_test_${TESTNAME}"
 
diff --git a/tools/docker/run_test_master.sh b/tools/docker/run_test_master.sh
index 60851b019d..03540639c5 100755
--- a/tools/docker/run_test_master.sh
+++ b/tools/docker/run_test_master.sh
@@ -19,7 +19,7 @@ echo "Running ${TESTNAME} on Branch ${GIT_BRANCH} (ref: ${GIT_REF})..."
 set -x
 
 # SYS_PTRACE needed by LeakSanitizer.
-${DOCKER:-docker} run -i --init --rm --cap-add=SYS_PTRACE \
+${DOCKER:-docker} run -i --init --rm --cap-add=SYS_PTRACE --shm-size=1g \
   -e "GIT_BRANCH=${GIT_BRANCH}" -e "GIT_REF=${GIT_REF}" \
   "$@" "img_cp2k_test_${TESTNAME}"
 
diff --git a/tools/docker/scripts/install_performance.sh b/tools/docker/scripts/install_performance.sh
new file mode 100755
index 0000000000..834b08135f
--- /dev/null
+++ b/tools/docker/scripts/install_performance.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -e
+
+# author: Ole Schuett
+
+# setup arch files
+cd /workspace/cp2k/arch
+ln -vs /opt/cp2k-toolchain/install/arch/local* .
+
+# shellcheck disable=SC1091
+source /opt/cp2k-toolchain/install/setup
+
+# pre-build cp2k
+cd /workspace/cp2k
+echo -n "Warming cache by trying to compile cp2k... "
+if make -j VERSION="psmp" &> /dev/null; then
+  echo 'done.'
+else
+  echo 'failed.'
+fi
+
+rm -rf lib exe
+
+#EOF
diff --git a/tools/docker/scripts/plot_performance.py b/tools/docker/scripts/plot_performance.py
new file mode 100755
index 0000000000..896ee0267b
--- /dev/null
+++ b/tools/docker/scripts/plot_performance.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+# author: Ole Schuett
+
+import re
+import sys
+from collections import OrderedDict
+
+# ======================================================================================
+def main():
+    if len(sys.argv) != 5:
+        print("Usage: plot_performance.py <title> <label> <output_omp> <output_mpi>")
+        sys.exit(1)
+    title, label, output_omp, output_mpi = sys.argv[1:]
+
+    timings_omp = parse_timings(output_omp)
+    timings_mpi = parse_timings(output_mpi)
+    routines = list(timings_omp.keys())[:6]
+
+    full_title = f"Timings of {title} with OpenMP"
+    plot = f"{label}_timings"
+    print(f'Plot: name="{plot}", title="{full_title}", ylabel="time [s]"')
+    for r in routines:
+        t = timings_omp[r]
+        print(f'PlotPoint: plot="{plot}", name="{r}", label="{r}", y={t}, yerr=0.0')
+    print("")
+
+    full_title = f"Parallel Efficiency of {title}"
+    plot = f"{label}_scaling"
+    print(f'Plot: name="{plot}", title="{full_title}", ylabel="MPI / OMP time"')
+    for r in routines:
+        e = timings_mpi[r] / timings_omp[r]
+        print(f'PlotPoint: plot="{plot}", name="{r}", label="{r}", y={e}, yerr=0.0')
+
+
+# ======================================================================================
+def parse_timings(out_fn):
+    output = open(out_fn, encoding="utf8").read()
+
+    pattern = r"\n( -+\n - +-\n - +T I M I N G +-\n([^\n]*\n){4}.*? -+)\n"
+    report_lines = re.search(pattern, output, re.DOTALL).group(1).split("\n")[7:-1]
+
+    # Extract average self time.
+    timings = {}
+    for line in report_lines:
+        parts = line.split()
+        timings[parts[0]] = float(parts[3])
+
+    # Add total runtime.
+    assert report_lines[0].split()[0] == "CP2K"
+    timings["total"] = float(report_lines[0].split()[5])
+
+    # Sort by time, longest first.
+    return OrderedDict(sorted(timings.items(), reverse=True, key=lambda kv: kv[1]))
+
+
+# ======================================================================================
+main()
+
+# EOF
diff --git a/tools/docker/scripts/test_performance.sh b/tools/docker/scripts/test_performance.sh
new file mode 100755
index 0000000000..626dece6b4
--- /dev/null
+++ b/tools/docker/scripts/test_performance.sh
@@ -0,0 +1,59 @@
+#!/bin/bash -e
+
+# author: Ole Schuett
+
+function run_benchmark {
+  set +e #disable error trapping
+  OMP_THREADS=$1
+  MPI_RANKS=$2
+  INPUT=$3
+  OUTPUT=$4
+  echo -n "Running ${INPUT} with ${OMP_THREADS} threads and ${MPI_RANKS} ranks... "
+  if OMP_NUM_THREADS="${OMP_THREADS}" mpiexec -np "${MPI_RANKS}" \
+    /workspace/cp2k/exe/local/cp2k.psmp "${INPUT}" &> "${OUTPUT}"; then
+    echo "done."
+  else
+    echo -e "failed.\n\n"
+    tail -n 100 "${OUTPUT}"
+    echo -e "\nSummary: Running ${INPUT} failed."
+    echo -e "Status: FAILED\n"
+    exit 0
+  fi
+  set -e #re-enable error trapping
+}
+
+# shellcheck disable=SC1091
+source /opt/cp2k-toolchain/install/setup
+
+echo -e '\n========== Compiling CP2K =========='
+cd /workspace/cp2k
+echo -n "Compiling cp2k... "
+if make -j VERSION="psmp" &> make.out; then
+  echo "done."
+else
+  echo -e "failed.\n\n"
+  tail -n 100 make.out
+  echo -e "\nSummary: Compilation failed."
+  echo -e "Status: FAILED\n"
+  exit 0
+fi
+
+echo -e '\n========== Running Performance Test =========='
+mkdir -p /workspace/artifacts
+cd ./benchmarks/QS
+
+for INPUT in "H2O-64.inp" "H2O-64_nonortho.inp"; do
+  LABEL="${INPUT%.*}"
+  OUTPUT_MPI="/workspace/artifacts/${LABEL}_32mpi.out"
+  OUTPUT_OMP="/workspace/artifacts/${LABEL}_32omp.out"
+  run_benchmark 1 32 "${INPUT}" "${OUTPUT_MPI}"
+  run_benchmark 32 1 "${INPUT}" "${OUTPUT_OMP}"
+  echo ""
+  /workspace/plot_performance.py "${LABEL}" "${LABEL}" "${OUTPUT_OMP}" "${OUTPUT_MPI}"
+  echo ""
+done
+
+echo -e "\nSummary: Performance test works fine."
+echo -e "Status: OK\n"
+
+#EOF