infra: speedup test by precomputing during docker build

svn-origin-rev: 18385

infra/docker/Dockerfile.psmp

@@ -1,5 +1,6 @@
 FROM cp2k/toolchain:latest
 
+# author: Ole Schuett
 # Has to be build from cp2k root directory:
 # docker build -f ./infra/docker/Dockerfile.psmp .
 

infra/run_test.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/bin/bash
 
 # author: Ole Schuett
 
@@ -8,15 +8,16 @@ if [ "$#" -ne "1" ]; then
     exit 1
 fi
 
+set -e
 TESTNAME=$1
 echo "Running ${TESTNAME} ..."
-CP2K_LOCAL=`realpath ../`
+CP2K_LOCAL=`realpath ../../`
 
 date --utc
-svn info || true
+echo "Commit:" `git rev-parse HEAD`
 
 set -x
 docker build -t img_cp2k_test_${TESTNAME} ./test_${TESTNAME}/
-docker run -i --init --volume ${CP2K_LOCAL}:/opt/cp2k_local/ img_cp2k_test_${TESTNAME}
+docker run -i --init --volume ${CP2K_LOCAL}:/opt/cp2k-local/:ro img_cp2k_test_${TESTNAME}
 
 #EOF

infra/test_aiida/Dockerfile

@@ -1,4 +1,6 @@
-FROM cp2k/cp2k:trunk
+FROM cp2k/toolchain:latest
+
+# author: Ole Schuett
 
 # install Debian packages
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -13,28 +15,32 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     sudo                  \
     ssh                   \
     git                   \
+    rsync                 \
   && rm -rf /var/lib/apt/lists/*
 
-# install python dependencies early to leverage docker build cache
-RUN pip install flake8 aiida ase
+# download and compile cp2k snapshot
+WORKDIR /opt/
+RUN wget -O cp2k-master.zip https://github.com/cp2k/cp2k/archive/master.zip && \
+    unzip cp2k-master.zip  && \
+    rm cp2k-master.zip
 
-# install aiida-cp2k
-WORKDIR /opt/aiida-cp2k/
-RUN git clone https://github.com/cp2k/aiida-cp2k.git .
-RUN pip install .
+WORKDIR /opt/cp2k-master/cp2k/arch
+RUN ln -vs /opt/cp2k-toolchain/install/arch/local* .
+
+WORKDIR /opt/cp2k-master/cp2k/makefiles
+RUN source /opt/cp2k-toolchain/install/setup  && \
+    make -j VERSION=pdbg cp2k                 && \
+    rm -rf ../lib/ ../exe/
+
+# install python packages
+RUN pip install flake8 aiida ase
 
 # create ubuntu user with sudo powers
 RUN adduser --disabled-password --gecos "" ubuntu               && \
     echo "ubuntu ALL=(ALL) NOPASSWD: ALL" >>  /etc/sudoers
 
-
-# configure aiida
-RUN ln -s /opt/cp2k/exe/local/cp2k.psmp /usr/bin/cp2k
-USER ubuntu
-WORKDIR /opt/aiida-cp2k/test/
-RUN ./configure_aiida.sh
-
+WORKDIR /opt/cp2k_test_aiida
 COPY cmd.sh .
-CMD ["./cmd.sh"]
+CMD ["/opt/cp2k_test_aiida/cmd.sh"]
 
 #EOF

infra/test_aiida/cmd.sh

@@ -1,10 +1,45 @@
 #!/bin/bash
 
-echo "========== running AiiDA-CP2K tests =========="
+# author: Ole Schuett
 
+set -e
+
+echo -e "\n========== Rsyncing =========="
+rsync --exclude="*~"          \
+      --exclude=".*/"         \
+      --exclude="*.pyc"       \
+      --exclude=/cp2k/obj/    \
+      --exclude=/cp2k/lib/    \
+      --exclude=/cp2k/exe/    \
+      --ignore-times          \
+      --update                \
+      --verbose               \
+      --recursive             \
+      --checksum              \
+      /opt/cp2k-local/  /opt/cp2k-master/
+
+echo -e "\n========== Compiling CP2K =========="
+source /opt/cp2k-toolchain/install/setup
+cd /opt/cp2k-master/cp2k/makefiles
+make -j VERSION=pdbg cp2k
+ln -s /opt/cp2k-master/cp2k/exe/local/cp2k.pdbg /usr/bin/cp2k
+
+echo -e "\n========== Installing AiiDA-CP2K plugin =========="
+cd /opt/
+git clone https://github.com/cp2k/aiida-cp2k.git
+pip install ./aiida-cp2k/
+
+echo -e "\n========== Configuring AiiDA =========="
+SUDO="sudo -u ubuntu -H"
+cd /opt/aiida-cp2k/test/
+$SUDO ./configure_aiida.sh
+
+echo -e "\n========== Running AiiDA-CP2K tests =========="
+cd  /opt/aiida-cp2k/test/
+set +e
 (
-set -e # abort if error is encountered
-./run_tests.sh
+set -e # abort on error
+$SUDO ./run_tests.sh
 )
 
 EXIT_CODE=$?

infra/test_ase/Dockerfile

@@ -1,21 +1,38 @@
-FROM cp2k/cp2k:trunk
+FROM cp2k/toolchain:latest
 
+# author: Ole Schuett
+
+# install Debian packages
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    git                   \
-    python-dev            \
-    python-pip            \
-    python-wheel          \
-    python-setuptools     \
-    build-essential       \
-&& rm -rf /var/lib/apt/lists/*
+    python3                                                      \
+    python3-dev                                                  \
+    python3-pip                                                  \
+    python3-wheel                                                \
+    python3-setuptools                                           \
+    build-essential                                              \
+    rsync                                                        \
+    git                                                          \
+  && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /opt/ase
-RUN git clone https://gitlab.com/ase/ase.git .
-RUN pip install .
+# download and compile cp2k snapshot
+WORKDIR /opt/
+RUN wget -O cp2k-master.zip https://github.com/cp2k/cp2k/archive/master.zip && \
+    unzip cp2k-master.zip  && \
+    rm cp2k-master.zip
 
-#RUN pip install --upgrade git+https://gitlab.com/ase/ase.git@master
+WORKDIR /opt/cp2k-master/cp2k/arch
+RUN ln -vs /opt/cp2k-toolchain/install/arch/local* .
 
+WORKDIR /opt/cp2k-master/cp2k/makefiles
+RUN source /opt/cp2k-toolchain/install/setup  && \
+    make -j VERSION=pdbg cp2k_shell           && \
+    rm -rf ../lib/ ../exe/
+
+# install python packages
+RUN pip3 install numpy scipy matplotlib flask
+
+WORKDIR /opt/cp2k_test_ase
 COPY cmd.sh .
-CMD ["./cmd.sh"]
+CMD ["/opt/cp2k_test_ase/cmd.sh"]
 
 #EOF

infra/test_ase/cmd.sh

@@ -1,15 +1,44 @@
 #!/bin/bash
 
-echo "========== running ASE tests =========="
-export OMP_NUM_THREADS=1
-export ASE_CP2K_COMMAND="mpiexec -np 2 cp2k_shell.psmp"
+# author: Ole Schuett
+
+set -e
+
+echo -e "\n========== Rsyncing =========="
+rsync --exclude="*~"          \
+      --exclude=".*/"         \
+      --exclude="*.pyc"       \
+      --exclude=/cp2k/obj/    \
+      --exclude=/cp2k/lib/    \
+      --exclude=/cp2k/exe/    \
+      --ignore-times          \
+      --update                \
+      --verbose               \
+      --recursive             \
+      --checksum              \
+      /opt/cp2k-local/  /opt/cp2k-master/
+
+echo -e "\n========== Compiling CP2K =========="
+source /opt/cp2k-toolchain/install/setup
+cd /opt/cp2k-master/cp2k/makefiles
+make -j VERSION=pdbg cp2k_shell
+ln -s /opt/cp2k-master/cp2k/exe/local/cp2k.pdbg /usr/bin/cp2k
+
+echo -e "\n========== Installing ASE =========="
+cd /opt/
+git clone https://gitlab.com/ase/ase.git
+pip3 install ./ase/
+
+echo -e "\n========== Running ASE tests =========="
+cd /opt/ase/
+export ASE_CP2K_COMMAND="mpiexec -np 2 /opt/cp2k-master/cp2k/exe/local/cp2k_shell.pdbg"
 
 (
 set -e # abort if error is encountered
 for i in ./ase/test/cp2k/cp2k_*.py
 do
   echo "Running $i ..."
-  python $i
+  python3 $i
 done
 )
 

infra/test_conventions/Dockerfile

@@ -1,17 +1,29 @@
 FROM cp2k/toolchain:latest
 
-# Has to be build from cp2k root directory:
-# docker build -f ./infra/test_conventions/Dockerfile .
+# author: Ole Schuett
 
-WORKDIR /opt/cp2k/
-COPY makefiles ./makefiles
-COPY tools ./tools
-COPY arch ./arch
-COPY src ./src
-RUN ln -s /opt/cp2k-toolchain/install/arch/* ./arch/
+# download and compile cp2k snapshot
+WORKDIR /opt/
+RUN wget -O cp2k-master.zip https://github.com/cp2k/cp2k/archive/master.zip && \
+    unzip cp2k-master.zip  && \
+    rm cp2k-master.zip
 
-RUN ln -sf bash /bin/sh
-WORKDIR ./tools/conventions
-CMD source /opt/cp2k-toolchain/install/setup && ./test_conventions.sh
+WORKDIR /opt/cp2k-master/cp2k/arch
+RUN ln -vs /opt/cp2k-toolchain/install/arch/local* .
+
+#TODO remove redirection into  make_conventions[12].out
+WORKDIR /opt/cp2k-master/cp2k/tools/conventions/
+RUN source /opt/cp2k-toolchain/install/setup            && \
+    ./test_conventions.sh                               && \
+    rm -rf ../lib/ ../exe/
+
+# install Debian packages
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    rsync                                                        \
+  && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt/cp2k_test_conventions
+COPY cmd.sh .
+CMD ["/opt/cp2k_test_conventions/cmd.sh"]
 
 #EOF

infra/test_conventions/cmd.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# author: Ole Schuett
+
+set -e
+
+echo -e "\n========== Rsyncing =========="
+rsync --exclude="*~"          \
+      --exclude=".*/"         \
+      --exclude="*.pyc"       \
+      --exclude=/cp2k/obj/    \
+      --exclude=/cp2k/lib/    \
+      --exclude=/cp2k/exe/    \
+      --ignore-times          \
+      --update                \
+      --verbose               \
+      --recursive             \
+      --checksum              \
+      /opt/cp2k-local/  /opt/cp2k-master/
+
+echo -e "\n========== Running Conventions test =========="
+source /opt/cp2k-toolchain/install/setup
+cd /opt/cp2k-master/cp2k/tools/conventions/
+#TODO port to Python3
+./test_conventions.sh
+
+#EOF

infra/test_formatting/Dockerfile

@@ -0,0 +1,30 @@
+FROM ubuntu:rolling
+
+# author: Ole Schuett
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    python                \
+    ca-certificates       \
+    wget                  \
+    unzip                 \
+    less                  \
+    make                  \
+    rsync                 \
+    libfindbin-libs-perl  \
+&& rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt/
+RUN wget -O cp2k-master.zip https://github.com/cp2k/cp2k/archive/master.zip && \
+    unzip cp2k-master.zip  && \
+    rm cp2k-master.zip
+
+WORKDIR /opt/cp2k-master/cp2k
+#TODO change test_formatting.sh
+RUN sed -i s/svn/#svn/g ./tools/formatting/test_formatting.sh
+RUN ./tools/formatting/test_formatting.sh
+
+WORKDIR /opt/cp2k_test_formatting
+COPY cmd.sh .
+CMD ["/opt/cp2k_test_formatting/cmd.sh"]
+
+#EOF

infra/test_formatting/cmd.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# author: Ole Schuett
+
+set -e
+
+echo -e "\n========== Rsyncing =========="
+rsync --exclude="*~"          \
+      --exclude=".*/"         \
+      --exclude="*.pyc"       \
+      --exclude=/cp2k/obj/    \
+      --exclude=/cp2k/lib/    \
+      --exclude=/cp2k/exe/    \
+      --ignore-times          \
+      --update                \
+      --verbose               \
+      --recursive             \
+      --checksum              \
+      /opt/cp2k-local/  /opt/cp2k-master/
+
+echo -e "\n========== Running formatting test =========="
+cd /opt/cp2k-master/cp2k
+#TODO change test_formatting.sh
+sed -i s/svn/#svn/g ./tools/formatting/test_formatting.sh
+./tools/formatting/test_formatting.sh
+
+#EOF

infra/test_python/Dockerfile

@@ -1,5 +1,9 @@
 FROM ubuntu:rolling
 
+# author: Ole Schuett
+
+# TODO: maybe use conda instead?
+
 RUN apt-get update && apt-get install -y --no-install-recommends \
     python                \
     python3               \

infra/test_python/cmd.sh

@@ -1,4 +1,8 @@
-#!/bin/bash -e
+#!/bin/bash
+
+# author: Ole Schuett
+
+set -e
 
 run_selftests() {
     PYTHON=$1
@@ -7,12 +11,16 @@ run_selftests() {
     VERSION=`$PYTHON -V 2>&1 | tr -d '\n'`
     echo ""
     echo "==========" $VERSION "=========="
-    find ./src/ ./tools/ -name "*.pyc" -exec rm {} \;
+    #find ./src/ ./tools/ -name "*.pyc" -exec rm {} \;
     for i in $@ ; do
+        set +e #disable error trapping
+        if [[ `head -n1 $i` =~ "python3" && "$PYTHON" != "python3" ]]; then
+          echo "Skipping $i - it's Python3 only."
+          continue
+        fi
         echo "Selftesting" $i
         cd $(dirname $i)
-        set +e #disable error trapping
-        $PYTHON ./$(basename $i) --selftest
+        $PYTHON -B ./$(basename $i) --selftest
         if [ $? -ne 0 ]; then
             echo "Selftest of $i failed"
             ERRORS=$((ERRORS+1))
@@ -24,8 +32,12 @@ run_selftests() {
 
 
 #===============================================================================
-
-cd /opt/cp2k_local/
+echo "Copy start..."
+mkdir /opt/cp2k-local-rw/
+cp -r /opt/cp2k-local/cp2k/tools /opt/cp2k-local-rw/tools
+cp -r /opt/cp2k-local/cp2k/src /opt/cp2k-local-rw/src
+cd /opt/cp2k-local-rw/
+echo "Copy done."
 
 ERRORS=0