From 9dd2f63cea67b4301344660af9182a411d86dcc7 Mon Sep 17 00:00:00 2001 From: Matthias Krack Date: Thu, 29 Jan 2026 10:20:49 +0100 Subject: [PATCH] Replace old spack-based testers with new ones (use make_cp2k.sh) --- tools/dashboard/dashboard.conf | 18 +-- .../docker/Dockerfile.make_cp2k_openmpi-psmp | 52 ------- tools/docker/Dockerfile.make_cp2k_psmp | 52 ------- tools/docker/Dockerfile.make_cp2k_psmp-4x2 | 52 ------- tools/docker/Dockerfile.make_cp2k_psmp-gcc10 | 52 ------- tools/docker/Dockerfile.make_cp2k_ssmp | 52 ------- .../docker/Dockerfile.test_spack_openmpi-psmp | 105 +++---------- tools/docker/Dockerfile.test_spack_psmp | 102 +++---------- tools/docker/Dockerfile.test_spack_psmp-4x2 | 102 +++---------- tools/docker/Dockerfile.test_spack_ssmp | 102 +++---------- tools/docker/cp2k-ci.conf | 16 -- tools/docker/generate_dockerfiles.py | 144 ++---------------- 12 files changed, 91 insertions(+), 758 deletions(-) delete mode 100644 tools/docker/Dockerfile.make_cp2k_openmpi-psmp delete mode 100644 tools/docker/Dockerfile.make_cp2k_psmp delete mode 100644 tools/docker/Dockerfile.make_cp2k_psmp-4x2 delete mode 100644 tools/docker/Dockerfile.make_cp2k_psmp-gcc10 delete mode 100644 tools/docker/Dockerfile.make_cp2k_ssmp diff --git a/tools/dashboard/dashboard.conf b/tools/dashboard/dashboard.conf index bcfc9004eb..e4c3e350bf 100644 --- a/tools/dashboard/dashboard.conf +++ b/tools/dashboard/dashboard.conf @@ -318,20 +318,4 @@ timeout: 170 host: GCP report_url: https://storage.googleapis.com/cp2k-ci/dashboard_asan-psmp_report.txt -# ============================================================================== - -[make-cp2k-psmp] -sortkey: 2000 -name: make_cp2k (psmp) -timeout: 170 -host: GCP -report_url: https://storage.googleapis.com/cp2k-ci/dashboard_make-cp2k-psmp_report.txt - -[make-cp2k-ssmp] -sortkey: 2010 -name: make_cp2k (ssmp) -timeout: 170 -host: GCP -report_url: https://storage.googleapis.com/cp2k-ci/dashboard_make-cp2k-ssmp_report.txt - -#EOF +# EOF diff --git a/tools/docker/Dockerfile.make_cp2k_openmpi-psmp b/tools/docker/Dockerfile.make_cp2k_openmpi-psmp deleted file mode 100644 index a8a5329041..0000000000 --- a/tools/docker/Dockerfile.make_cp2k_openmpi-psmp +++ /dev/null @@ -1,52 +0,0 @@ -# -# This file was created by generate_dockerfiles.py. -# Usage: ./spack_cache_start.sh; podman build --network=host --shm-size=1g -f ./Dockerfile.make_cp2k_openmpi-psmp ../../ -# - -FROM "ubuntu:24.04" - -RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ - bzip2 \ - ca-certificates \ - cmake \ - g++-13 \ - gcc-13 \ - gfortran-13 \ - git \ - gnupg \ - libssh-dev \ - libssl-dev \ - libtool \ - libtool-bin \ - lsb-release \ - make \ - ninja-build \ - patch \ - pkgconf \ - python3 \ - python3-dev \ - python3-pip \ - python3-venv \ - unzip \ - wget \ - xxd \ - xz-utils \ - zstd \ - && rm -rf /var/lib/apt/lists/* - -# Create links in /usr/local/bin to overrule links in /usr/bin -RUN ln -sf /usr/bin/gcc-13 /usr/local/bin/gcc && \ - ln -sf /usr/bin/g++-13 /usr/local/bin/g++ && \ - ln -sf /usr/bin/gfortran-13 /usr/local/bin/gfortran - -ARG SPACK_CACHE="s3://spack-cache --s3-endpoint-url=http://localhost:9000" - -WORKDIR /opt -COPY . cp2k/ - -WORKDIR /opt/cp2k -RUN /bin/bash -o pipefail -c "source ./make_cp2k.sh -cv psmp -mpi openmpi -t \"\"" - -WORKDIR /mnt -ENTRYPOINT ["/opt/cp2k/install/bin/entrypoint.sh"] -CMD ["cp2k", "--help"] diff --git a/tools/docker/Dockerfile.make_cp2k_psmp b/tools/docker/Dockerfile.make_cp2k_psmp deleted file mode 100644 index 46b2d48947..0000000000 --- a/tools/docker/Dockerfile.make_cp2k_psmp +++ /dev/null @@ -1,52 +0,0 @@ -# -# This file was created by generate_dockerfiles.py. -# Usage: ./spack_cache_start.sh; podman build --network=host --shm-size=1g -f ./Dockerfile.make_cp2k_psmp ../../ -# - -FROM "ubuntu:24.04" - -RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ - bzip2 \ - ca-certificates \ - cmake \ - g++-13 \ - gcc-13 \ - gfortran-13 \ - git \ - gnupg \ - libssh-dev \ - libssl-dev \ - libtool \ - libtool-bin \ - lsb-release \ - make \ - ninja-build \ - patch \ - pkgconf \ - python3 \ - python3-dev \ - python3-pip \ - python3-venv \ - unzip \ - wget \ - xxd \ - xz-utils \ - zstd \ - && rm -rf /var/lib/apt/lists/* - -# Create links in /usr/local/bin to overrule links in /usr/bin -RUN ln -sf /usr/bin/gcc-13 /usr/local/bin/gcc && \ - ln -sf /usr/bin/g++-13 /usr/local/bin/g++ && \ - ln -sf /usr/bin/gfortran-13 /usr/local/bin/gfortran - -ARG SPACK_CACHE="s3://spack-cache --s3-endpoint-url=http://localhost:9000" - -WORKDIR /opt -COPY . cp2k/ - -WORKDIR /opt/cp2k -RUN /bin/bash -o pipefail -c "source ./make_cp2k.sh -cv psmp -mpi mpich -t \"\"" - -WORKDIR /mnt -ENTRYPOINT ["/opt/cp2k/install/bin/entrypoint.sh"] -CMD ["cp2k", "--help"] diff --git a/tools/docker/Dockerfile.make_cp2k_psmp-4x2 b/tools/docker/Dockerfile.make_cp2k_psmp-4x2 deleted file mode 100644 index e01bd74fc9..0000000000 --- a/tools/docker/Dockerfile.make_cp2k_psmp-4x2 +++ /dev/null @@ -1,52 +0,0 @@ -# -# This file was created by generate_dockerfiles.py. -# Usage: ./spack_cache_start.sh; podman build --network=host --shm-size=1g -f ./Dockerfile.make_cp2k_psmp-4x2 ../../ -# - -FROM "ubuntu:24.04" - -RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ - bzip2 \ - ca-certificates \ - cmake \ - g++-13 \ - gcc-13 \ - gfortran-13 \ - git \ - gnupg \ - libssh-dev \ - libssl-dev \ - libtool \ - libtool-bin \ - lsb-release \ - make \ - ninja-build \ - patch \ - pkgconf \ - python3 \ - python3-dev \ - python3-pip \ - python3-venv \ - unzip \ - wget \ - xxd \ - xz-utils \ - zstd \ - && rm -rf /var/lib/apt/lists/* - -# Create links in /usr/local/bin to overrule links in /usr/bin -RUN ln -sf /usr/bin/gcc-13 /usr/local/bin/gcc && \ - ln -sf /usr/bin/g++-13 /usr/local/bin/g++ && \ - ln -sf /usr/bin/gfortran-13 /usr/local/bin/gfortran - -ARG SPACK_CACHE="s3://spack-cache --s3-endpoint-url=http://localhost:9000" - -WORKDIR /opt -COPY . cp2k/ - -WORKDIR /opt/cp2k -RUN /bin/bash -o pipefail -c "source ./make_cp2k.sh -cv psmp -mpi mpich -t \"--mpiranks=4 --ompthreads=2\"" - -WORKDIR /mnt -ENTRYPOINT ["/opt/cp2k/install/bin/entrypoint.sh"] -CMD ["cp2k", "--help"] diff --git a/tools/docker/Dockerfile.make_cp2k_psmp-gcc10 b/tools/docker/Dockerfile.make_cp2k_psmp-gcc10 deleted file mode 100644 index 03d70e868c..0000000000 --- a/tools/docker/Dockerfile.make_cp2k_psmp-gcc10 +++ /dev/null @@ -1,52 +0,0 @@ -# -# This file was created by generate_dockerfiles.py. -# Usage: ./spack_cache_start.sh; podman build --network=host --shm-size=1g -f ./Dockerfile.make_cp2k_psmp-gcc10 ../../ -# - -FROM "ubuntu:24.04" - -RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ - bzip2 \ - ca-certificates \ - cmake \ - g++-10 \ - gcc-10 \ - gfortran-10 \ - git \ - gnupg \ - libssh-dev \ - libssl-dev \ - libtool \ - libtool-bin \ - lsb-release \ - make \ - ninja-build \ - patch \ - pkgconf \ - python3 \ - python3-dev \ - python3-pip \ - python3-venv \ - unzip \ - wget \ - xxd \ - xz-utils \ - zstd \ - && rm -rf /var/lib/apt/lists/* - -# Create links in /usr/local/bin to overrule links in /usr/bin -RUN ln -sf /usr/bin/gcc-10 /usr/local/bin/gcc && \ - ln -sf /usr/bin/g++-10 /usr/local/bin/g++ && \ - ln -sf /usr/bin/gfortran-10 /usr/local/bin/gfortran - -ARG SPACK_CACHE="s3://spack-cache --s3-endpoint-url=http://localhost:9000" - -WORKDIR /opt -COPY . cp2k/ - -WORKDIR /opt/cp2k -RUN /bin/bash -o pipefail -c "source ./make_cp2k.sh -cv psmp -mpi mpich -t \"\"" - -WORKDIR /mnt -ENTRYPOINT ["/opt/cp2k/install/bin/entrypoint.sh"] -CMD ["cp2k", "--help"] diff --git a/tools/docker/Dockerfile.make_cp2k_ssmp b/tools/docker/Dockerfile.make_cp2k_ssmp deleted file mode 100644 index 77a9cccc2e..0000000000 --- a/tools/docker/Dockerfile.make_cp2k_ssmp +++ /dev/null @@ -1,52 +0,0 @@ -# -# This file was created by generate_dockerfiles.py. -# Usage: ./spack_cache_start.sh; podman build --network=host --shm-size=1g -f ./Dockerfile.make_cp2k_ssmp ../../ -# - -FROM "ubuntu:24.04" - -RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ - bzip2 \ - ca-certificates \ - cmake \ - g++-13 \ - gcc-13 \ - gfortran-13 \ - git \ - gnupg \ - libssh-dev \ - libssl-dev \ - libtool \ - libtool-bin \ - lsb-release \ - make \ - ninja-build \ - patch \ - pkgconf \ - python3 \ - python3-dev \ - python3-pip \ - python3-venv \ - unzip \ - wget \ - xxd \ - xz-utils \ - zstd \ - && rm -rf /var/lib/apt/lists/* - -# Create links in /usr/local/bin to overrule links in /usr/bin -RUN ln -sf /usr/bin/gcc-13 /usr/local/bin/gcc && \ - ln -sf /usr/bin/g++-13 /usr/local/bin/g++ && \ - ln -sf /usr/bin/gfortran-13 /usr/local/bin/gfortran - -ARG SPACK_CACHE="s3://spack-cache --s3-endpoint-url=http://localhost:9000" - -WORKDIR /opt -COPY . cp2k/ - -WORKDIR /opt/cp2k -RUN /bin/bash -o pipefail -c "source ./make_cp2k.sh -cv ssmp -mpi no -t \"\"" - -WORKDIR /mnt -ENTRYPOINT ["/opt/cp2k/install/bin/entrypoint.sh"] -CMD ["cp2k", "--help"] diff --git a/tools/docker/Dockerfile.test_spack_openmpi-psmp b/tools/docker/Dockerfile.test_spack_openmpi-psmp index e206115767..d8e796c7f7 100644 --- a/tools/docker/Dockerfile.test_spack_openmpi-psmp +++ b/tools/docker/Dockerfile.test_spack_openmpi-psmp @@ -3,16 +3,15 @@ # Usage: ./spack_cache_start.sh; podman build --network=host --shm-size=1g -f ./Dockerfile.test_spack_openmpi-psmp ../../ # -FROM ubuntu:24.04 +FROM "ubuntu:24.04" -# Install packages required to build the CP2K dependencies with Spack RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ bzip2 \ ca-certificates \ cmake \ - g++ \ - gcc \ - gfortran \ + g++-13 \ + gcc-13 \ + gfortran-13 \ git \ gnupg \ libssh-dev \ @@ -32,91 +31,25 @@ RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ wget \ xxd \ xz-utils \ - zstd && rm -rf /var/lib/apt/lists/* + zstd \ + && rm -rf /var/lib/apt/lists/* -# Create and activate a virtual environment for Python packages -RUN python3 -m venv /opt/venv -ENV PATH="/opt/venv/bin:${PATH}" -RUN pip3 install --quiet boto3==1.38.11 google-cloud-storage==3.1.0 +# Create links in /usr/local/bin to override tbe links in /usr/bin +RUN ln -sf /usr/bin/gcc-13 /usr/local/bin/gcc && \ + ln -sf /usr/bin/g++-13 /usr/local/bin/g++ && \ + ln -sf /usr/bin/gfortran-13 /usr/local/bin/gfortran -# Retrieve the number of available CPU cores -ARG NUM_PROCS -ENV NUM_PROCS=${NUM_PROCS:-32} - -# Install Spack and Spack packages -WORKDIR /root/spack -ARG SPACK_VERSION -ENV SPACK_VERSION=${SPACK_VERSION:-1.1.0} -ARG SPACK_PACKAGES_VERSION -ENV SPACK_PACKAGES_VERSION=${SPACK_PACKAGES_VERSION:-2025.11.0} -ARG SPACK_REPO=https://github.com/spack/spack -ENV SPACK_ROOT=/opt/spack-${SPACK_VERSION} -ARG SPACK_PACKAGES_REPO=https://github.com/spack/spack-packages -ENV SPACK_PACKAGES_ROOT=/opt/spack-packages-${SPACK_PACKAGES_VERSION} -RUN mkdir -p ${SPACK_ROOT} \ - && wget -q ${SPACK_REPO}/archive/v${SPACK_VERSION}.tar.gz \ - && tar -xzf v${SPACK_VERSION}.tar.gz -C /opt && rm -f v${SPACK_VERSION}.tar.gz \ - && mkdir -p ${SPACK_PACKAGES_ROOT} \ - && wget -q ${SPACK_PACKAGES_REPO}/archive/v${SPACK_PACKAGES_VERSION}.tar.gz \ - && tar -xzf v${SPACK_PACKAGES_VERSION}.tar.gz -C /opt && rm -f v${SPACK_PACKAGES_VERSION}.tar.gz - -ENV PATH="${SPACK_ROOT}/bin:${PATH}" - -# Add Spack packages builtin repository -RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/builtin - -# Find all compilers -RUN spack compiler find - -# Find all external packages -RUN spack external find --all --not-buildable - -# Add local Spack cache ARG SPACK_CACHE="s3://spack-cache --s3-endpoint-url=http://localhost:9000" -COPY ./tools/docker/scripts/setup_spack_cache.sh ./ -RUN ./setup_spack_cache.sh -# Copy Spack configuration and build recipes -ARG CP2K_VERSION -ENV CP2K_VERSION=${CP2K_VERSION:-psmp} -COPY ./tools/spack/cp2k_deps_${CP2K_VERSION}.yaml ./ -RUN sed -E -e '/\s*-\s+"mpich@/ s/^ /#/' \ - -E -e '/\s*#\s*-\s+"openmpi@/ s/#/ /' \ - -E -e '/\s*-\s+mpich/ s/mpich$/openmpi/' \ - -i cp2k_deps_${CP2K_VERSION}.yaml -COPY ./tools/spack/spack_repo/cp2k_dev ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev/ -RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev/ -RUN spack env create myenv cp2k_deps_${CP2K_VERSION}.yaml && \ - spack -e myenv repo list +# Copy CP2K repository into container +WORKDIR /opt +COPY . cp2k/ -# Install CP2K dependencies via Spack -RUN spack -e myenv concretize -f -RUN spack -e myenv env depfile -o spack_makefile -RUN make -j${NUM_PROCS} --file=spack_makefile SPACK_COLOR=never --output-sync=recurse - -# Install CP2K sources. +# Build and install CP2K WORKDIR /opt/cp2k -COPY ./src ./src -COPY ./data ./data -COPY ./tools/build_utils ./tools/build_utils -COPY ./cmake ./cmake -COPY ./CMakeLists.txt . +RUN /bin/bash -o pipefail -c "source ./make_cp2k.sh -cv psmp -mpi openmpi -t \"\"" -# Compile CP2K. -COPY ./tools/docker/scripts/build_cp2k.sh . -RUN ./build_cp2k.sh spack psmp - -# Run regression tests. -ARG TESTOPTS="" -COPY ./tests ./tests -COPY ./tools/docker/scripts/test_regtest.sh ./ -RUN /bin/bash -o pipefail -c " \ - TESTOPTS='${TESTOPTS}' \ - ./test_regtest.sh spack psmp |& tee report.log && \ - rm -rf regtesting" - -# Output the report if the image is old and was therefore pulled from the build cache. -CMD cat $(find ./report.log -mmin +10) | sed '/^Summary:/ s/$/ (cached)/' -ENTRYPOINT [] - -#EOF +# Finalise container build +WORKDIR /mnt +ENTRYPOINT ["/opt/cp2k/install/bin/entrypoint.sh"] +CMD ["cp2k", "--help"] diff --git a/tools/docker/Dockerfile.test_spack_psmp b/tools/docker/Dockerfile.test_spack_psmp index 785a9e8550..46c637ec01 100644 --- a/tools/docker/Dockerfile.test_spack_psmp +++ b/tools/docker/Dockerfile.test_spack_psmp @@ -3,16 +3,15 @@ # Usage: ./spack_cache_start.sh; podman build --network=host --shm-size=1g -f ./Dockerfile.test_spack_psmp ../../ # -FROM ubuntu:24.04 +FROM "ubuntu:24.04" -# Install packages required to build the CP2K dependencies with Spack RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ bzip2 \ ca-certificates \ cmake \ - g++ \ - gcc \ - gfortran \ + g++-13 \ + gcc-13 \ + gfortran-13 \ git \ gnupg \ libssh-dev \ @@ -32,88 +31,25 @@ RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ wget \ xxd \ xz-utils \ - zstd && rm -rf /var/lib/apt/lists/* + zstd \ + && rm -rf /var/lib/apt/lists/* -# Create and activate a virtual environment for Python packages -RUN python3 -m venv /opt/venv -ENV PATH="/opt/venv/bin:${PATH}" -RUN pip3 install --quiet boto3==1.38.11 google-cloud-storage==3.1.0 +# Create links in /usr/local/bin to override tbe links in /usr/bin +RUN ln -sf /usr/bin/gcc-13 /usr/local/bin/gcc && \ + ln -sf /usr/bin/g++-13 /usr/local/bin/g++ && \ + ln -sf /usr/bin/gfortran-13 /usr/local/bin/gfortran -# Retrieve the number of available CPU cores -ARG NUM_PROCS -ENV NUM_PROCS=${NUM_PROCS:-32} - -# Install Spack and Spack packages -WORKDIR /root/spack -ARG SPACK_VERSION -ENV SPACK_VERSION=${SPACK_VERSION:-1.1.0} -ARG SPACK_PACKAGES_VERSION -ENV SPACK_PACKAGES_VERSION=${SPACK_PACKAGES_VERSION:-2025.11.0} -ARG SPACK_REPO=https://github.com/spack/spack -ENV SPACK_ROOT=/opt/spack-${SPACK_VERSION} -ARG SPACK_PACKAGES_REPO=https://github.com/spack/spack-packages -ENV SPACK_PACKAGES_ROOT=/opt/spack-packages-${SPACK_PACKAGES_VERSION} -RUN mkdir -p ${SPACK_ROOT} \ - && wget -q ${SPACK_REPO}/archive/v${SPACK_VERSION}.tar.gz \ - && tar -xzf v${SPACK_VERSION}.tar.gz -C /opt && rm -f v${SPACK_VERSION}.tar.gz \ - && mkdir -p ${SPACK_PACKAGES_ROOT} \ - && wget -q ${SPACK_PACKAGES_REPO}/archive/v${SPACK_PACKAGES_VERSION}.tar.gz \ - && tar -xzf v${SPACK_PACKAGES_VERSION}.tar.gz -C /opt && rm -f v${SPACK_PACKAGES_VERSION}.tar.gz - -ENV PATH="${SPACK_ROOT}/bin:${PATH}" - -# Add Spack packages builtin repository -RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/builtin - -# Find all compilers -RUN spack compiler find - -# Find all external packages -RUN spack external find --all --not-buildable - -# Add local Spack cache ARG SPACK_CACHE="s3://spack-cache --s3-endpoint-url=http://localhost:9000" -COPY ./tools/docker/scripts/setup_spack_cache.sh ./ -RUN ./setup_spack_cache.sh -# Copy Spack configuration and build recipes -ARG CP2K_VERSION -ENV CP2K_VERSION=${CP2K_VERSION:-psmp} -COPY ./tools/spack/cp2k_deps_${CP2K_VERSION}.yaml ./ +# Copy CP2K repository into container +WORKDIR /opt +COPY . cp2k/ -COPY ./tools/spack/spack_repo/cp2k_dev ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev/ -RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev/ -RUN spack env create myenv cp2k_deps_${CP2K_VERSION}.yaml && \ - spack -e myenv repo list - -# Install CP2K dependencies via Spack -RUN spack -e myenv concretize -f -RUN spack -e myenv env depfile -o spack_makefile -RUN make -j${NUM_PROCS} --file=spack_makefile SPACK_COLOR=never --output-sync=recurse - -# Install CP2K sources. +# Build and install CP2K WORKDIR /opt/cp2k -COPY ./src ./src -COPY ./data ./data -COPY ./tools/build_utils ./tools/build_utils -COPY ./cmake ./cmake -COPY ./CMakeLists.txt . +RUN /bin/bash -o pipefail -c "source ./make_cp2k.sh -cv psmp -mpi mpich -t \"\"" -# Compile CP2K. -COPY ./tools/docker/scripts/build_cp2k.sh . -RUN ./build_cp2k.sh spack psmp - -# Run regression tests. -ARG TESTOPTS="" -COPY ./tests ./tests -COPY ./tools/docker/scripts/test_regtest.sh ./ -RUN /bin/bash -o pipefail -c " \ - TESTOPTS='${TESTOPTS}' \ - ./test_regtest.sh spack psmp |& tee report.log && \ - rm -rf regtesting" - -# Output the report if the image is old and was therefore pulled from the build cache. -CMD cat $(find ./report.log -mmin +10) | sed '/^Summary:/ s/$/ (cached)/' -ENTRYPOINT [] - -#EOF +# Finalise container build +WORKDIR /mnt +ENTRYPOINT ["/opt/cp2k/install/bin/entrypoint.sh"] +CMD ["cp2k", "--help"] diff --git a/tools/docker/Dockerfile.test_spack_psmp-4x2 b/tools/docker/Dockerfile.test_spack_psmp-4x2 index e949e3862c..3419378153 100644 --- a/tools/docker/Dockerfile.test_spack_psmp-4x2 +++ b/tools/docker/Dockerfile.test_spack_psmp-4x2 @@ -3,16 +3,15 @@ # Usage: ./spack_cache_start.sh; podman build --network=host --shm-size=1g -f ./Dockerfile.test_spack_psmp-4x2 ../../ # -FROM ubuntu:24.04 +FROM "ubuntu:24.04" -# Install packages required to build the CP2K dependencies with Spack RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ bzip2 \ ca-certificates \ cmake \ - g++ \ - gcc \ - gfortran \ + g++-13 \ + gcc-13 \ + gfortran-13 \ git \ gnupg \ libssh-dev \ @@ -32,88 +31,25 @@ RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ wget \ xxd \ xz-utils \ - zstd && rm -rf /var/lib/apt/lists/* + zstd \ + && rm -rf /var/lib/apt/lists/* -# Create and activate a virtual environment for Python packages -RUN python3 -m venv /opt/venv -ENV PATH="/opt/venv/bin:${PATH}" -RUN pip3 install --quiet boto3==1.38.11 google-cloud-storage==3.1.0 +# Create links in /usr/local/bin to override tbe links in /usr/bin +RUN ln -sf /usr/bin/gcc-13 /usr/local/bin/gcc && \ + ln -sf /usr/bin/g++-13 /usr/local/bin/g++ && \ + ln -sf /usr/bin/gfortran-13 /usr/local/bin/gfortran -# Retrieve the number of available CPU cores -ARG NUM_PROCS -ENV NUM_PROCS=${NUM_PROCS:-32} - -# Install Spack and Spack packages -WORKDIR /root/spack -ARG SPACK_VERSION -ENV SPACK_VERSION=${SPACK_VERSION:-1.1.0} -ARG SPACK_PACKAGES_VERSION -ENV SPACK_PACKAGES_VERSION=${SPACK_PACKAGES_VERSION:-2025.11.0} -ARG SPACK_REPO=https://github.com/spack/spack -ENV SPACK_ROOT=/opt/spack-${SPACK_VERSION} -ARG SPACK_PACKAGES_REPO=https://github.com/spack/spack-packages -ENV SPACK_PACKAGES_ROOT=/opt/spack-packages-${SPACK_PACKAGES_VERSION} -RUN mkdir -p ${SPACK_ROOT} \ - && wget -q ${SPACK_REPO}/archive/v${SPACK_VERSION}.tar.gz \ - && tar -xzf v${SPACK_VERSION}.tar.gz -C /opt && rm -f v${SPACK_VERSION}.tar.gz \ - && mkdir -p ${SPACK_PACKAGES_ROOT} \ - && wget -q ${SPACK_PACKAGES_REPO}/archive/v${SPACK_PACKAGES_VERSION}.tar.gz \ - && tar -xzf v${SPACK_PACKAGES_VERSION}.tar.gz -C /opt && rm -f v${SPACK_PACKAGES_VERSION}.tar.gz - -ENV PATH="${SPACK_ROOT}/bin:${PATH}" - -# Add Spack packages builtin repository -RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/builtin - -# Find all compilers -RUN spack compiler find - -# Find all external packages -RUN spack external find --all --not-buildable - -# Add local Spack cache ARG SPACK_CACHE="s3://spack-cache --s3-endpoint-url=http://localhost:9000" -COPY ./tools/docker/scripts/setup_spack_cache.sh ./ -RUN ./setup_spack_cache.sh -# Copy Spack configuration and build recipes -ARG CP2K_VERSION -ENV CP2K_VERSION=${CP2K_VERSION:-psmp} -COPY ./tools/spack/cp2k_deps_${CP2K_VERSION}.yaml ./ +# Copy CP2K repository into container +WORKDIR /opt +COPY . cp2k/ -COPY ./tools/spack/spack_repo/cp2k_dev ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev/ -RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev/ -RUN spack env create myenv cp2k_deps_${CP2K_VERSION}.yaml && \ - spack -e myenv repo list - -# Install CP2K dependencies via Spack -RUN spack -e myenv concretize -f -RUN spack -e myenv env depfile -o spack_makefile -RUN make -j${NUM_PROCS} --file=spack_makefile SPACK_COLOR=never --output-sync=recurse - -# Install CP2K sources. +# Build and install CP2K WORKDIR /opt/cp2k -COPY ./src ./src -COPY ./data ./data -COPY ./tools/build_utils ./tools/build_utils -COPY ./cmake ./cmake -COPY ./CMakeLists.txt . +RUN /bin/bash -o pipefail -c "source ./make_cp2k.sh -cv psmp -mpi mpich -t \"--mpiranks=4 --ompthreads=2\"" -# Compile CP2K. -COPY ./tools/docker/scripts/build_cp2k.sh . -RUN ./build_cp2k.sh spack psmp - -# Run regression tests. -ARG TESTOPTS="--mpiranks=4 --ompthreads=2" -COPY ./tests ./tests -COPY ./tools/docker/scripts/test_regtest.sh ./ -RUN /bin/bash -o pipefail -c " \ - TESTOPTS='${TESTOPTS}' \ - ./test_regtest.sh spack psmp |& tee report.log && \ - rm -rf regtesting" - -# Output the report if the image is old and was therefore pulled from the build cache. -CMD cat $(find ./report.log -mmin +10) | sed '/^Summary:/ s/$/ (cached)/' -ENTRYPOINT [] - -#EOF +# Finalise container build +WORKDIR /mnt +ENTRYPOINT ["/opt/cp2k/install/bin/entrypoint.sh"] +CMD ["cp2k", "--help"] diff --git a/tools/docker/Dockerfile.test_spack_ssmp b/tools/docker/Dockerfile.test_spack_ssmp index 6725860d4c..9b9ad909c3 100644 --- a/tools/docker/Dockerfile.test_spack_ssmp +++ b/tools/docker/Dockerfile.test_spack_ssmp @@ -3,16 +3,15 @@ # Usage: ./spack_cache_start.sh; podman build --network=host --shm-size=1g -f ./Dockerfile.test_spack_ssmp ../../ # -FROM ubuntu:24.04 +FROM "ubuntu:24.04" -# Install packages required to build the CP2K dependencies with Spack RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ bzip2 \ ca-certificates \ cmake \ - g++ \ - gcc \ - gfortran \ + g++-13 \ + gcc-13 \ + gfortran-13 \ git \ gnupg \ libssh-dev \ @@ -32,88 +31,25 @@ RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ wget \ xxd \ xz-utils \ - zstd && rm -rf /var/lib/apt/lists/* + zstd \ + && rm -rf /var/lib/apt/lists/* -# Create and activate a virtual environment for Python packages -RUN python3 -m venv /opt/venv -ENV PATH="/opt/venv/bin:${PATH}" -RUN pip3 install --quiet boto3==1.38.11 google-cloud-storage==3.1.0 +# Create links in /usr/local/bin to override tbe links in /usr/bin +RUN ln -sf /usr/bin/gcc-13 /usr/local/bin/gcc && \ + ln -sf /usr/bin/g++-13 /usr/local/bin/g++ && \ + ln -sf /usr/bin/gfortran-13 /usr/local/bin/gfortran -# Retrieve the number of available CPU cores -ARG NUM_PROCS -ENV NUM_PROCS=${NUM_PROCS:-32} - -# Install Spack and Spack packages -WORKDIR /root/spack -ARG SPACK_VERSION -ENV SPACK_VERSION=${SPACK_VERSION:-1.1.0} -ARG SPACK_PACKAGES_VERSION -ENV SPACK_PACKAGES_VERSION=${SPACK_PACKAGES_VERSION:-2025.11.0} -ARG SPACK_REPO=https://github.com/spack/spack -ENV SPACK_ROOT=/opt/spack-${SPACK_VERSION} -ARG SPACK_PACKAGES_REPO=https://github.com/spack/spack-packages -ENV SPACK_PACKAGES_ROOT=/opt/spack-packages-${SPACK_PACKAGES_VERSION} -RUN mkdir -p ${SPACK_ROOT} \ - && wget -q ${SPACK_REPO}/archive/v${SPACK_VERSION}.tar.gz \ - && tar -xzf v${SPACK_VERSION}.tar.gz -C /opt && rm -f v${SPACK_VERSION}.tar.gz \ - && mkdir -p ${SPACK_PACKAGES_ROOT} \ - && wget -q ${SPACK_PACKAGES_REPO}/archive/v${SPACK_PACKAGES_VERSION}.tar.gz \ - && tar -xzf v${SPACK_PACKAGES_VERSION}.tar.gz -C /opt && rm -f v${SPACK_PACKAGES_VERSION}.tar.gz - -ENV PATH="${SPACK_ROOT}/bin:${PATH}" - -# Add Spack packages builtin repository -RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/builtin - -# Find all compilers -RUN spack compiler find - -# Find all external packages -RUN spack external find --all --not-buildable - -# Add local Spack cache ARG SPACK_CACHE="s3://spack-cache --s3-endpoint-url=http://localhost:9000" -COPY ./tools/docker/scripts/setup_spack_cache.sh ./ -RUN ./setup_spack_cache.sh -# Copy Spack configuration and build recipes -ARG CP2K_VERSION -ENV CP2K_VERSION=${CP2K_VERSION:-ssmp} -COPY ./tools/spack/cp2k_deps_${CP2K_VERSION}.yaml ./ +# Copy CP2K repository into container +WORKDIR /opt +COPY . cp2k/ -COPY ./tools/spack/spack_repo/cp2k_dev ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev/ -RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev/ -RUN spack env create myenv cp2k_deps_${CP2K_VERSION}.yaml && \ - spack -e myenv repo list - -# Install CP2K dependencies via Spack -RUN spack -e myenv concretize -f -RUN spack -e myenv env depfile -o spack_makefile -RUN make -j${NUM_PROCS} --file=spack_makefile SPACK_COLOR=never --output-sync=recurse - -# Install CP2K sources. +# Build and install CP2K WORKDIR /opt/cp2k -COPY ./src ./src -COPY ./data ./data -COPY ./tools/build_utils ./tools/build_utils -COPY ./cmake ./cmake -COPY ./CMakeLists.txt . +RUN /bin/bash -o pipefail -c "source ./make_cp2k.sh -cv ssmp -mpi no -t \"\"" -# Compile CP2K. -COPY ./tools/docker/scripts/build_cp2k.sh . -RUN ./build_cp2k.sh spack ssmp - -# Run regression tests. -ARG TESTOPTS="" -COPY ./tests ./tests -COPY ./tools/docker/scripts/test_regtest.sh ./ -RUN /bin/bash -o pipefail -c " \ - TESTOPTS='${TESTOPTS}' \ - ./test_regtest.sh spack ssmp |& tee report.log && \ - rm -rf regtesting" - -# Output the report if the image is old and was therefore pulled from the build cache. -CMD cat $(find ./report.log -mmin +10) | sed '/^Summary:/ s/$/ (cached)/' -ENTRYPOINT [] - -#EOF +# Finalise container build +WORKDIR /mnt +ENTRYPOINT ["/opt/cp2k/install/bin/entrypoint.sh"] +CMD ["cp2k", "--help"] diff --git a/tools/docker/cp2k-ci.conf b/tools/docker/cp2k-ci.conf index 1ceaf13591..07a053582b 100644 --- a/tools/docker/cp2k-ci.conf +++ b/tools/docker/cp2k-ci.conf @@ -98,22 +98,6 @@ nodepools: pool-main build_path: / dockerfile: /tools/docker/Dockerfile.test_spack_openmpi-psmp -[make-cp2k-psmp] -display_name: make_cp2k (psmp) -tags: daily -cpu: 32 -nodepools: pool-main -build_path: / -dockerfile: /tools/docker/Dockerfile.make_cp2k_psmp - -[make-cp2k-ssmp] -display_name: make_cp2k (ssmp) -tags: daily -cpu: 32 -nodepools: pool-main -build_path: / -dockerfile: /tools/docker/Dockerfile.make_cp2k_ssmp - [conventions] display_name: Conventions tags: daily diff --git a/tools/docker/generate_dockerfiles.py b/tools/docker/generate_dockerfiles.py index ad8d7c6700..6722446b95 100755 --- a/tools/docker/generate_dockerfiles.py +++ b/tools/docker/generate_dockerfiles.py @@ -53,38 +53,23 @@ def main() -> None: f.write(install_deps_ubuntu()) f.write(regtest("minimal", "ssmp")) - with OutputFile(f"Dockerfile.make_cp2k_psmp", args.check) as f: - f.write(install_make_cp2k("psmp", mpi_mode="mpich")) - - with OutputFile(f"Dockerfile.make_cp2k_psmp-gcc10", args.check) as f: - f.write(install_make_cp2k("psmp", mpi_mode="mpich", gcc_version=10)) - - with OutputFile(f"Dockerfile.make_cp2k_psmp-4x2", args.check) as f: - testopts = f"--mpiranks=4 --ompthreads=2" - f.write(install_make_cp2k("psmp", mpi_mode="mpich", testopts=testopts)) - - with OutputFile(f"Dockerfile.make_cp2k_openmpi-psmp", args.check) as f: - f.write(install_make_cp2k("psmp", mpi_mode="openmpi")) - - with OutputFile(f"Dockerfile.make_cp2k_ssmp", args.check) as f: - f.write(install_make_cp2k("ssmp", mpi_mode="no")) - + # Spack/CMake based testers with OutputFile(f"Dockerfile.test_spack_psmp", args.check) as f: - f.write(install_deps_spack("psmp", mpi_mode="mpich")) - f.write(regtest("spack", "psmp")) + f.write(install_cp2k_spack("psmp", mpi_mode="mpich")) + + with OutputFile(f"Dockerfile.test_spack_psmp-gcc10", args.check) as f: + f.write(install_cp2k_spack("psmp", mpi_mode="mpich", gcc_version=10)) with OutputFile(f"Dockerfile.test_spack_psmp-4x2", args.check) as f: testopts = f"--mpiranks=4 --ompthreads=2" - f.write(install_deps_spack("psmp", mpi_mode="mpich")) - f.write(regtest("spack", "psmp", testopts=testopts)) + f.write(install_cp2k_spack("psmp", mpi_mode="mpich", testopts=testopts)) with OutputFile(f"Dockerfile.test_spack_openmpi-psmp", args.check) as f: - f.write(install_deps_spack("psmp", mpi_mode="openmpi")) - f.write(regtest("spack", "psmp")) + f.write(install_cp2k_spack("psmp", mpi_mode="openmpi")) with OutputFile(f"Dockerfile.test_spack_ssmp", args.check) as f: - f.write(install_deps_spack("ssmp", mpi_mode="no")) - f.write(regtest("spack", "ssmp")) + f.write(install_cp2k_spack("ssmp", mpi_mode="no")) + # End Spack/CMake based tester with OutputFile(f"Dockerfile.test_asan-psmp", args.check) as f: f.write(install_deps_toolchain()) @@ -570,111 +555,7 @@ RUN ./scripts/stage9/install_stage9.sh && rm -rf ./build # ====================================================================================== -def install_deps_spack(version: str, mpi_mode: str) -> str: - output = rf""" -FROM ubuntu:24.04 - -# Install packages required to build the CP2K dependencies with Spack -RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ - bzip2 \ - ca-certificates \ - cmake \ - g++ \ - gcc \ - gfortran \ - git \ - gnupg \ - libssh-dev \ - libssl-dev \ - libtool \ - libtool-bin \ - lsb-release \ - make \ - ninja-build \ - patch \ - pkgconf \ - python3 \ - python3-dev \ - python3-pip \ - python3-venv \ - unzip \ - wget \ - xxd \ - xz-utils \ - zstd && rm -rf /var/lib/apt/lists/* - -# Create and activate a virtual environment for Python packages -RUN python3 -m venv /opt/venv -ENV PATH="/opt/venv/bin:${{PATH}}" -RUN pip3 install --quiet boto3==1.38.11 google-cloud-storage==3.1.0 - -# Retrieve the number of available CPU cores -ARG NUM_PROCS -ENV NUM_PROCS=${{NUM_PROCS:-32}} - -# Install Spack and Spack packages -WORKDIR /root/spack -ARG SPACK_VERSION -ENV SPACK_VERSION=${{SPACK_VERSION:-1.1.0}} -ARG SPACK_PACKAGES_VERSION -ENV SPACK_PACKAGES_VERSION=${{SPACK_PACKAGES_VERSION:-2025.11.0}} -ARG SPACK_REPO=https://github.com/spack/spack -ENV SPACK_ROOT=/opt/spack-${{SPACK_VERSION}} -ARG SPACK_PACKAGES_REPO=https://github.com/spack/spack-packages -ENV SPACK_PACKAGES_ROOT=/opt/spack-packages-${{SPACK_PACKAGES_VERSION}} -RUN mkdir -p ${{SPACK_ROOT}} \ - && wget -q ${{SPACK_REPO}}/archive/v${{SPACK_VERSION}}.tar.gz \ - && tar -xzf v${{SPACK_VERSION}}.tar.gz -C /opt && rm -f v${{SPACK_VERSION}}.tar.gz \ - && mkdir -p ${{SPACK_PACKAGES_ROOT}} \ - && wget -q ${{SPACK_PACKAGES_REPO}}/archive/v${{SPACK_PACKAGES_VERSION}}.tar.gz \ - && tar -xzf v${{SPACK_PACKAGES_VERSION}}.tar.gz -C /opt && rm -f v${{SPACK_PACKAGES_VERSION}}.tar.gz - -ENV PATH="${{SPACK_ROOT}}/bin:${{PATH}}" - -# Add Spack packages builtin repository -RUN spack repo add --scope site ${{SPACK_PACKAGES_ROOT}}/repos/spack_repo/builtin - -# Find all compilers -RUN spack compiler find - -# Find all external packages -RUN spack external find --all --not-buildable - -# Add local Spack cache -ARG SPACK_CACHE="s3://spack-cache --s3-endpoint-url=http://localhost:9000" -COPY ./tools/docker/scripts/setup_spack_cache.sh ./ -RUN ./setup_spack_cache.sh - -# Copy Spack configuration and build recipes -ARG CP2K_VERSION -ENV CP2K_VERSION=${{CP2K_VERSION:-{version}}} -COPY ./tools/spack/cp2k_deps_${{CP2K_VERSION}}.yaml ./ -""" - - if mpi_mode == "openmpi": - output += rf""" -RUN sed -E -e '/\s*-\s+"mpich@/ s/^ /#/' \ - -E -e '/\s*#\s*-\s+"openmpi@/ s/#/ /' \ - -E -e '/\s*-\s+mpich/ s/mpich$/openmpi/' \ - -i cp2k_deps_${{CP2K_VERSION}}.yaml -""".strip() - - output += rf""" -COPY ./tools/spack/spack_repo/cp2k_dev ${{SPACK_PACKAGES_ROOT}}/repos/spack_repo/cp2k_dev/ -RUN spack repo add --scope site ${{SPACK_PACKAGES_ROOT}}/repos/spack_repo/cp2k_dev/ -RUN spack env create myenv cp2k_deps_${{CP2K_VERSION}}.yaml && \ - spack -e myenv repo list - -# Install CP2K dependencies via Spack -RUN spack -e myenv concretize -f -RUN spack -e myenv env depfile -o spack_makefile -RUN make -j${{NUM_PROCS}} --file=spack_makefile SPACK_COLOR=never --output-sync=recurse -""" - return output - - -# ====================================================================================== -def install_make_cp2k( +def install_cp2k_spack( version: str, mpi_mode: str, gcc_version: int = 13, @@ -712,19 +593,22 @@ RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ zstd \ && rm -rf /var/lib/apt/lists/* -# Create links in /usr/local/bin to overrule links in /usr/bin +# Create links in /usr/local/bin to override tbe links in /usr/bin RUN ln -sf /usr/bin/gcc-{gcc_version} /usr/local/bin/gcc && \ ln -sf /usr/bin/g++-{gcc_version} /usr/local/bin/g++ && \ ln -sf /usr/bin/gfortran-{gcc_version} /usr/local/bin/gfortran ARG SPACK_CACHE="s3://spack-cache --s3-endpoint-url=http://localhost:9000" +# Copy CP2K repository into container WORKDIR /opt COPY . cp2k/ +# Build and install CP2K WORKDIR /opt/cp2k RUN /bin/bash -o pipefail -c "source ./make_cp2k.sh -cv {version} -mpi {mpi_mode} -t \"{testopts}\"" +# Finalise container build WORKDIR /mnt ENTRYPOINT ["/opt/cp2k/install/bin/entrypoint.sh"] CMD ["cp2k", "--help"]