Use 2-stage dockerfiles for CSCS CI/CD

to reduce/minimize image size for production container deployment
2026-07-28 06:05:29 -04:00 · 2025-02-15 19:33:12 +01:00 · 2025-02-15 19:33:12 +01:00 · 8abc00d379
commit 8abc00d379
parent f05d45d108
5 changed files with 152 additions and 47 deletions
--- a/ci/cscs-ci.yml
+++ b/ci/cscs-ci.yml
@ -4,7 +4,7 @@ include:
 stages:
  - build_deps
  - build_cp2k
-#  - test_cp2k
+  - test_cp2k

 variables:
  PERSIST_IMAGE_NAME: $CSCS_REGISTRY_PATH/cp2k:$CI_COMMIT_SHORT_SHA
@ -23,14 +23,14 @@ build_cp2k_job:
  variables:
    DOCKERFILE: ci/docker/build_cp2k_psmp.Dockerfile

-#test_cp2k_job:
-#  extends: .container-runner-eiger
-#  stage: test_cp2k
-#  timeout: 2 hours
-#  image: $PERSIST_IMAGE_NAME
-#  script:
-#    - /opt/cp2k/tools/docker/scripts/test_regtest_cmake.sh toolchain psmp
-#  variables:
-#    OMP_NUM_THREADS: 2
-#    SLURM_NTASKS: 64
-#    SLURM_TIMELIMIT: "60:00"
+test_cp2k_job:
+  extends: .container-runner-eiger
+  stage: test_cp2k
+  timeout: 2 hours
+  image: $PERSIST_IMAGE_NAME
+  script:
+    - /opt/cp2k/bin/run_tests
+  variables:
+    OMP_NUM_THREADS: 2
+    SLURM_NTASKS: 64
+    SLURM_TIMELIMIT: "60:00"
--- a/ci/docker/build_cp2k_psmp.Dockerfile
+++ b/ci/docker/build_cp2k_psmp.Dockerfile
@ -1,14 +1,108 @@
+# Stage 1b: Build CP2K with CMake
 WORKDIR /opt/cp2k
-COPY ./src ./src
+COPY ./CMakeLists.txt .
+COPY ./cmake ./cmake
 COPY ./data ./data
+COPY ./src ./src
 COPY ./tests ./tests
 COPY ./tools/build_utils ./tools/build_utils
-COPY ./cmake ./cmake
-COPY ./CMakeLists.txt .

-# Build CP2K with CMake and run regression tests
-COPY ./tools/docker/scripts/build_cp2k_cmake.sh ./
-RUN /bin/bash -o pipefail -c " \
-    ./build_cp2k_cmake.sh toolchain psmp |& tee build_cp2k.log"
+# Run CMake
+RUN /bin/bash -c -o pipefail " \
+    TOOLCHAIN_DIR=/opt/cp2k/tools/toolchain; \
+    source ./cmake/cmake_cp2k.sh toolchain psmp |& tee cmake.log"

-#EOF
+# Compile CP2K
+WORKDIR /opt/cp2k/build
+RUN /bin/bash -c -o pipefail " \
+    source /opt/cp2k/tools/toolchain/install/setup; \
+    echo -en '\nCompiling CP2K ... '; \
+    if ninja --verbose &> ninja.log; then \
+      echo -e 'done\n'; \
+      echo -en 'Installing CP2K ... '; \
+      if ninja --verbose install &> install.log; then \
+        echo -e 'done\n'; \
+      else \
+        echo -e 'failed\n'; \
+        cat install.log; \
+      fi; \
+    else \
+      echo -e 'failed\n'; \
+      cat ninja.log; \
+    fi"
+
+# Update environment variables
+ENV LD_LIBRARY_PATH=/opt/cp2k/lib:${LD_LIBRARY} \
+    PATH=/opt/cp2k/bin:${PATH}
+
+# Collect components for installation
+WORKDIR /opt/cp2k/bin
+RUN /bin/bash -c -o pipefail " \
+    source /opt/cp2k/tools/toolchain/install/setup; \
+    mkdir -p /toolchain/install /toolchain/scripts; \
+    for libdir in \$(ldd /opt/cp2k/bin/cp2k.psmp | \
+                     grep /opt/cp2k/tools/toolchain/install | \
+                     awk '{print \$3}' | cut -d/ -f7 | \
+                     sort | uniq) setup; do \
+      cp -ar /opt/cp2k/tools/toolchain/install/\${libdir} /toolchain/install; \
+    done; \
+    cp /opt/cp2k/tools/toolchain/scripts/tool_kit.sh /toolchain/scripts"
+
+# Stage 2: Install CP2K
+FROM ubuntu:24.04 AS install
+
+# Install required packages
+RUN apt-get update -qq && apt-get install -qq --no-install-recommends \
+    g++ \
+    gcc \
+    gfortran \
+    python3 && rm -rf /var/lib/apt/lists/*
+
+# Install CP2K binaries and libraries
+WORKDIR /opt/cp2k
+COPY --from=build /opt/cp2k/bin ./bin
+COPY --from=build /opt/cp2k/lib ./lib
+
+# Install CP2K database files
+COPY --from=build /opt/cp2k/share ./share
+
+# Install CP2K regression tests
+COPY --from=build /opt/cp2k/tests ./tests
+COPY --from=build /opt/cp2k/src/grid/sample_tasks ./src/grid/sample_tasks
+
+# Create links to CP2K binaries
+WORKDIR /opt/cp2k/bin
+RUN ln -sf cp2k.psmp cp2k
+RUN ln -sf cp2k.psmp cp2k.popt
+RUN ln -sf cp2k.psmp cp2k_shell
+
+# Install shared libraries required by the CP2K binaries
+COPY --from=build /toolchain /opt/cp2k/tools/toolchain
+
+# Create entrypoint script file
+RUN printf "#!/bin/bash\n\
+ulimit -c 0 -s unlimited\n\
+\
+export OMP_STACKSIZE=64M\n\
+export LD_LIBRARY_PATH=/opt/cp2k/lib:\${LD_LIBRARY_PATH}\n\
+export PATH=/opt/cp2k/bin:\${PATH}\n\
+source /opt/cp2k/tools/toolchain/install/setup\n\
+\"\$@\"" \
+>/opt/cp2k/bin/entrypoint.sh && chmod 755 /opt/cp2k/bin/entrypoint.sh
+
+# Create shortcut for regression test
+RUN printf "/opt/cp2k/tests/do_regtest.py --maxtasks 12 --workbasedir /mnt \$* /opt/cp2k/bin psmp" \
+>/opt/cp2k/bin/run_tests && chmod 755 /opt/cp2k/bin/run_tests
+
+# Define entrypoint
+WORKDIR /mnt
+ENTRYPOINT ["/opt/cp2k/bin/entrypoint.sh"]
+CMD ["cp2k", "--help"]
+
+# Label docker image
+LABEL author="CP2K Developers" \
+      cp2k_version="master" \
+      container_type="production_psmp" \
+      dockerfile_generator_version="0.1"
+
+# EOF
--- a/ci/docker/build_deps_psmp.Dockerfile
+++ b/ci/docker/build_deps_psmp.Dockerfile
@ -1,27 +1,36 @@
+# Stage 1a: Build CP2K toolchain
 FROM ubuntu:24.04 AS build

-# Install requirements for the toolchain
-WORKDIR /opt/cp2k-toolchain
-COPY ./tools/toolchain/install_requirements*.sh ./
-RUN ./install_requirements.sh ubuntu:24.04
+# Install packages required for the CP2K toolchain build
+RUN apt-get update -qq && apt-get install -qq --no-install-recommends \
+    bzip2 \
+    ca-certificates \
+    g++ \
+    gcc \
+    gfortran \
+    git \
+    libtool \
+    libtool-bin \
+    make \
+    patch \
+    pkg-config \
+    python3 \
+    unzip \
+    wget \
+    zlib1g-dev

-# Install the toolchain
-RUN mkdir scripts
-COPY ./tools/toolchain/scripts/VERSION \
-     ./tools/toolchain/scripts/parse_if.py \
-     ./tools/toolchain/scripts/tool_kit.sh \
-     ./tools/toolchain/scripts/common_vars.sh \
-     ./tools/toolchain/scripts/signal_trap.sh \
-     ./tools/toolchain/scripts/get_openblas_arch.sh \
-     ./scripts/
-COPY ./tools/toolchain/install_cp2k_toolchain.sh .
-RUN ./install_cp2k_toolchain.sh \
+# Build CP2K toolchain
+COPY ./tools/toolchain /opt/cp2k/tools/toolchain
+WORKDIR /opt/cp2k/tools/toolchain
+RUN ./install_cp2k_toolchain.sh -j \
+    --dry-run \
    --install-all \
+    --no-arch-files \
+    --target-cpu=native \
    --with-gcc=system \
-    --dry-run
+    --with-mpich

-# Dry-run leaves behind config files for the followup install scripts
-# This breaks up the lengthy installation into smaller docker build steps
+# Perform toolchain build step-wise in stages after its initialization with dry-run
 COPY ./tools/toolchain/scripts/stage0/ ./scripts/stage0/
 RUN  ./scripts/stage0/install_stage0.sh && rm -rf ./build

@ -49,13 +58,7 @@ RUN  ./scripts/stage7/install_stage7.sh && rm -rf ./build
 COPY ./tools/toolchain/scripts/stage8/ ./scripts/stage8/
 RUN  ./scripts/stage8/install_stage8.sh && rm -rf ./build

-COPY ./tools/toolchain/scripts/arch_base.tmpl \
-     ./tools/toolchain/scripts/generate_arch_files.sh \
-     ./scripts/
-RUN ./scripts/generate_arch_files.sh && rm -rf ./build
+COPY ./tools/toolchain/scripts/stage9/ ./scripts/stage9/
+RUN  ./scripts/stage9/install_stage9.sh && rm -rf ./build

-# Install DBCSR
-COPY ./tools/docker/scripts/install_dbcsr.sh ./
-RUN ./install_dbcsr.sh toolchain psmp
-
-#EOF
+# EOF
--- a/ci/docker/test_cp2k_psmp.Dockerfile
+++ b/ci/docker/test_cp2k_psmp.Dockerfile
--- a/cmake/cmake_cp2k.sh
+++ b/cmake/cmake_cp2k.sh
@ -40,6 +40,7 @@ if [[ "${PROFILE}" == "spack" ]] && [[ "${VERSION}" == "psmp" ]]; then
  export Torch_DIR="/opt/spack/var/spack/environments/myenv/spack-env/view/lib/python3.11/site-packages/torch/share/cmake/Torch"
  cmake \
    -GNinja \
+    -DCMAKE_BUILD_TYPE="RelWithDebInfo" \
    -DCMAKE_C_FLAGS="-fno-lto" \
    -DCMAKE_Fortran_FLAGS="-fno-lto" \
    -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
@ -72,6 +73,7 @@ if [[ "${PROFILE}" == "spack" ]] && [[ "${VERSION}" == "psmp" ]]; then
 elif [[ "${PROFILE}" == "toolchain" ]] && [[ "${VERSION}" == "ssmp" ]]; then
  cmake \
    -GNinja \
+    -DCMAKE_BUILD_TYPE="RelWithDebInfo" \
    -DCMAKE_INSTALL_LIBDIR=lib \
    -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
    -DCP2K_BLAS_VENDOR=OpenBLAS \
@ -97,6 +99,7 @@ elif [[ "${PROFILE}" == "toolchain" ]] && [[ "${VERSION}" == "ssmp" ]]; then
 elif [[ "${PROFILE}" == "toolchain" ]] && [[ "${VERSION}" == "sdbg" ]]; then
  cmake \
    -GNinja \
+    -DCMAKE_BUILD_TYPE="RelWithDebInfo" \
    -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
    -DCP2K_BLAS_VENDOR=OpenBLAS \
    -DCP2K_DEBUG_MODE=ON \
@ -116,6 +119,7 @@ elif [[ "${PROFILE}" == "toolchain" ]] && [[ "${VERSION}" == "sdbg" ]]; then
 elif [[ "${PROFILE}" == "toolchain" ]] && [[ "${VERSION}" == "psmp" ]]; then
  cmake \
    -GNinja \
+    -DCMAKE_BUILD_TYPE="RelWithDebInfo" \
    -DCMAKE_INSTALL_LIBDIR=lib \
    -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
    -DCP2K_BLAS_VENDOR=OpenBLAS \
@ -146,6 +150,7 @@ elif [[ "${PROFILE}" == "toolchain" ]] && [[ "${VERSION}" == "psmp" ]]; then
 elif [[ "${PROFILE}" == "toolchain" ]] && [[ "${VERSION}" == "pdbg" ]]; then
  cmake \
    -GNinja \
+    -DCMAKE_BUILD_TYPE="RelWithDebInfo" \
    -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
    -DCP2K_BLAS_VENDOR=OpenBLAS \
    -DCP2K_DEBUG_MODE=ON \
@ -167,6 +172,7 @@ elif [[ "${PROFILE}" == "ubuntu" ]] && [[ "${VERSION}" == "ssmp" ]]; then
  # NOTE: libxc 5.2.3 is provided, CP2K requires libxc 7
  cmake \
    -GNinja \
+    -DCMAKE_BUILD_TYPE="RelWithDebInfo" \
    -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
    -DCP2K_BLAS_VENDOR=OpenBLAS \
    -DCP2K_USE_COSMA=OFF \
@ -188,6 +194,7 @@ elif [[ "${PROFILE}" == "ubuntu_i386" ]] && [[ "${VERSION}" == "ssmp" ]]; then
  # TODO fix spglib https://github.com/cp2k/cp2k/issues/3414
  cmake \
    -GNinja \
+    -DCMAKE_BUILD_TYPE="RelWithDebInfo" \
    -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
    -DCP2K_BLAS_VENDOR=OpenBLAS \
    -DCP2K_USE_COSMA=OFF \
@ -207,6 +214,7 @@ elif [[ "${PROFILE}" == "ubuntu_i386" ]] && [[ "${VERSION}" == "ssmp" ]]; then
 elif [[ "${PROFILE}" == "minimal" ]] && [[ "${VERSION}" == "ssmp" ]]; then
  cmake \
    -GNinja \
+    -DCMAKE_BUILD_TYPE="RelWithDebInfo" \
    -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
    -DCP2K_BLAS_VENDOR=OpenBLAS \
    -DCP2K_USE_COSMA=OFF \