comp-chem/cp2k
1
0
Fork 0
mirror of https://github.com/cp2k/cp2k.git synced 2026-07-27 13:45:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Enable efficient multi-node CI runs

Browse source 
Use MPICH with the libfabric of the Cray host system
This commit is contained in:
Matthias Krack 2025-07-16 10:45:06 +02:00
parent 405ec67042
commit ef2364c498
10 changed files with 86 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

4
ci/cscs_daint_spack_psmp.yaml
View file

@ -31,15 +31,13 @@ build deps daint:
"BASE_IMAGE",
"CP2K_BUILD_TYPE",
"CP2K_VERSION",
"LOG_LINES",
"MPICH_VERSION"
"LOG_LINES"
]'
DOCKERFILE: ci/docker/build_${CI_JOB_STAGE}_${BUILDER}.Dockerfile
BASE_IMAGE: ${UBUNTU_IMAGE_NAME}
CP2K_BUILD_TYPE: all
CP2K_VERSION: ${VERSION}
LOG_LINES: 200
MPICH_VERSION: 4.3.0
build cp2k daint:
extends: .container-builder-cscs-gh200

2
ci/cscs_daint_toolchain_psmp.yaml
View file

@ -31,6 +31,7 @@ build deps daint:
"BASE_IMAGE",
"CP2K_BUILD_TYPE",
"CP2K_VERSION",
"LIBFABRIC_VERSION",
"LOG_LINES",
"MPICH_VERSION"
]'
@ -38,6 +39,7 @@ build deps daint:
BASE_IMAGE: ${UBUNTU_IMAGE_NAME}
CP2K_BUILD_TYPE: minimal
CP2K_VERSION: ${VERSION}
LIBFABRIC_VERSION: 1.22.0
LOG_LINES: 200
MPICH_VERSION: 4.3.0

10
ci/cscs_eiger_spack_psmp.yaml
View file

@ -32,15 +32,13 @@ build deps eiger:
"BASE_IMAGE",
"CP2K_BUILD_TYPE",
"CP2K_VERSION",
"LOG_LINES",
"MPICH_VERSION"
"LOG_LINES"
]'
DOCKERFILE: ci/docker/build_${CI_JOB_STAGE}_${BUILDER}.Dockerfile
BASE_IMAGE: ${UBUNTU_IMAGE_NAME}
CP2K_BUILD_TYPE: all
CP2K_VERSION: ${VERSION}
LOG_LINES: 200
MPICH_VERSION: 3.4.3
build cp2k eiger:
extends: .container-builder-cscs-zen2
@ -103,9 +101,9 @@ benchmark cp2k 1 eiger node:
stage: benchmark
needs: ["build cp2k eiger","test cp2k eiger","regression test cp2k eiger"]
timeout: 1h
image: ${CSCS_IMAGE_NAME}
before_script:
- export CP2K_DATA_DIR=/opt/cp2k/share/cp2k/data
image: ${CSCS_IMAGE_NAME}
script:
- cp2k /opt/cp2k/benchmarks/CI/H2O-128_md.inp
variables:
@ -114,6 +112,7 @@ benchmark cp2k 1 eiger node:
SLURM_DEBUG: 1
SLURM_HINT: nomultithread
SLURM_JOB_NUM_NODES: 1
SLURM_MPI_TYPE: pmi2
SLURM_NTASKS: 128
SLURM_NTASKS_PER_NODE: 128
SLURM_TIMELIMIT: 30
@ -127,8 +126,6 @@ benchmark cp2k 2 eiger nodes:
image: ${CSCS_IMAGE_NAME}
before_script:
- export CP2K_DATA_DIR=/opt/cp2k/share/cp2k/data
# This setting is currently needed to allow for multi-node runs, but it degrades the performance by a factor of two or more
- export MPICH_NOLOCAL=1
script:
- cp2k /opt/cp2k/benchmarks/CI/H2O-512_md.inp
variables:
@ -137,6 +134,7 @@ benchmark cp2k 2 eiger nodes:
SLURM_DEBUG: 1
SLURM_HINT: nomultithread
SLURM_JOB_NUM_NODES: 2
SLURM_MPI_TYPE: pmi2
SLURM_NTASKS: 256
SLURM_NTASKS_PER_NODE: 128
SLURM_TIMELIMIT: 30

6
ci/cscs_eiger_toolchain_psmp.yaml
View file

@ -32,6 +32,7 @@ build deps eiger:
"BASE_IMAGE",
"CP2K_BUILD_TYPE",
"CP2K_VERSION",
"LIBFABRIC_VERSION",
"LOG_LINES",
"MPICH_VERSION"
]'
@ -39,6 +40,7 @@ build deps eiger:
BASE_IMAGE: ${UBUNTU_IMAGE_NAME}
CP2K_BUILD_TYPE: all
CP2K_VERSION: ${VERSION}
LIBFABRIC_VERSION: 1.22.0
LOG_LINES: 200
MPICH_VERSION: 3.4.3
@ -114,6 +116,7 @@ benchmark cp2k 1 eiger node:
SLURM_DEBUG: 1
SLURM_HINT: nomultithread
SLURM_JOB_NUM_NODES: 1
SLURM_MPI_TYPE: pmi2
SLURM_NTASKS: 128
SLURM_NTASKS_PER_NODE: 128
SLURM_TIMELIMIT: 30
@ -127,8 +130,6 @@ benchmark cp2k 2 eiger nodes:
image: ${CSCS_IMAGE_NAME}
before_script:
- export CP2K_DATA_DIR=/opt/cp2k/share/cp2k/data
# This setting is currently needed to allow for multi-node runs, but it degrades the performance by a factor of two or more
- export MPICH_NOLOCAL=1
script:
- cp2k /opt/cp2k/benchmarks/CI/H2O-512_md.inp
variables:
@ -137,6 +138,7 @@ benchmark cp2k 2 eiger nodes:
SLURM_DEBUG: 1
SLURM_HINT: nomultithread
SLURM_JOB_NUM_NODES: 2
SLURM_MPI_TYPE: pmi2
SLURM_NTASKS: 256
SLURM_NTASKS_PER_NODE: 128
SLURM_TIMELIMIT: 30

19
ci/docker/build_deps_spack.Dockerfile
View file

@ -40,6 +40,19 @@ RUN apt-get update -qq && apt-get install -qq --no-install-recommends \
xz-utils \
zstd && rm -rf /var/lib/apt/lists/*
# Create dummy xpmem library for the MPICH build. At runtime the
# container engine will inject the xpmem library from the host system
RUN git clone https://github.com/hpc/xpmem \
&& cd xpmem/lib \
&& gcc -I../include -shared -o libxpmem.so.1 libxpmem.c \
&& ln -s libxpmem.so.1 libxpmem.so \
&& mkdir -p /opt/spack/lib /opt/spack/include \
&& mv libxpmem.so* /opt/spack/lib \
&& cp ../include/xpmem.h /opt/spack/include/ \
&& ldconfig /opt/spack/lib \
&& cd ../../ \
&& rm -rf xpmem
# Retrieve the number of available CPU cores
ARG NUM_PROCS
ENV NUM_PROCS=${NUM_PROCS:-32}
@ -80,12 +93,6 @@ ENV CP2K_BUILD_TYPE=${CP2K_BUILD_TYPE:-minimal}
COPY ./tools/spack/cp2k_deps_${CP2K_BUILD_TYPE}_${CP2K_VERSION}.yaml ./
COPY ./tools/spack/cp2k_dev_repo ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev_repo/
RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev_repo/
# Sarus containers must be dynamically linked to an MPI implementation that is ABI-compatible
# with the MPI on the compute nodes at CSCS like MPICH@3
ARG MPICH_VERSION
ENV MPICH_VERSION=${MPICH_VERSION:-3.4.3}
RUN sed -i -e "s/mpich@[0-9.]*/mpich@${MPICH_VERSION}/" cp2k_deps_${CP2K_BUILD_TYPE}_${CP2K_VERSION}.yaml
RUN spack env create myenv cp2k_deps_${CP2K_BUILD_TYPE}_${CP2K_VERSION}.yaml && \
spack -e myenv repo list

31
ci/docker/build_deps_toolchain.Dockerfile
View file

@ -46,14 +46,41 @@ ENV CP2K_VERSION=${CP2K_VERSION:-psmp}
ARG CP2K_BUILD_TYPE
ENV CP2K_BUILD_TYPE=${CP2K_BUILD_TYPE:-minimal}
# Create dummy xpmem library for the MPICH build. At runtime the
# container engine will inject the xpmem library from the host system
RUN git clone https://github.com/hpc/xpmem \
&& cd xpmem/lib \
&& gcc -I../include -shared -o libxpmem.so.1 libxpmem.c \
&& ln -s libxpmem.so.1 libxpmem.so \
&& mkdir -p /opt/spack/lib /opt/spack/include \
&& mv libxpmem.so* /opt/spack/lib \
&& cp ../include/xpmem.h /opt/spack/include/ \
&& ldconfig /opt/spack/lib \
&& cd ../../ \
&& rm -rf xpmem
# Install libfabric version currently used by the host system
ARG LIBFABRIC_VERSION
ENV LIBFABRIC_VERSION=${LIBFABRIC_VERSION:-1.22.0}
RUN wget -q https://github.com/ofiwg/libfabric/archive/v${LIBFABRIC_VERSION}.tar.gz \
&& tar -xf v${LIBFABRIC_VERSION}.tar.gz \
&& cd libfabric-${LIBFABRIC_VERSION} \
&& ./autogen.sh \
&& ./configure --prefix=/opt/spack \
&& make -j ${NUM_PROCS} \
&& make install \
&& ldconfig \
&& cd .. \
&& rm -rf v${LIBFABRIC_VERSION}.tar.gz libfabric-${LIBFABRIC_VERSION}
# Install an MPI version ABI-compatible with the host MPI on Cray at CSCS
ARG MPICH_VERSION
ENV MPICH_VERSION=${MPICH_VERSION:-3.4.3}
ENV MPICH_VERSION=${MPICH_VERSION:-4.3.0}
RUN /bin/bash -c -o pipefail "[[ "${CP2K_VERSION}" == "psmp" ]] && (\
wget -q https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz; \
tar -xf mpich-${MPICH_VERSION}.tar.gz; \
cd mpich-${MPICH_VERSION}; \
./configure --prefix='/usr/local' --enable-fast=all,O3 --with-device=ch3 \
./configure --prefix='/usr/local' --enable-fast=all,O3 --with-device=ch4:ofi --with-libfabric=/opt/spack --with-xpmem=/opt/spack \
FFLAGS='${FFLAGS} -fallow-argument-mismatch' \
FCFLAGS='${FCFLAGS} -fallow-argument-mismatch' \
&>configure.log || tail -n ${LOG_LINES} configure.log; \

20
tools/spack/cp2k_deps_all_psmp.yaml
View file

@ -8,6 +8,7 @@ spack:
root: /opt/spack
packages:
all:
buildable: true
prefer:
- ~cuda
- +mpi
@ -19,8 +20,12 @@ spack:
- mpich
mpich:
require:
- device=ch3
- netmod=tcp
- +xpmem
xpmem:
buildable: false
externals:
- spec: xpmem@master
prefix: /opt/spack
blas:
require:
- openblas
@ -95,11 +100,14 @@ spack:
- +hdf5
- build_system=cmake
specs:
- "mpich@4.3.0"
- "openblas@0.3.29"
- "netlib-scalapack@2.2.2"
- "cosma@2.7.0"
# Minimal
- "dbcsr@2.8.0"
- "libfabric@1.22.0"
- "mpich@4.3.0"
- "netlib-scalapack@2.2.2"
- "openblas@0.3.29"
# All
- "cosma@2.7.0"
- "deepmdkit@3.0.2"
- "dftd4@3.7.0"
- "dla-future@0.10.0"

5
tools/spack/cp2k_deps_all_ssmp.yaml
View file

@ -8,6 +8,7 @@ spack:
root: /opt/spack
packages:
all:
buildable: true
prefer:
- ~cuda
- ~mpi
@ -64,8 +65,10 @@ spack:
- +hdf5
- build_system=cmake
specs:
- "openblas@0.3.29"
# Minimal
- "dbcsr@2.8.0"
- "openblas@0.3.29"
# All
- "deepmdkit@3.0.2"
- "dftd4@3.7.0"
- "fftw@3.3.10"

13
tools/spack/cp2k_deps_minimal_psmp.yaml
View file

@ -8,16 +8,24 @@ spack:
root: /opt/spack
packages:
all:
buildable: true
prefer:
- ~cuda
- +mpi
- +openmp
- +pic
- ~rocm
mpi:
require:
- mpich
mpich:
require:
- device=ch3
- netmod=tcp
- +xpmem
xpmem:
buildable: false
externals:
- spec: xpmem@master
prefix: /opt/spack
blas:
require:
- openblas
@ -39,6 +47,7 @@ spack:
- smm=blas
specs:
- "dbcsr@2.8.0"
- "libfabric@1.22.0"
- "mpich@4.3.0"
- "netlib-scalapack@2.2.2"
- "openblas@0.3.29"

4
tools/spack/cp2k_deps_minimal_ssmp.yaml
View file

@ -8,9 +8,13 @@ spack:
root: /opt/spack
packages:
all:
buildable: true
prefer:
- ~cuda
- ~mpi
- +openmp
- +pic
- ~rocm
blas:
require:
- openblas