diff --git a/ci/cscs_daint_spack_psmp.yaml b/ci/cscs_daint_spack_psmp.yaml index e1445effcb..e9b6bff9b5 100644 --- a/ci/cscs_daint_spack_psmp.yaml +++ b/ci/cscs_daint_spack_psmp.yaml @@ -31,15 +31,13 @@ build deps daint: "BASE_IMAGE", "CP2K_BUILD_TYPE", "CP2K_VERSION", - "LOG_LINES", - "MPICH_VERSION" + "LOG_LINES" ]' DOCKERFILE: ci/docker/build_${CI_JOB_STAGE}_${BUILDER}.Dockerfile BASE_IMAGE: ${UBUNTU_IMAGE_NAME} CP2K_BUILD_TYPE: all CP2K_VERSION: ${VERSION} LOG_LINES: 200 - MPICH_VERSION: 4.3.0 build cp2k daint: extends: .container-builder-cscs-gh200 diff --git a/ci/cscs_daint_toolchain_psmp.yaml b/ci/cscs_daint_toolchain_psmp.yaml index ba6228b1c6..a2fb657197 100644 --- a/ci/cscs_daint_toolchain_psmp.yaml +++ b/ci/cscs_daint_toolchain_psmp.yaml @@ -31,6 +31,7 @@ build deps daint: "BASE_IMAGE", "CP2K_BUILD_TYPE", "CP2K_VERSION", + "LIBFABRIC_VERSION", "LOG_LINES", "MPICH_VERSION" ]' @@ -38,6 +39,7 @@ build deps daint: BASE_IMAGE: ${UBUNTU_IMAGE_NAME} CP2K_BUILD_TYPE: minimal CP2K_VERSION: ${VERSION} + LIBFABRIC_VERSION: 1.22.0 LOG_LINES: 200 MPICH_VERSION: 4.3.0 diff --git a/ci/cscs_eiger_spack_psmp.yaml b/ci/cscs_eiger_spack_psmp.yaml index 97bed0d4a3..642a2293ea 100644 --- a/ci/cscs_eiger_spack_psmp.yaml +++ b/ci/cscs_eiger_spack_psmp.yaml @@ -32,15 +32,13 @@ build deps eiger: "BASE_IMAGE", "CP2K_BUILD_TYPE", "CP2K_VERSION", - "LOG_LINES", - "MPICH_VERSION" + "LOG_LINES" ]' DOCKERFILE: ci/docker/build_${CI_JOB_STAGE}_${BUILDER}.Dockerfile BASE_IMAGE: ${UBUNTU_IMAGE_NAME} CP2K_BUILD_TYPE: all CP2K_VERSION: ${VERSION} LOG_LINES: 200 - MPICH_VERSION: 3.4.3 build cp2k eiger: extends: .container-builder-cscs-zen2 @@ -103,9 +101,9 @@ benchmark cp2k 1 eiger node: stage: benchmark needs: ["build cp2k eiger","test cp2k eiger","regression test cp2k eiger"] timeout: 1h + image: ${CSCS_IMAGE_NAME} before_script: - export CP2K_DATA_DIR=/opt/cp2k/share/cp2k/data - image: ${CSCS_IMAGE_NAME} script: - cp2k /opt/cp2k/benchmarks/CI/H2O-128_md.inp variables: @@ -114,6 +112,7 @@ benchmark cp2k 1 eiger node: SLURM_DEBUG: 1 SLURM_HINT: nomultithread SLURM_JOB_NUM_NODES: 1 + SLURM_MPI_TYPE: pmi2 SLURM_NTASKS: 128 SLURM_NTASKS_PER_NODE: 128 SLURM_TIMELIMIT: 30 @@ -127,8 +126,6 @@ benchmark cp2k 2 eiger nodes: image: ${CSCS_IMAGE_NAME} before_script: - export CP2K_DATA_DIR=/opt/cp2k/share/cp2k/data - # This setting is currently needed to allow for multi-node runs, but it degrades the performance by a factor of two or more - - export MPICH_NOLOCAL=1 script: - cp2k /opt/cp2k/benchmarks/CI/H2O-512_md.inp variables: @@ -137,6 +134,7 @@ benchmark cp2k 2 eiger nodes: SLURM_DEBUG: 1 SLURM_HINT: nomultithread SLURM_JOB_NUM_NODES: 2 + SLURM_MPI_TYPE: pmi2 SLURM_NTASKS: 256 SLURM_NTASKS_PER_NODE: 128 SLURM_TIMELIMIT: 30 diff --git a/ci/cscs_eiger_toolchain_psmp.yaml b/ci/cscs_eiger_toolchain_psmp.yaml index 3e99f220ee..ea443f2938 100644 --- a/ci/cscs_eiger_toolchain_psmp.yaml +++ b/ci/cscs_eiger_toolchain_psmp.yaml @@ -32,6 +32,7 @@ build deps eiger: "BASE_IMAGE", "CP2K_BUILD_TYPE", "CP2K_VERSION", + "LIBFABRIC_VERSION", "LOG_LINES", "MPICH_VERSION" ]' @@ -39,6 +40,7 @@ build deps eiger: BASE_IMAGE: ${UBUNTU_IMAGE_NAME} CP2K_BUILD_TYPE: all CP2K_VERSION: ${VERSION} + LIBFABRIC_VERSION: 1.22.0 LOG_LINES: 200 MPICH_VERSION: 3.4.3 @@ -114,6 +116,7 @@ benchmark cp2k 1 eiger node: SLURM_DEBUG: 1 SLURM_HINT: nomultithread SLURM_JOB_NUM_NODES: 1 + SLURM_MPI_TYPE: pmi2 SLURM_NTASKS: 128 SLURM_NTASKS_PER_NODE: 128 SLURM_TIMELIMIT: 30 @@ -127,8 +130,6 @@ benchmark cp2k 2 eiger nodes: image: ${CSCS_IMAGE_NAME} before_script: - export CP2K_DATA_DIR=/opt/cp2k/share/cp2k/data - # This setting is currently needed to allow for multi-node runs, but it degrades the performance by a factor of two or more - - export MPICH_NOLOCAL=1 script: - cp2k /opt/cp2k/benchmarks/CI/H2O-512_md.inp variables: @@ -137,6 +138,7 @@ benchmark cp2k 2 eiger nodes: SLURM_DEBUG: 1 SLURM_HINT: nomultithread SLURM_JOB_NUM_NODES: 2 + SLURM_MPI_TYPE: pmi2 SLURM_NTASKS: 256 SLURM_NTASKS_PER_NODE: 128 SLURM_TIMELIMIT: 30 diff --git a/ci/docker/build_deps_spack.Dockerfile b/ci/docker/build_deps_spack.Dockerfile index 96dcf4ad89..d76704556e 100644 --- a/ci/docker/build_deps_spack.Dockerfile +++ b/ci/docker/build_deps_spack.Dockerfile @@ -40,6 +40,19 @@ RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ xz-utils \ zstd && rm -rf /var/lib/apt/lists/* +# Create dummy xpmem library for the MPICH build. At runtime the +# container engine will inject the xpmem library from the host system +RUN git clone https://github.com/hpc/xpmem \ + && cd xpmem/lib \ + && gcc -I../include -shared -o libxpmem.so.1 libxpmem.c \ + && ln -s libxpmem.so.1 libxpmem.so \ + && mkdir -p /opt/spack/lib /opt/spack/include \ + && mv libxpmem.so* /opt/spack/lib \ + && cp ../include/xpmem.h /opt/spack/include/ \ + && ldconfig /opt/spack/lib \ + && cd ../../ \ + && rm -rf xpmem + # Retrieve the number of available CPU cores ARG NUM_PROCS ENV NUM_PROCS=${NUM_PROCS:-32} @@ -80,12 +93,6 @@ ENV CP2K_BUILD_TYPE=${CP2K_BUILD_TYPE:-minimal} COPY ./tools/spack/cp2k_deps_${CP2K_BUILD_TYPE}_${CP2K_VERSION}.yaml ./ COPY ./tools/spack/cp2k_dev_repo ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev_repo/ RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev_repo/ - -# Sarus containers must be dynamically linked to an MPI implementation that is ABI-compatible -# with the MPI on the compute nodes at CSCS like MPICH@3 -ARG MPICH_VERSION -ENV MPICH_VERSION=${MPICH_VERSION:-3.4.3} -RUN sed -i -e "s/mpich@[0-9.]*/mpich@${MPICH_VERSION}/" cp2k_deps_${CP2K_BUILD_TYPE}_${CP2K_VERSION}.yaml RUN spack env create myenv cp2k_deps_${CP2K_BUILD_TYPE}_${CP2K_VERSION}.yaml && \ spack -e myenv repo list diff --git a/ci/docker/build_deps_toolchain.Dockerfile b/ci/docker/build_deps_toolchain.Dockerfile index c47978f540..052602b68e 100644 --- a/ci/docker/build_deps_toolchain.Dockerfile +++ b/ci/docker/build_deps_toolchain.Dockerfile @@ -46,14 +46,41 @@ ENV CP2K_VERSION=${CP2K_VERSION:-psmp} ARG CP2K_BUILD_TYPE ENV CP2K_BUILD_TYPE=${CP2K_BUILD_TYPE:-minimal} +# Create dummy xpmem library for the MPICH build. At runtime the +# container engine will inject the xpmem library from the host system +RUN git clone https://github.com/hpc/xpmem \ + && cd xpmem/lib \ + && gcc -I../include -shared -o libxpmem.so.1 libxpmem.c \ + && ln -s libxpmem.so.1 libxpmem.so \ + && mkdir -p /opt/spack/lib /opt/spack/include \ + && mv libxpmem.so* /opt/spack/lib \ + && cp ../include/xpmem.h /opt/spack/include/ \ + && ldconfig /opt/spack/lib \ + && cd ../../ \ + && rm -rf xpmem + +# Install libfabric version currently used by the host system +ARG LIBFABRIC_VERSION +ENV LIBFABRIC_VERSION=${LIBFABRIC_VERSION:-1.22.0} +RUN wget -q https://github.com/ofiwg/libfabric/archive/v${LIBFABRIC_VERSION}.tar.gz \ + && tar -xf v${LIBFABRIC_VERSION}.tar.gz \ + && cd libfabric-${LIBFABRIC_VERSION} \ + && ./autogen.sh \ + && ./configure --prefix=/opt/spack \ + && make -j ${NUM_PROCS} \ + && make install \ + && ldconfig \ + && cd .. \ + && rm -rf v${LIBFABRIC_VERSION}.tar.gz libfabric-${LIBFABRIC_VERSION} + # Install an MPI version ABI-compatible with the host MPI on Cray at CSCS ARG MPICH_VERSION -ENV MPICH_VERSION=${MPICH_VERSION:-3.4.3} +ENV MPICH_VERSION=${MPICH_VERSION:-4.3.0} RUN /bin/bash -c -o pipefail "[[ "${CP2K_VERSION}" == "psmp" ]] && (\ wget -q https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz; \ tar -xf mpich-${MPICH_VERSION}.tar.gz; \ cd mpich-${MPICH_VERSION}; \ - ./configure --prefix='/usr/local' --enable-fast=all,O3 --with-device=ch3 \ + ./configure --prefix='/usr/local' --enable-fast=all,O3 --with-device=ch4:ofi --with-libfabric=/opt/spack --with-xpmem=/opt/spack \ FFLAGS='${FFLAGS} -fallow-argument-mismatch' \ FCFLAGS='${FCFLAGS} -fallow-argument-mismatch' \ &>configure.log || tail -n ${LOG_LINES} configure.log; \ diff --git a/tools/spack/cp2k_deps_all_psmp.yaml b/tools/spack/cp2k_deps_all_psmp.yaml index 570a3292bf..6e1aaac001 100644 --- a/tools/spack/cp2k_deps_all_psmp.yaml +++ b/tools/spack/cp2k_deps_all_psmp.yaml @@ -8,6 +8,7 @@ spack: root: /opt/spack packages: all: + buildable: true prefer: - ~cuda - +mpi @@ -19,8 +20,12 @@ spack: - mpich mpich: require: - - device=ch3 - - netmod=tcp + - +xpmem + xpmem: + buildable: false + externals: + - spec: xpmem@master + prefix: /opt/spack blas: require: - openblas @@ -95,11 +100,14 @@ spack: - +hdf5 - build_system=cmake specs: - - "mpich@4.3.0" - - "openblas@0.3.29" - - "netlib-scalapack@2.2.2" - - "cosma@2.7.0" + # Minimal - "dbcsr@2.8.0" + - "libfabric@1.22.0" + - "mpich@4.3.0" + - "netlib-scalapack@2.2.2" + - "openblas@0.3.29" + # All + - "cosma@2.7.0" - "deepmdkit@3.0.2" - "dftd4@3.7.0" - "dla-future@0.10.0" diff --git a/tools/spack/cp2k_deps_all_ssmp.yaml b/tools/spack/cp2k_deps_all_ssmp.yaml index f4980c4d63..6c1ec202de 100644 --- a/tools/spack/cp2k_deps_all_ssmp.yaml +++ b/tools/spack/cp2k_deps_all_ssmp.yaml @@ -8,6 +8,7 @@ spack: root: /opt/spack packages: all: + buildable: true prefer: - ~cuda - ~mpi @@ -64,8 +65,10 @@ spack: - +hdf5 - build_system=cmake specs: - - "openblas@0.3.29" + # Minimal - "dbcsr@2.8.0" + - "openblas@0.3.29" + # All - "deepmdkit@3.0.2" - "dftd4@3.7.0" - "fftw@3.3.10" diff --git a/tools/spack/cp2k_deps_minimal_psmp.yaml b/tools/spack/cp2k_deps_minimal_psmp.yaml index 315d3e3b70..4f5a8a2790 100644 --- a/tools/spack/cp2k_deps_minimal_psmp.yaml +++ b/tools/spack/cp2k_deps_minimal_psmp.yaml @@ -8,16 +8,24 @@ spack: root: /opt/spack packages: all: + buildable: true prefer: + - ~cuda - +mpi + - +openmp - +pic + - ~rocm mpi: require: - mpich mpich: require: - - device=ch3 - - netmod=tcp + - +xpmem + xpmem: + buildable: false + externals: + - spec: xpmem@master + prefix: /opt/spack blas: require: - openblas @@ -39,6 +47,7 @@ spack: - smm=blas specs: - "dbcsr@2.8.0" + - "libfabric@1.22.0" - "mpich@4.3.0" - "netlib-scalapack@2.2.2" - "openblas@0.3.29" diff --git a/tools/spack/cp2k_deps_minimal_ssmp.yaml b/tools/spack/cp2k_deps_minimal_ssmp.yaml index 105d4abba0..2407489bbe 100644 --- a/tools/spack/cp2k_deps_minimal_ssmp.yaml +++ b/tools/spack/cp2k_deps_minimal_ssmp.yaml @@ -8,9 +8,13 @@ spack: root: /opt/spack packages: all: + buildable: true prefer: + - ~cuda - ~mpi + - +openmp - +pic + - ~rocm blas: require: - openblas