diff --git a/ci/cscs_eiger_spack_psmp.yaml b/ci/cscs_eiger_spack_psmp.yaml index bf86b2f8fd..f9693a47fc 100644 --- a/ci/cscs_eiger_spack_psmp.yaml +++ b/ci/cscs_eiger_spack_psmp.yaml @@ -4,7 +4,7 @@ include: stages: - deps - cp2k - - test + # - test - benchmark variables: @@ -13,7 +13,7 @@ variables: VERSION: psmp TAG: "${BUILDER}_${VERSION}" UBUNTU_IMAGE_NAME: "ubuntu:24.04" - CSCS_IMAGE_NAME: "${CSCS_REGISTRY_PATH}/${REPOSITORY}:${TAG}_${CI_COMMIT_SHORT_SHA}" + CSCS_IMAGE_NAME: "${CSCS_REGISTRY_PATH}/public/${REPOSITORY}:${TAG}_${CI_COMMIT_SHORT_SHA}" DOCKERHUB_IMAGE_NAME: "docker.io/${DOCKERHUB_USERNAME}/${REPOSITORY}:${TAG}" build deps eiger: @@ -21,7 +21,11 @@ build deps eiger: stage: deps timeout: 1h before_script: - - DEPS_IMAGE_TAG=`cat ci/cscs_eiger_${TAG}.yaml ci/docker/build_${CI_JOB_STAGE}_${BUILDER}.Dockerfile $(find tools/spack -type f) $(find cmake -type f) | sha256sum - | head -c 16` + - TAG_BASEIMG=`echo ${BASE_IMAGE##*/} | sed 's/[:]//g'` + - TAG_DOCKER=`sha256sum ci/docker/build_${CI_JOB_STAGE}_${BUILDER}.Dockerfile | head -c 16` + - TAG_SPACK=`cat $(find tools/spack -type f) | sha256sum - | head -c 16` + - TAG_ALPS=`if [ ${USE_ALPS_SPACK_REPO} -eq 1 ]; then echo "alps"; else echo ""; fi` + - DEPS_IMAGE_TAG=${TAG_BASEIMG}-${TAG_DOCKER}-${TAG_SPACK}${TAG_ALPS} - export PERSIST_IMAGE_NAME=${CSCS_REGISTRY_PATH}/base/${REPOSITORY}:${TAG}_${DEPS_IMAGE_TAG} - echo "DEPS_IMAGE_NAME=${PERSIST_IMAGE_NAME}" | tee build.env artifacts: @@ -31,12 +35,14 @@ build deps eiger: DOCKER_BUILD_ARGS: '[ "BASE_IMAGE", "CP2K_VERSION", + "USE_ALPS_SPACK_REPO", "LOG_LINES" ]' DOCKERFILE: ci/docker/build_${CI_JOB_STAGE}_${BUILDER}.Dockerfile BASE_IMAGE: ${UBUNTU_IMAGE_NAME} CP2K_VERSION: ${VERSION} LOG_LINES: 200 + USE_ALPS_SPACK_REPO: 1 build cp2k eiger: extends: .container-builder-cscs-zen2 @@ -47,7 +53,7 @@ build cp2k eiger: - echo ${DOCKERHUB_TOKEN} | podman login docker.io --username ${DOCKERHUB_USERNAME} --password-stdin after_script: - podman images - - podman push ${PERSIST_IMAGE_NAME} ${DOCKERHUB_IMAGE_NAME} +# - podman push ${PERSIST_IMAGE_NAME} ${DOCKERHUB_IMAGE_NAME} variables: DOCKER_BUILD_ARGS: '[ "BASE_IMAGE", @@ -58,49 +64,50 @@ build cp2k eiger: DEPS_IMAGE: ${DEPS_IMAGE_NAME} PERSIST_IMAGE_NAME: ${CSCS_IMAGE_NAME} -test cp2k eiger: - extends: .baremetal-runner-eiger-zen2 - stage: test - needs: ["build cp2k eiger"] - timeout: 1h - script: - - podman run --shm-size=1g -v ${PWD}:/mnt ${DOCKERHUB_IMAGE_NAME} mpiexec -genv OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} -n ${SLURM_NTASKS} cp2k /opt/cp2k/benchmarks/CI/H2O-32_md.inp - variables: - SLURM_CONSTRAINT: mc - SLURM_CPUS_PER_TASK: 4 - SLURM_DEBUG: 1 - SLURM_HINT: nomultithread - SLURM_JOB_NUM_NODES: 1 - SLURM_NTASKS: 8 - SLURM_NTASKS_PER_NODE: 8 - SLURM_TIMELIMIT: 10 - USE_MPI: YES - -regression test cp2k eiger: - extends: .baremetal-runner-eiger-zen2 - stage: test - needs: ["build cp2k eiger","test cp2k eiger"] - timeout: 1h - script: - - git --no-pager log -1 --pretty="%nCommitSHA:%x20%H%nCommitTime:%x20%ci%nCommitAuthor:%x20%an%nCommitSubject:%x20%s%n" - - podman run --shm-size=1g -v ${PWD}:/mnt ${DOCKERHUB_IMAGE_NAME} /opt/cp2k/tests/do_regtest.py --maxtasks ${SLURM_NTASKS} --mpiexec "mpiexec -bind-to none" --keepalive /opt/cp2k/bin ${VERSION} 2>&1 || true - variables: - SLURM_CONSTRAINT: mc - SLURM_DEBUG: 1 - SLURM_HINT: nomultithread - SLURM_JOB_NUM_NODES: 1 - SLURM_NTASKS: 128 - SLURM_TIMELIMIT: 30 - USE_MPI: YES +# test cp2k eiger: +# extends: .baremetal-runner-eiger-zen2 +# stage: test +# needs: ["build cp2k eiger"] +# timeout: 1h +# script: +# - podman run --shm-size=1g -v ${PWD}:/mnt ${CSCS_IMAGE_NAME} mpiexec -genv OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} -n ${SLURM_NTASKS} cp2k /opt/cp2k/benchmarks/CI/H2O-32_md.inp +# variables: +# SLURM_CONSTRAINT: mc +# SLURM_CPUS_PER_TASK: 4 +# SLURM_DEBUG: 1 +# SLURM_HINT: nomultithread +# SLURM_JOB_NUM_NODES: 1 +# SLURM_NTASKS: 8 +# SLURM_NTASKS_PER_NODE: 8 +# SLURM_TIMELIMIT: 10 +# USE_MPI: YES +# +# regression test cp2k eiger: +# extends: .baremetal-runner-eiger-zen2 +# stage: test +# needs: ["build cp2k eiger","test cp2k eiger"] +# timeout: 1h +# script: +# - git --no-pager log -1 --pretty="%nCommitSHA:%x20%H%nCommitTime:%x20%ci%nCommitAuthor:%x20%an%nCommitSubject:%x20%s%n" +# - podman run --shm-size=1g -v ${PWD}:/mnt ${CSCS_IMAGE_NAME} /opt/cp2k/tests/do_regtest.py --maxtasks ${SLURM_NTASKS} --mpiexec "mpiexec -bind-to none" --keepalive /opt/cp2k/bin ${VERSION} 2>&1 || true +# variables: +# SLURM_CONSTRAINT: mc +# SLURM_DEBUG: 1 +# SLURM_HINT: nomultithread +# SLURM_JOB_NUM_NODES: 1 +# SLURM_NTASKS: 128 +# SLURM_TIMELIMIT: 30 +# USE_MPI: YES benchmark cp2k 1 eiger node: extends: .container-runner-eiger-zen2 stage: benchmark - needs: ["build cp2k eiger","test cp2k eiger","regression test cp2k eiger"] + #needs: ["build cp2k eiger","test cp2k eiger","regression test cp2k eiger"] timeout: 1h image: ${CSCS_IMAGE_NAME} before_script: - export CP2K_DATA_DIR=/opt/cp2k/share/cp2k/data + - ldd $(which cp2k) script: - cp2k /opt/cp2k/benchmarks/CI/H2O-128_md.inp variables: @@ -109,7 +116,7 @@ benchmark cp2k 1 eiger node: SLURM_DEBUG: 1 SLURM_HINT: nomultithread SLURM_JOB_NUM_NODES: 1 - SLURM_MPI_TYPE: pmi2 + SLURM_MPI_TYPE: cray_shasta SLURM_NTASKS: 128 SLURM_NTASKS_PER_NODE: 128 SLURM_TIMELIMIT: 30 @@ -118,11 +125,12 @@ benchmark cp2k 1 eiger node: benchmark cp2k 2 eiger nodes: extends: .container-runner-eiger-zen2 stage: benchmark - needs: ["build cp2k eiger","test cp2k eiger","regression test cp2k eiger"] + #needs: ["build cp2k eiger","test cp2k eiger","regression test cp2k eiger"] timeout: 1h image: ${CSCS_IMAGE_NAME} before_script: - export CP2K_DATA_DIR=/opt/cp2k/share/cp2k/data + - ldd $(which cp2k) script: - cp2k /opt/cp2k/benchmarks/CI/H2O-512_md.inp variables: @@ -131,7 +139,7 @@ benchmark cp2k 2 eiger nodes: SLURM_DEBUG: 1 SLURM_HINT: nomultithread SLURM_JOB_NUM_NODES: 2 - SLURM_MPI_TYPE: pmi2 + SLURM_MPI_TYPE: cray_shasta SLURM_NTASKS: 256 SLURM_NTASKS_PER_NODE: 128 SLURM_TIMELIMIT: 30 diff --git a/ci/docker/build_deps_spack.Dockerfile b/ci/docker/build_deps_spack.Dockerfile index 4b772ebef6..3c2748362b 100644 --- a/ci/docker/build_deps_spack.Dockerfile +++ b/ci/docker/build_deps_spack.Dockerfile @@ -79,6 +79,16 @@ ENV PATH="${SPACK_ROOT}/bin:${PATH}" # Add Spack packages builtin repository RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/builtin +ARG USE_ALPS_SPACK_REPO +ENV USE_ALPS_SPACK_REPO=${USE_ALPS_SPACK_REPO:-0} +ENV ALPS_SPACK_REPO_SHA="57d5c2f39e4f0cf6acc8779dcaedc1e90264d639" +RUN if [ "${USE_ALPS_SPACK_REPO}" -ne 0 ]; then \ + mkdir -p /root/alps-cluster-config && \ + wget -qO- "https://api.github.com/repos/eth-cscs/alps-cluster-config/tarball/$ALPS_SPACK_REPO_SHA" | \ + tar --strip-components=1 -xz -C /root/alps-cluster-config && \ + spack repo add --scope site /root/alps-cluster-config/site/spack_repo/alps; \ + fi + # Find all compilers RUN spack compiler find @@ -89,13 +99,32 @@ RUN spack external find --all --not-buildable ARG CP2K_VERSION ENV CP2K_VERSION=${CP2K_VERSION:-psmp} COPY ./tools/spack/cp2k_deps_${CP2K_VERSION}.yaml ./ -RUN sed -e "s/~xpmem/+xpmem/" cp2k_deps_${CP2K_VERSION}.yaml COPY ./tools/spack/cp2k_dev_repo ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev_repo/ RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev_repo/ RUN spack env create myenv cp2k_deps_${CP2K_VERSION}.yaml && \ spack -e myenv repo list +# Use Cray-MPICH if the ALPS Spack repository is used and add supported dependencies +# NOTE: This is a workaround for ALPS until the CE provides full MPI replacement +# FIXME: SIRIUS with libvdwxc +RUN if [ "${USE_ALPS_SPACK_REPO}" -ne 0 ]; then \ + spack -e myenv config remove "packages:mpi" && \ + spack -e myenv config add "packages:mpi:require:cray-mpich" && \ + spack -e myenv remove "mpich"; \ + spack -e myenv config remove "packages:sirius" && \ + spack -e myenv config add "packages:sirius:require:+fortran+pugixml~apps~vdwxc" && \ + spack -e myenv add "cray-mpich"; \ + spack -e myenv add "xpmem"; \ + spack -e myenv config add "packages:xpmem:require:~kernel-module"; \ + spack -e myenv add "dla-future-fortran"; \ + spack -e myenv remove "elpa"; \ + spack -e myenv add "osu-micro-benchmarks"; \ + else \ + spack -e myenv config change "packages:mpich:+xpmem"; \ + fi + # Install CP2K dependencies via Spack +RUN spack -e myenv config get RUN spack -e myenv concretize -f ENV SPACK_ENV_VIEW="${SPACK_ROOT}/var/spack/environments/myenv/spack-env/view" RUN spack -e myenv env depfile -o spack_makefile && \ diff --git a/cmake/cmake_cp2k.sh b/cmake/cmake_cp2k.sh index 259c7bdd32..861675de37 100755 --- a/cmake/cmake_cp2k.sh +++ b/cmake/cmake_cp2k.sh @@ -44,9 +44,11 @@ if [[ "${PROFILE}" == "spack" ]] && [[ "${VERSION}" == "psmp" ]]; then -GNinja \ -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ -DCP2K_USE_EVERYTHING=ON \ - -DCP2K_USE_DLAF=OFF \ + -DCP2K_USE_DLAF=ON \ -DCP2K_USE_LIBXSMM=OFF \ -DCP2K_USE_TBLITE=OFF \ + -DCP2K_USE_ELPA=OFF\ + -DCP2K_USE_LIBVDWXC=OFF \ -Werror=dev \ .. |& tee ./cmake.log CMAKE_EXIT_CODE=$? diff --git a/tools/spack/cp2k_deps_psmp.yaml b/tools/spack/cp2k_deps_psmp.yaml index 8c10367bf5..0acdc6118c 100644 --- a/tools/spack/cp2k_deps_psmp.yaml +++ b/tools/spack/cp2k_deps_psmp.yaml @@ -22,6 +22,14 @@ spack: mpich: require: - ~xpmem + cray-gtl: + buildable: True + cray-mpich: + buildable: True + cray-pals: + buildable: True + cray-pmi: + buildable: True openmpi: require: - +internal-hwloc @@ -65,6 +73,9 @@ spack: elpa: require: - +openmp + fft-api: + require: + - fftw fftw: require: - +openmp