cray-mpich

2026-07-21 06:25:15 -04:00 · 2025-08-22 17:08:22 +02:00 · 2025-08-22 17:08:22 +02:00 · f05998e8f8
commit f05998e8f8
parent 851dfd464f
4 changed files with 94 additions and 44 deletions
--- a/ci/cscs_eiger_spack_psmp.yaml
+++ b/ci/cscs_eiger_spack_psmp.yaml
@ -4,7 +4,7 @@ include:
 stages:
  - deps
  - cp2k
-  - test
+  #  - test
  - benchmark

 variables:
@ -13,7 +13,7 @@ variables:
  VERSION: psmp
  TAG: "${BUILDER}_${VERSION}"
  UBUNTU_IMAGE_NAME: "ubuntu:24.04"
-  CSCS_IMAGE_NAME: "${CSCS_REGISTRY_PATH}/${REPOSITORY}:${TAG}_${CI_COMMIT_SHORT_SHA}"
+  CSCS_IMAGE_NAME: "${CSCS_REGISTRY_PATH}/public/${REPOSITORY}:${TAG}_${CI_COMMIT_SHORT_SHA}"
  DOCKERHUB_IMAGE_NAME: "docker.io/${DOCKERHUB_USERNAME}/${REPOSITORY}:${TAG}"

 build deps eiger:
@ -21,7 +21,11 @@ build deps eiger:
  stage: deps
  timeout: 1h
  before_script:
-    - DEPS_IMAGE_TAG=`cat ci/cscs_eiger_${TAG}.yaml ci/docker/build_${CI_JOB_STAGE}_${BUILDER}.Dockerfile $(find tools/spack -type f) $(find cmake -type f) | sha256sum - | head -c 16`
+    - TAG_BASEIMG=`echo ${BASE_IMAGE##*/} | sed 's/[:]//g'`
+    - TAG_DOCKER=`sha256sum ci/docker/build_${CI_JOB_STAGE}_${BUILDER}.Dockerfile | head -c 16`
+    - TAG_SPACK=`cat $(find tools/spack -type f) | sha256sum - | head -c 16`
+    - TAG_ALPS=`if [ ${USE_ALPS_SPACK_REPO} -eq 1 ]; then echo "alps"; else echo ""; fi`
+    - DEPS_IMAGE_TAG=${TAG_BASEIMG}-${TAG_DOCKER}-${TAG_SPACK}${TAG_ALPS}
    - export PERSIST_IMAGE_NAME=${CSCS_REGISTRY_PATH}/base/${REPOSITORY}:${TAG}_${DEPS_IMAGE_TAG}
    - echo "DEPS_IMAGE_NAME=${PERSIST_IMAGE_NAME}" | tee build.env
  artifacts:
@ -31,12 +35,14 @@ build deps eiger:
    DOCKER_BUILD_ARGS: '[
      "BASE_IMAGE",
      "CP2K_VERSION",
+      "USE_ALPS_SPACK_REPO",
      "LOG_LINES"
    ]'
    DOCKERFILE: ci/docker/build_${CI_JOB_STAGE}_${BUILDER}.Dockerfile
    BASE_IMAGE: ${UBUNTU_IMAGE_NAME}
    CP2K_VERSION: ${VERSION}
    LOG_LINES: 200
+    USE_ALPS_SPACK_REPO: 1

 build cp2k eiger:
  extends: .container-builder-cscs-zen2
@ -47,7 +53,7 @@ build cp2k eiger:
    - echo ${DOCKERHUB_TOKEN} | podman login docker.io --username ${DOCKERHUB_USERNAME} --password-stdin
  after_script:
    - podman images
-    - podman push ${PERSIST_IMAGE_NAME} ${DOCKERHUB_IMAGE_NAME}
+#    - podman push ${PERSIST_IMAGE_NAME} ${DOCKERHUB_IMAGE_NAME}
  variables:
    DOCKER_BUILD_ARGS: '[
      "BASE_IMAGE",
@ -58,49 +64,50 @@ build cp2k eiger:
    DEPS_IMAGE: ${DEPS_IMAGE_NAME}
    PERSIST_IMAGE_NAME: ${CSCS_IMAGE_NAME}

-test cp2k eiger:
-  extends: .baremetal-runner-eiger-zen2
-  stage: test
-  needs: ["build cp2k eiger"]
-  timeout: 1h
-  script:
-    - podman run --shm-size=1g -v ${PWD}:/mnt ${DOCKERHUB_IMAGE_NAME} mpiexec -genv OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} -n ${SLURM_NTASKS} cp2k /opt/cp2k/benchmarks/CI/H2O-32_md.inp
-  variables:
-    SLURM_CONSTRAINT: mc
-    SLURM_CPUS_PER_TASK: 4
-    SLURM_DEBUG: 1
-    SLURM_HINT: nomultithread
-    SLURM_JOB_NUM_NODES: 1
-    SLURM_NTASKS: 8
-    SLURM_NTASKS_PER_NODE: 8
-    SLURM_TIMELIMIT: 10
-    USE_MPI: YES
-
-regression test cp2k eiger:
-  extends: .baremetal-runner-eiger-zen2
-  stage: test
-  needs: ["build cp2k eiger","test cp2k eiger"]
-  timeout: 1h
-  script:
-    - git --no-pager log -1 --pretty="%nCommitSHA:%x20%H%nCommitTime:%x20%ci%nCommitAuthor:%x20%an%nCommitSubject:%x20%s%n"
-    - podman run --shm-size=1g -v ${PWD}:/mnt ${DOCKERHUB_IMAGE_NAME} /opt/cp2k/tests/do_regtest.py --maxtasks ${SLURM_NTASKS} --mpiexec "mpiexec -bind-to none" --keepalive /opt/cp2k/bin ${VERSION} 2>&1 || true
-  variables:
-    SLURM_CONSTRAINT: mc
-    SLURM_DEBUG: 1
-    SLURM_HINT: nomultithread
-    SLURM_JOB_NUM_NODES: 1
-    SLURM_NTASKS: 128
-    SLURM_TIMELIMIT: 30
-    USE_MPI: YES
+# test cp2k eiger:
+#   extends: .baremetal-runner-eiger-zen2
+#   stage: test
+#   needs: ["build cp2k eiger"]
+#   timeout: 1h
+#   script:
+#     - podman run --shm-size=1g -v ${PWD}:/mnt ${CSCS_IMAGE_NAME} mpiexec -genv OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} -n ${SLURM_NTASKS} cp2k /opt/cp2k/benchmarks/CI/H2O-32_md.inp
+#   variables:
+#     SLURM_CONSTRAINT: mc
+#     SLURM_CPUS_PER_TASK: 4
+#     SLURM_DEBUG: 1
+#     SLURM_HINT: nomultithread
+#     SLURM_JOB_NUM_NODES: 1
+#     SLURM_NTASKS: 8
+#     SLURM_NTASKS_PER_NODE: 8
+#     SLURM_TIMELIMIT: 10
+#     USE_MPI: YES
+#
+# regression test cp2k eiger:
+#   extends: .baremetal-runner-eiger-zen2
+#   stage: test
+#   needs: ["build cp2k eiger","test cp2k eiger"]
+#   timeout: 1h
+#   script:
+#     - git --no-pager log -1 --pretty="%nCommitSHA:%x20%H%nCommitTime:%x20%ci%nCommitAuthor:%x20%an%nCommitSubject:%x20%s%n"
+#     - podman run --shm-size=1g -v ${PWD}:/mnt ${CSCS_IMAGE_NAME} /opt/cp2k/tests/do_regtest.py --maxtasks ${SLURM_NTASKS} --mpiexec "mpiexec -bind-to none" --keepalive /opt/cp2k/bin ${VERSION} 2>&1 || true
+#   variables:
+#     SLURM_CONSTRAINT: mc
+#     SLURM_DEBUG: 1
+#     SLURM_HINT: nomultithread
+#     SLURM_JOB_NUM_NODES: 1
+#     SLURM_NTASKS: 128
+#     SLURM_TIMELIMIT: 30
+#     USE_MPI: YES

 benchmark cp2k 1 eiger node:
  extends: .container-runner-eiger-zen2
  stage: benchmark
-  needs: ["build cp2k eiger","test cp2k eiger","regression test cp2k eiger"]
+  #needs: ["build cp2k eiger","test cp2k eiger","regression test cp2k eiger"]
  timeout: 1h
  image: ${CSCS_IMAGE_NAME}
  before_script:
    - export CP2K_DATA_DIR=/opt/cp2k/share/cp2k/data
+    - ldd $(which cp2k)
  script:
    - cp2k /opt/cp2k/benchmarks/CI/H2O-128_md.inp
  variables:
@ -109,7 +116,7 @@ benchmark cp2k 1 eiger node:
    SLURM_DEBUG: 1
    SLURM_HINT: nomultithread
    SLURM_JOB_NUM_NODES: 1
-    SLURM_MPI_TYPE: pmi2
+    SLURM_MPI_TYPE: cray_shasta
    SLURM_NTASKS: 128
    SLURM_NTASKS_PER_NODE: 128
    SLURM_TIMELIMIT: 30
@ -118,11 +125,12 @@ benchmark cp2k 1 eiger node:
 benchmark cp2k 2 eiger nodes:
  extends: .container-runner-eiger-zen2
  stage: benchmark
-  needs: ["build cp2k eiger","test cp2k eiger","regression test cp2k eiger"]
+  #needs: ["build cp2k eiger","test cp2k eiger","regression test cp2k eiger"]
  timeout: 1h
  image: ${CSCS_IMAGE_NAME}
  before_script:
    - export CP2K_DATA_DIR=/opt/cp2k/share/cp2k/data
+    - ldd $(which cp2k)
  script:
    - cp2k /opt/cp2k/benchmarks/CI/H2O-512_md.inp
  variables:
@ -131,7 +139,7 @@ benchmark cp2k 2 eiger nodes:
    SLURM_DEBUG: 1
    SLURM_HINT: nomultithread
    SLURM_JOB_NUM_NODES: 2
-    SLURM_MPI_TYPE: pmi2
+    SLURM_MPI_TYPE: cray_shasta
    SLURM_NTASKS: 256
    SLURM_NTASKS_PER_NODE: 128
    SLURM_TIMELIMIT: 30
--- a/ci/docker/build_deps_spack.Dockerfile
+++ b/ci/docker/build_deps_spack.Dockerfile
@ -79,6 +79,16 @@ ENV PATH="${SPACK_ROOT}/bin:${PATH}"
 # Add Spack packages builtin repository
 RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/builtin

+ARG USE_ALPS_SPACK_REPO
+ENV USE_ALPS_SPACK_REPO=${USE_ALPS_SPACK_REPO:-0}
+ENV ALPS_SPACK_REPO_SHA="57d5c2f39e4f0cf6acc8779dcaedc1e90264d639"
+RUN if [ "${USE_ALPS_SPACK_REPO}" -ne 0 ]; then \
+    mkdir -p /root/alps-cluster-config && \
+    wget -qO- "https://api.github.com/repos/eth-cscs/alps-cluster-config/tarball/$ALPS_SPACK_REPO_SHA" | \
+    tar --strip-components=1 -xz -C /root/alps-cluster-config && \
+    spack repo add --scope site /root/alps-cluster-config/site/spack_repo/alps; \
+    fi
+
 # Find all compilers
 RUN spack compiler find

@ -89,13 +99,32 @@ RUN spack external find --all --not-buildable
 ARG CP2K_VERSION
 ENV CP2K_VERSION=${CP2K_VERSION:-psmp}
 COPY ./tools/spack/cp2k_deps_${CP2K_VERSION}.yaml ./
-RUN sed -e "s/~xpmem/+xpmem/" cp2k_deps_${CP2K_VERSION}.yaml
 COPY ./tools/spack/cp2k_dev_repo ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev_repo/
 RUN spack repo add --scope site ${SPACK_PACKAGES_ROOT}/repos/spack_repo/cp2k_dev_repo/
 RUN spack env create myenv cp2k_deps_${CP2K_VERSION}.yaml && \
    spack -e myenv repo list

+# Use Cray-MPICH if the ALPS Spack repository is used and add supported dependencies
+# NOTE: This is a workaround for ALPS until the CE provides full MPI replacement
+# FIXME: SIRIUS with libvdwxc
+RUN if [ "${USE_ALPS_SPACK_REPO}" -ne 0 ]; then \
+    spack -e myenv config remove "packages:mpi" && \
+    spack -e myenv config add "packages:mpi:require:cray-mpich" && \
+    spack -e myenv remove "mpich"; \
+    spack -e myenv config remove "packages:sirius" && \
+    spack -e myenv config add "packages:sirius:require:+fortran+pugixml~apps~vdwxc" && \
+    spack -e myenv add "cray-mpich"; \
+    spack -e myenv add "xpmem"; \
+    spack -e myenv config add "packages:xpmem:require:~kernel-module"; \
+    spack -e myenv add "dla-future-fortran"; \
+    spack -e myenv remove "elpa"; \
+    spack -e myenv add "osu-micro-benchmarks"; \
+    else \
+    spack -e myenv config change "packages:mpich:+xpmem"; \
+    fi
+
 # Install CP2K dependencies via Spack
+RUN spack -e myenv config get
 RUN spack -e myenv concretize -f
 ENV SPACK_ENV_VIEW="${SPACK_ROOT}/var/spack/environments/myenv/spack-env/view"
 RUN spack -e myenv env depfile -o spack_makefile && \
--- a/cmake/cmake_cp2k.sh
+++ b/cmake/cmake_cp2k.sh
@ -44,9 +44,11 @@ if [[ "${PROFILE}" == "spack" ]] && [[ "${VERSION}" == "psmp" ]]; then
    -GNinja \
    -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
    -DCP2K_USE_EVERYTHING=ON \
-    -DCP2K_USE_DLAF=OFF \
+    -DCP2K_USE_DLAF=ON \
    -DCP2K_USE_LIBXSMM=OFF \
    -DCP2K_USE_TBLITE=OFF \
+    -DCP2K_USE_ELPA=OFF\
+    -DCP2K_USE_LIBVDWXC=OFF \
    -Werror=dev \
    .. |& tee ./cmake.log
  CMAKE_EXIT_CODE=$?
--- a/tools/spack/cp2k_deps_psmp.yaml
+++ b/tools/spack/cp2k_deps_psmp.yaml
@ -22,6 +22,14 @@ spack:
    mpich:
      require:
        - ~xpmem
+    cray-gtl:
+      buildable: True
+    cray-mpich:
+      buildable: True
+    cray-pals:
+      buildable: True
+    cray-pmi:
+      buildable: True
    openmpi:
      require:
        - +internal-hwloc
@ -65,6 +73,9 @@ spack:
    elpa:
      require:
        - +openmp
+    fft-api:
+      require:
+        - fftw
    fftw:
      require:
        - +openmp