cp2k/ci/cscs_daint_spack_psmp_h100.yaml

include:
  - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml'

stages:
  - deps
  - cp2k
  - test
  - benchmark

variables:
  REPOSITORY: cp2k_daint
  BUILDER: spack
  VERSION: psmp
  TAG: "${BUILDER}_${VERSION}_h100"
  UBUNTU_IMAGE_NAME: "docker.io/nvidia/cuda:12.9.2-devel-ubuntu24.04"
  CSCS_IMAGE_NAME: "${CSCS_REGISTRY_PATH}/${REPOSITORY}:${TAG}_${CI_COMMIT_SHORT_SHA}"
  DOCKERHUB_IMAGE_NAME: "docker.io/${DOCKERHUB_USERNAME}/${REPOSITORY}:${TAG}"

build deps daint:
  extends: .container-builder-cscs-gh200
  stage: deps
  timeout: 6h
  before_script:
    - DEPS_IMAGE_TAG=$(cat make_cp2k.sh ci/cscs_daint_${TAG}.yaml ci/docker/build_deps_spack.Dockerfile $(find tools/spack -type f | sort) | sha256sum | head -c 16)
    - export PERSIST_IMAGE_NAME=${CSCS_REGISTRY_PATH}/base/${REPOSITORY}:${TAG}_${DEPS_IMAGE_TAG}
    - echo "DEPS_IMAGE_NAME=${PERSIST_IMAGE_NAME}" | tee build.env
  artifacts:
    reports:
      dotenv: build.env
  variables:
    DOCKER_BUILD_ARGS: '[
      "BASE_IMAGE",
      "CP2K_VERSION",
      "FEATURE_FLAGS",
      "NUM_PROCS"
    ]'
    DOCKERFILE: ci/docker/build_${CI_JOB_STAGE}_${BUILDER}.Dockerfile
    BASE_IMAGE: ${UBUNTU_IMAGE_NAME}
    CP2K_VERSION: ${VERSION}
    FEATURE_FLAGS: "-ef openpmd -gpu GH200 -np 4"
    NUM_PROCS: 64
    KUBERNETES_CPU_REQUEST: 64
    KUBERNETES_CPU_LIMIT: 64
    KUBERNETES_MEMORY_REQUEST: 256Gi
    KUBERNETES_MEMORY_LIMIT: 256Gi

build cp2k daint:
  extends: .container-builder-cscs-gh200
  stage: cp2k
  needs: ["build deps daint"]
  timeout: 6h
  before_script:
    - echo ${DOCKERHUB_TOKEN} | podman login docker.io --username ${DOCKERHUB_USERNAME} --password-stdin
  after_script:
    - podman images
    - podman push ${PERSIST_IMAGE_NAME} ${DOCKERHUB_IMAGE_NAME}
  variables:
    DOCKER_BUILD_ARGS: '[
      "BASE_IMAGE",
      "DEPS_IMAGE",
      "FEATURE_FLAGS",
      "NUM_PROCS"
    ]'
    DOCKERFILE: ci/docker/build_${CI_JOB_STAGE}_${BUILDER}.Dockerfile
    BASE_IMAGE: ${UBUNTU_IMAGE_NAME}
    DEPS_IMAGE: ${DEPS_IMAGE_NAME}
    FEATURE_FLAGS: "-ef openpmd -gpu GH200"
    NUM_PROCS: 32
    KUBERNETES_CPU_REQUEST: 32
    KUBERNETES_CPU_LIMIT: 32
    PERSIST_IMAGE_NAME: ${CSCS_IMAGE_NAME}

test cp2k daint:
  extends: .baremetal-runner-daint-gh200
  stage: test
  needs: ["build cp2k daint"]
  timeout: 6h
  script:
    - podman run --device nvidia.com/gpu=0 --shm-size=1g -v ${PWD}:/mnt ${DOCKERHUB_IMAGE_NAME} mpiexec -bind-to socket -genv OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} -n ${SLURM_NTASKS} cp2k /opt/cp2k/benchmarks/CI/H2O-32_md.inp
    - podman run --device nvidia.com/gpu=0 --shm-size=1g -v ${PWD}:/mnt ${DOCKERHUB_IMAGE_NAME} mpiexec -bind-to socket -genv OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} -n ${SLURM_NTASKS} cp2k /opt/cp2k/benchmarks/CI/H2O-32_md_elpa.inp
  variables:
    SLURM_CPUS_PER_TASK: 4
    SLURM_DEBUG: 1
    SLURM_HINT: nomultithread
    SLURM_JOB_NUM_NODES: 1
    SLURM_NTASKS: 8
    SLURM_NTASKS_PER_NODE: 8
    SLURM_TIMELIMIT: 15
    USE_MPI: YES

regression test cp2k daint:
  extends: .baremetal-runner-daint-gh200
  stage: test
  needs: ["build cp2k daint"]
  timeout: 6h
  script:
    - git --no-pager log -1 --pretty="%nCommitSHA:%x20%H%nCommitTime:%x20%ci%nCommitAuthor:%x20%an%nCommitSubject:%x20%s%n"
    - podman run --device nvidia.com/gpu=all --shm-size=1g -v ${PWD}:/mnt ${DOCKERHUB_IMAGE_NAME} /opt/cp2k/tests/do_regtest.py --cp2kdatadir /opt/cp2k/install/share/cp2k/data --maxtasks ${SLURM_NTASKS} --mpiexec "mpiexec -bind-to none" --keepalive /opt/cp2k/install/bin ${VERSION} 2>&1 || true
  variables:
    SLURM_DEBUG: 1
    SLURM_HINT: nomultithread
    SLURM_JOB_NUM_NODES: 1
    SLURM_NTASKS: 224
    SLURM_TIMELIMIT: 60
    USE_MPI: YES

benchmark cp2k 1 daint node:
  extends: .container-runner-daint-gh200
  stage: benchmark
  needs: ["build cp2k daint","test cp2k daint"]
  timeout: 6h
  image: ${CSCS_IMAGE_NAME}
  before_script:
    - export CP2K_DATA_DIR=/opt/cp2k/install/share/cp2k/data
    - export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
  script:
    - cp2k /opt/cp2k/benchmarks/CI/H2O-512_md.inp
  variables:
    SLURM_CPU_BIND: cores
    SLURM_CPUS_PER_TASK: 3
    SLURM_DEBUG: 1
    SLURM_HINT: nomultithread
    SLURM_JOB_NUM_NODES: 1
    SLURM_MPI_TYPE: pmi2
    SLURM_NTASKS: 72
    SLURM_NTASKS_PER_NODE: 72
    SLURM_TIMELIMIT: 30
    USE_MPI: YES

benchmark cp2k 1 daint node elpa:
  extends: .container-runner-daint-gh200
  stage: benchmark
  needs: ["build cp2k daint","test cp2k daint","benchmark cp2k 1 daint node"]
  timeout: 6h
  image: ${CSCS_IMAGE_NAME}
  before_script:
    - export CP2K_DATA_DIR=/opt/cp2k/install/share/cp2k/data
    - export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
  script:
    - cp2k /opt/cp2k/benchmarks/CI/H2O-512_md_elpa.inp
  variables:
    SLURM_CPU_BIND: cores
    SLURM_CPUS_PER_TASK: 3
    SLURM_DEBUG: 1
    SLURM_HINT: nomultithread
    SLURM_JOB_NUM_NODES: 1
    SLURM_MPI_TYPE: pmi2
    SLURM_NTASKS: 72
    SLURM_NTASKS_PER_NODE: 72
    SLURM_TIMELIMIT: 30
    USE_MPI: YES