comp-chem/cp2k
1
0
Fork 0
mirror of https://github.com/cp2k/cp2k.git synced 2026-07-28 06:05:29 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

ELPA: compile-time choice for solver-kind

Browse source 
- For the OpenCL based GPU code-path, or by choice, ELPA may not be built with GPU support.
- Introduced __NO_OFFLOAD_ELPA to allow CPU-only ELPA (related to #4617).
- Allow compile-time selection of ELPA solver-kind (1-stage vs 2-stage).
- For ELPA/SYCL, use 1-stage ELPA solver by default for the time being.
- ELPA/SYCL (MKL in general?) has an index-fix currently only in a branch.

Index/ILP issue (undefined behavior):
- ELPA eventually mismatches index-kinds (LP64 vs ILP64 when linking BLAS/MKL).
- For static linkage it usually does not not matter (at least CPU-only).
- CP2K links the correct BLAS flavor (SO and GPU-build may be affected?).
- ELPA 2-stage may be even affected beyond MKL (or for GPU in general).
- ELPA 1-stage is apparently not affected by index issue.
This commit is contained in:
Hans Pabst 2026-01-09 08:59:02 +01:00
parent 62f3f8ee30
commit ffde6a3702
7 changed files with 75 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

13
CMakeLists.txt
View file

@ -188,13 +188,17 @@ cmake_dependent_option(
"NOT CP2K_USE_ACCEL MATCHES \"OPENCL|HIP|CUDA\"" OFF)
cmake_dependent_option(
CP2K_ENABLE_DBM_GPU "Enable the dbm accelerated backend (mostly GPU)." ON
"CP2K_USE_ACCEL" OFF)
CP2K_ENABLE_ELPA_GPU "Enable acceleration for elpa related functions." ON
"CP2K_USE_ACCEL MATCHES \"HIP|CUDA\"" OFF)
cmake_dependent_option(
CP2K_ENABLE_GRID_GPU "Enable acceleration for grid related functions." ON
"CP2K_USE_ACCEL MATCHES \"HIP|CUDA\"" OFF)
cmake_dependent_option(
CP2K_ENABLE_DBM_GPU "Enable the dbm accelerated backend (mostly GPU)." ON
"CP2K_USE_ACCEL" OFF)
cmake_dependent_option(
CP2K_ENABLE_PW_GPU "Enable the ffts accelerated backend (mostly GPU)." ON
"CP2K_USE_ACCEL MATCHES \"HIP|CUDA\"" OFF)
@ -950,9 +954,10 @@ if((CP2K_USE_ACCEL MATCHES "CUDA") OR (CP2K_USE_ACCEL MATCHES "HIP"))
message(
" - GPU accelerated modules\n"
" - PW module: ${CP2K_ENABLE_PW_GPU}\n"
" - ELPA module: ${CP2K_ENABLE_ELPA_GPU}\n"
" - GRID module: ${CP2K_ENABLE_GRID_GPU}\n"
" - DBM module: ${CP2K_ENABLE_DBM_GPU}\n\n")
" - DBM module: ${CP2K_ENABLE_DBM_GPU}\n"
" - PW module: ${CP2K_ENABLE_PW_GPU}\n\n")
endif()
if(CP2K_USE_CUSOLVER_MP)

3
docs/technologies/eigensolvers/elpa.md
View file

@ -11,6 +11,7 @@ The [ELPA] library provides highly efficient and highly scalable direct eigensol
-DCP2K_USE_ELPA=ON
```
Pass `-DCP2K_ENABLE_ELPA_OPENMP_SUPPORT=ON` CMake if ELPA was built with OpenMP support.
Pass `-DCP2K_ENABLE_ELPA_OPENMP_SUPPORT=ON` to CMake if ELPA was built with OpenMP support. Pass
`-DCP2K_ENABLE_ELPA_GPU=OFF` to CMake if ELPA was not built with GPU support.
[elpa]: https://elpa.mpcdf.mpg.de/

5
src/CMakeLists.txt
View file

@ -1818,9 +1818,10 @@ if(CP2K_USE_CUDA
cp2k
PUBLIC $<$<BOOL:${CP2K_WITH_GPU_PROFILING}>:__OFFLOAD_PROFILING>
$<$<NOT:$<BOOL:${CP2K_DBCSR_USE_CPU_ONLY}>>:__DBCSR_ACC>
$<$<NOT:$<BOOL:${CP2K_ENABLE_ELPA_GPU}>>:__NO_OFFLOAD_ELPA>
$<$<NOT:$<BOOL:${CP2K_ENABLE_GRID_GPU}>>:__NO_OFFLOAD_GRID>
$<$<NOT:$<BOOL:${CP2K_ENABLE_PW_GPU}>>:__NO_OFFLOAD_PW>
$<$<NOT:$<BOOL:${CP2K_ENABLE_DBM_GPU}>>:__NO_OFFLOAD_DBM>)
$<$<NOT:$<BOOL:${CP2K_ENABLE_DBM_GPU}>>:__NO_OFFLOAD_DBM>
$<$<NOT:$<BOOL:${CP2K_ENABLE_PW_GPU}>>:__NO_OFFLOAD_PW>)
endif()
# add an alias for the FetchContent. Note that FindPackage will automatically

2
src/base/base_uses.f90
View file

@ -12,7 +12,7 @@
timeset,&
timestop
#if defined(__OFFLOAD_CUDA) || defined(__OFFLOAD_HIP)
#if defined(__OFFLOAD_CUDA) || defined(__OFFLOAD_HIP) || defined(__OFFLOAD_OPENCL)
#define __OFFLOAD
#endif

3
src/cp2k_info.F
View file

@ -234,6 +234,9 @@ CONTAINS
#if defined(__OFFLOAD_OPENCL)
flags = TRIM(flags)//" offload_opencl"
#endif
#if defined(__NO_OFFLOAD_ELPA)
flags = TRIM(flags)//" no_offload_elpa"
#endif
#if defined(__NO_OFFLOAD_GRID)
flags = TRIM(flags)//" no_offload_grid"
#endif

96
src/fm/cp_fm_elpa.F
View file

@ -31,15 +31,28 @@ MODULE cp_fm_elpa
USE cp_log_handling, ONLY: cp_get_default_logger, &
cp_logger_get_default_io_unit, &
cp_logger_type
USE kinds, ONLY: default_string_length, &
dp
USE kinds, ONLY: default_string_length, dp
USE message_passing, ONLY: mp_comm_type
USE OMP_LIB, ONLY: omp_get_max_threads
#include "../base/base_uses.f90"
#if defined (__ELPA)
USE elpa_constants, ONLY: ELPA_2STAGE_REAL_INVALID, &
#if defined(__ELPA)
#if defined(__OFFLOAD_OPENCL)
#define CPELPA_SOLVER_STAGES 1
#else
#define CPELPA_SOLVER_STAGES 2
#endif
#if 2 == CPELPA_SOLVER_STAGES
#define CPELPA_SOLVER ELPA_SOLVER_2STAGE
#else
#define CPELPA_SOLVER ELPA_SOLVER_1STAGE
#endif
USE elpa_constants, ONLY: ELPA_SOLVER_1STAGE, ELPA_SOLVER_2STAGE, ELPA_OK, &
ELPA_2STAGE_REAL_INVALID, &
ELPA_2STAGE_REAL_DEFAULT, &
ELPA_2STAGE_REAL_GENERIC, &
ELPA_2STAGE_REAL_GENERIC_SIMPLE, &
@ -62,9 +75,8 @@ MODULE cp_fm_elpa
ELPA_2STAGE_REAL_AMD_GPU, &
ELPA_2STAGE_REAL_INTEL_GPU_SYCL
USE elpa, ONLY: elpa_t, elpa_solver_2stage, &
elpa_init, elpa_uninit, &
elpa_allocate, elpa_deallocate, elpa_ok
USE elpa, ONLY: elpa_t, elpa_init, elpa_uninit, &
elpa_allocate, elpa_deallocate
#endif
IMPLICIT NONE
@ -175,7 +187,7 @@ CONTAINS
! **************************************************************************************************
SUBROUTINE initialize_elpa_library()
#if defined(__ELPA)
IF (elpa_init(20180525) /= elpa_ok) &
IF (elpa_init(20180525) /= ELPA_OK) &
CPABORT("The linked ELPA library does not support the required API version")
#else
CPABORT("Initialization of ELPA library requested but not enabled during build")
@ -200,7 +212,7 @@ CONTAINS
SUBROUTINE set_elpa_kernel(requested_kernel)
INTEGER, INTENT(IN) :: requested_kernel
#if defined (__ELPA)
#if defined(__ELPA)
INTEGER :: cpuid
elpa_kernel = requested_kernel
@ -222,16 +234,17 @@ CONTAINS
END IF
! Prefer GPU kernel if available.
#if defined (__OFFLOAD_CUDA)
#if !defined(__NO_OFFLOAD_ELPA)
#if defined(__OFFLOAD_CUDA)
elpa_kernel = ELPA_2STAGE_REAL_NVIDIA_GPU
#endif
#if defined (__OFFLOAD_HIP)
#if defined(__OFFLOAD_HIP)
elpa_kernel = ELPA_2STAGE_REAL_AMD_GPU
#endif
#if defined (__OFFLOAD_OPENCL)
#if defined(__OFFLOAD_OPENCL)
elpa_kernel = ELPA_2STAGE_REAL_INTEL_GPU_SYCL
#endif
#endif
! If we could not find a suitable kernel then use ELPA_2STAGE_REAL_DEFAULT.
IF (elpa_kernel == ELPA_2STAGE_REAL_INVALID) THEN
elpa_kernel = ELPA_2STAGE_REAL_DEFAULT
@ -411,7 +424,7 @@ CONTAINS
"ELPA| Matrix diagonalization information"
! Find name for given kernel id.
! In case ELPA_2STAGE_REAL_DEFAULT was used it might not be in our elpa_kernel_ids list.
! In case of ELPA_2STAGE_REAL_DEFAULT was used it might not be in our elpa_kernel_ids list.
kernel_name = "id: "//TRIM(ADJUSTL(cp_to_string(elpa_kernel)))
DO i = 1, SIZE(elpa_kernel_ids)
IF (elpa_kernel_ids(i) == elpa_kernel) THEN
@ -474,74 +487,75 @@ CONTAINS
elpa_obj => elpa_allocate()
CALL elpa_obj%set("na", n, success)
CPASSERT(success == elpa_ok)
CPASSERT(success == ELPA_OK)
CALL elpa_obj%set("nev", neig, success)
CPASSERT(success == elpa_ok)
CPASSERT(success == ELPA_OK)
CALL elpa_obj%set("local_nrows", n_rows, success)
CPASSERT(success == elpa_ok)
CPASSERT(success == ELPA_OK)
CALL elpa_obj%set("local_ncols", n_cols, success)
CPASSERT(success == elpa_ok)
CPASSERT(success == ELPA_OK)
CALL elpa_obj%set("nblk", nblk, success)
CPASSERT(success == elpa_ok)
CPASSERT(success == ELPA_OK)
CALL elpa_obj%set("mpi_comm_parent", group%get_handle(), success)
CPASSERT(success == elpa_ok)
CPASSERT(success == ELPA_OK)
CALL elpa_obj%set("process_row", myprow, success)
CPASSERT(success == elpa_ok)
CPASSERT(success == ELPA_OK)
CALL elpa_obj%set("process_col", mypcol, success)
CPASSERT(success == elpa_ok)
CPASSERT(success == ELPA_OK)
success = elpa_obj%setup()
CPASSERT(success == elpa_ok)
CPASSERT(success == ELPA_OK)
CALL elpa_obj%set("solver", elpa_solver_2stage, success)
CPASSERT(success == elpa_ok)
CALL elpa_obj%set("solver", CPELPA_SOLVER, success)
CPASSERT(success == ELPA_OK)
! enabling the GPU must happen before setting the kernel
IF (elpa_kernel == ELPA_2STAGE_REAL_NVIDIA_GPU) THEN
SELECT CASE (elpa_kernel)
CASE (ELPA_2STAGE_REAL_NVIDIA_GPU)
CALL elpa_obj%set("nvidia-gpu", 1, success)
CPASSERT(success == elpa_ok)
END IF
IF (elpa_kernel == ELPA_2STAGE_REAL_AMD_GPU) THEN
CPASSERT(success == ELPA_OK)
CASE (ELPA_2STAGE_REAL_AMD_GPU)
CALL elpa_obj%set("amd-gpu", 1, success)
CPASSERT(success == elpa_ok)
END IF
IF (elpa_kernel == ELPA_2STAGE_REAL_INTEL_GPU_SYCL) THEN
CPASSERT(success == ELPA_OK)
CASE (ELPA_2STAGE_REAL_INTEL_GPU_SYCL)
CALL elpa_obj%set("intel-gpu", 1, success)
CPASSERT(success == elpa_ok)
END IF
CPASSERT(success == ELPA_OK)
END SELECT
#if 2 == CPELPA_SOLVER_STAGES
CALL elpa_obj%set("real_kernel", elpa_kernel, success)
CPWARN_IF(success /= elpa_ok, "Setting real_kernel for ELPA failed")
CPWARN_IF(success /= ELPA_OK, "Setting real_kernel for ELPA failed")
#endif
IF (use_qr) THEN
CALL elpa_obj%set("qr", 1, success)
CPASSERT(success == elpa_ok)
CPASSERT(success == ELPA_OK)
END IF
! Set number of threads only when ELPA was built with OpenMP support.
IF (elpa_obj%can_set("omp_threads", omp_get_max_threads()) == ELPA_OK) THEN
CALL elpa_obj%set("omp_threads", omp_get_max_threads(), success)
CPASSERT(success == elpa_ok)
CPASSERT(success == ELPA_OK)
END IF
CALL elpa_obj%eigenvectors(matrix%local_data, eval, eigenvectors%local_data, success)
IF (success /= elpa_ok) &
IF (success /= ELPA_OK) &
CPABORT("ELPA failed to diagonalize a matrix")
IF (check_eigenvalues) THEN
! run again without QR
CALL elpa_obj%set("qr", 0, success)
CPASSERT(success == elpa_ok)
CPASSERT(success == ELPA_OK)
CALL elpa_obj%eigenvectors(matrix_noqr%local_data, eval_noqr, eigenvectors_noqr%local_data, success)
IF (success /= elpa_ok) &
IF (success /= ELPA_OK) &
CPABORT("ELPA failed to diagonalize a matrix even without QR decomposition")
IF (ANY(ABS(eval(1:neig) - eval_noqr(1:neig)) > th)) &
@ -553,7 +567,7 @@ CONTAINS
END IF
CALL elpa_deallocate(elpa_obj, success)
CPASSERT(success == elpa_ok)
CPASSERT(success == ELPA_OK)
eigenvalues(1:neig) = eval(1:neig)
DEALLOCATE (eval)

3
tools/precommit/check_file_properties.py
View file

@ -51,8 +51,8 @@ FLAG_EXCEPTIONS = (
r"__FORCE_USE_FAST_MATH",
r"__INTEL_LLVM_COMPILER",
r"__INTEL_COMPILER",
r"OFFLOAD_MEMPOOL_OMPALLOC",
r"OFFLOAD_BUFFER_MEMPOOL",
r"OFFLOAD_MEMPOOL_.+",
r"OFFLOAD_CHECK",
r"__OFFLOAD_CUDA",
r"__OFFLOAD_HIP",
@ -75,6 +75,7 @@ FLAG_EXCEPTIONS = (
r"TEST_LIBGRPP_.+",
r"__LIBXSMM2",
r"CPVERSION",
r"CPELPA_SOLVER_STAGES",
r"_WIN32",
r"OPENPMDAPI_VERSION_GE",
r"openPMD_HAVE_MPI",