From ff817ca511d4a80b6768b41dec1eada24328cbb5 Mon Sep 17 00:00:00 2001 From: Holmberg Nico Date: Mon, 10 Sep 2018 12:01:05 +0000 Subject: [PATCH] Cube I/O: Add input option to disable MPI I/O svn-origin-rev: 18601 --- src/cp_realspace_grid_cube.F | 37 ++++++++++++++++++++++++++++-- src/environment.F | 10 ++++++-- src/input/cp_output_handling.F | 37 +++++++++++++++++++++++++++++- src/input_cp2k_global.F | 8 +++++++ src/optimize_embedding_potential.F | 8 +++---- src/pw/realspace_grid_cube.F | 12 +++------- src/qs_cdft_methods.F | 20 ++++++++-------- src/qs_external_potential.F | 4 ++-- 8 files changed, 106 insertions(+), 30 deletions(-) diff --git a/src/cp_realspace_grid_cube.F b/src/cp_realspace_grid_cube.F index 0a5fd06101..3e3cf883c5 100644 --- a/src/cp_realspace_grid_cube.F +++ b/src/cp_realspace_grid_cube.F @@ -9,10 +9,12 @@ ! ************************************************************************************************** MODULE cp_realspace_grid_cube USE atomic_kind_types, ONLY: get_atomic_kind + USE cp_output_handling, ONLY: cp_mpi_io_get USE kinds, ONLY: dp USE particle_list_types, ONLY: particle_list_type USE pw_types, ONLY: pw_type - USE realspace_grid_cube, ONLY: pw_to_cube,& + USE realspace_grid_cube, ONLY: cube_to_pw,& + pw_to_cube,& pw_to_simple_volumetric #include "./base/base_uses.f90" @@ -20,7 +22,7 @@ MODULE cp_realspace_grid_cube PRIVATE - PUBLIC :: cp_pw_to_cube, cp_pw_to_simple_volumetric + PUBLIC :: cp_pw_to_cube, cp_pw_to_simple_volumetric, cp_cube_to_pw CHARACTER(len=*), PARAMETER, PRIVATE :: moduleN = 'cp_realspace_grid_cube' @@ -94,4 +96,35 @@ CONTAINS END SUBROUTINE cp_pw_to_simple_volumetric +! ************************************************************************************************** +!> \brief Thin wrapper around routine cube_to_pw +!> \param grid pw to read from cube file +!> \param filename name of cube file +!> \param scaling scale values before storing +!> \par History +!> Created [Nico Holmberg] (09.2018) +! ************************************************************************************************** + SUBROUTINE cp_cube_to_pw(grid, filename, scaling) + TYPE(pw_type), POINTER :: grid + CHARACTER(len=*), INTENT(in) :: filename + REAL(kind=dp), INTENT(in) :: scaling + + CHARACTER(len=*), PARAMETER :: routineN = 'cp_cube_to_pw', routineP = moduleN//':'//routineN + + LOGICAL :: parallel_read + + + ! Determine whether to use MPI I/O for reading cube filename + parallel_read = .TRUE. + ! Parallel routine falls back to stream read in serial mode, + ! but it has slight overhead compared to sequential read + ! Therefore, we use sequential version in serial mode + IF (grid%pw_grid%para%group_size == 1) parallel_read = .FALSE. + ! Check if MPI I/O was disabled in GLOBAL section + IF (.NOT. cp_mpi_io_get()) parallel_read = .FALSE. + + CALL cube_to_pw(grid, filename, scaling, parallel_read) + + END SUBROUTINE cp_cube_to_pw + END MODULE cp_realspace_grid_cube diff --git a/src/environment.F b/src/environment.F index 0e08202400..5b61a24753 100644 --- a/src/environment.F +++ b/src/environment.F @@ -31,7 +31,8 @@ MODULE environment cp_add_default_logger, cp_get_default_logger, cp_logger_create, & cp_logger_get_default_unit_nr, cp_logger_release, cp_logger_set, cp_logger_type, & cp_rm_default_logger, cp_to_string - USE cp_output_handling, ONLY: cp_print_key_finished_output,& + USE cp_output_handling, ONLY: cp_mpi_io_set,& + cp_print_key_finished_output,& cp_print_key_unit_nr,& debug_print_level,& high_print_level,& @@ -496,7 +497,8 @@ CONTAINS SReclaimable_avr, SReclaimable_max, SReclaimable_min INTEGER, DIMENSION(:), POINTER :: i_force_eval LOGICAL :: ata, do_echo_all_hosts, efl, explicit, & - report_maxloc, trace, trace_master + flag, report_maxloc, trace, & + trace_master TYPE(cp_logger_type), POINTER :: logger TYPE(enumeration_type), POINTER :: enum1, enum2 TYPE(keyword_type), POINTER :: keyword @@ -512,6 +514,8 @@ CONTAINS CALL section_vals_val_get(global_section, "BLACS_GRID", i_val=globenv%blacs_grid_layout) CALL section_vals_val_get(global_section, "BLACS_REPEATABLE", l_val=globenv%blacs_repeatable) CALL section_vals_val_get(global_section, "PREFERRED_DIAG_LIBRARY", i_val=i_diag) + CALL section_vals_val_get(global_section, "ENABLE_MPI_IO", l_val=flag) + CALL cp_mpi_io_set(flag) CALL section_vals_val_get(global_section, "ELPA_KERNEL", i_val=globenv%k_elpa) CALL section_vals_val_get(global_section, "ELPA_QR", l_val=globenv%elpa_qr) CALL section_vals_val_get(global_section, "ELPA_QR_UNSAFE", l_val=globenv%elpa_qr_unsafe) @@ -772,6 +776,8 @@ CONTAINS WRITE (UNIT=output_unit, FMT="(T2,A,T75,A6)") & start_section_label//"| Global print level", print_level_string + WRITE (UNIT=output_unit, FMT="(T2,A,T75,L6)") & + start_section_label//"| MPI I/O enabled", flag WRITE (UNIT=output_unit, FMT="(T2,A,T75,I6)") & start_section_label//"| Total number of message passing processes", & para_env%num_pe, & diff --git a/src/input/cp_output_handling.F b/src/input/cp_output_handling.F index abe6101a15..2c316418cd 100644 --- a/src/input/cp_output_handling.F +++ b/src/input/cp_output_handling.F @@ -108,6 +108,11 @@ MODULE cp_output_handling cp_out_store_each = IBSET(0, cp_p_store_each), & cp_out_default = cp_out_file_if +! Flag determining if MPI I/O should be enabled for functions that support it + LOGICAL, PRIVATE, SAVE :: enable_mpi_io = .FALSE. +! Public functions to set/get the flags + PUBLIC :: cp_mpi_io_set, cp_mpi_io_get + ! ************************************************************************************************** !> \brief stores the flags_env controlling the output of properties !> \param ref_count reference count (see doc/ReferenceCounting.html) @@ -875,7 +880,7 @@ CONTAINS IF (PRESENT(is_new_file)) is_new_file = .FALSE. IF (PRESENT(mpi_io)) THEN #if defined(__parallel) - IF (logger%para_env%num_pe > 1 .AND. mpi_io) THEN + IF (cp_mpi_io_get() .AND. logger%para_env%num_pe > 1 .AND. mpi_io) THEN my_mpi_io = .TRUE. ELSE my_mpi_io = .FALSE. @@ -1083,5 +1088,35 @@ CONTAINS unit_nr = -1 END SUBROUTINE cp_print_key_finished_output +! ************************************************************************************************** +!> \brief Sets flag which determines whether or not to use MPI I/O for I/O routines that +!> have been parallized with MPI +!> \param flag ... +!> \par History +!> 09.2018 created [Nico Holmberg] +! ************************************************************************************************** + SUBROUTINE cp_mpi_io_set(flag) + LOGICAL, INTENT(IN) :: flag + + CHARACTER(len=*), PARAMETER :: routineN = 'cp_mpi_io_set', routineP = moduleN//':'//routineN + + enable_mpi_io = flag + END SUBROUTINE cp_mpi_io_set + +! ************************************************************************************************** +!> \brief Gets flag which determines whether or not to use MPI I/O for I/O routines that +!> have been parallized with MPI +!> \return ... +!> \par History +!> 09.2018 created [Nico Holmberg] +! ************************************************************************************************** + FUNCTION cp_mpi_io_get() RESULT(flag) + LOGICAL :: flag + + CHARACTER(len=*), PARAMETER :: routineN = 'cp_mpi_io_get', routineP = moduleN//':'//routineN + + flag = enable_mpi_io + END FUNCTION cp_mpi_io_get + END MODULE cp_output_handling diff --git a/src/input_cp2k_global.F b/src/input_cp2k_global.F index 8a48ffeb0f..c071bbb655 100644 --- a/src/input_cp2k_global.F +++ b/src/input_cp2k_global.F @@ -351,6 +351,14 @@ CONTAINS CALL section_add_keyword(section, keyword) CALL keyword_release(keyword) + CALL keyword_create(keyword, name="ENABLE_MPI_IO", & + description="Enable MPI parallelization for all supported I/O routines "// & + "Currently, only cube file writer/reader routines use MPI I/O. Disabling "// & + "this flag might speed up calculations dominated by I/O.", & + usage="ENABLE_MPI_IO FALSE", default_l_val=.TRUE., lone_keyword_l_val=.TRUE.) + CALL section_add_keyword(section, keyword) + CALL keyword_release(keyword) + CALL keyword_create(keyword, name="TRACE", & description="If a debug trace of the execution of the program should be written ", & usage="TRACE", & diff --git a/src/optimize_embedding_potential.F b/src/optimize_embedding_potential.F index efad426e6b..a90322014f 100644 --- a/src/optimize_embedding_potential.F +++ b/src/optimize_embedding_potential.F @@ -36,7 +36,8 @@ MODULE optimize_embedding_potential cp_print_key_should_output,& cp_print_key_unit_nr USE cp_para_types, ONLY: cp_para_env_type - USE cp_realspace_grid_cube, ONLY: cp_pw_to_cube,& + USE cp_realspace_grid_cube, ONLY: cp_cube_to_pw,& + cp_pw_to_cube,& cp_pw_to_simple_volumetric USE dbcsr_api, ONLY: dbcsr_p_type USE embed_types, ONLY: opt_embed_pot_type @@ -93,7 +94,6 @@ MODULE optimize_embedding_potential USE qs_neighbor_list_types, ONLY: neighbor_list_set_p_type USE qs_subsys_types, ONLY: qs_subsys_get,& qs_subsys_type - USE realspace_grid_cube, ONLY: cube_to_pw #include "./base/base_uses.f90" IMPLICIT NONE @@ -657,7 +657,7 @@ CONTAINS CPABORT("Embedding cube file not found. ") scaling_factor = 1.0_dp - CALL cube_to_pw(embed_pot%pw, filename, scaling_factor) + CALL cp_cube_to_pw(embed_pot%pw, filename, scaling_factor) ! Spin-dependent part of the potential IF (open_shell_embed) THEN @@ -668,7 +668,7 @@ CONTAINS CPABORT("Embedding spin cube file not found. ") scaling_factor = 1.0_dp - CALL cube_to_pw(spin_embed_pot%pw, filename, scaling_factor) + CALL cp_cube_to_pw(spin_embed_pot%pw, filename, scaling_factor) ENDIF END SUBROUTINE read_embed_pot_cube diff --git a/src/pw/realspace_grid_cube.F b/src/pw/realspace_grid_cube.F index a1495d0aca..11d50a5666 100644 --- a/src/pw/realspace_grid_cube.F +++ b/src/pw/realspace_grid_cube.F @@ -220,15 +220,17 @@ CONTAINS !> \param grid pw to read from cube file !> \param filename name of cube file !> \param scaling scale values before storing +!> \param parallel_read ... !> \par History !> Created [M.Watkins] (01.2014) !> Use blocking, collective MPI read for parallel simulations [Nico Holmberg] (05.2017) ! ************************************************************************************************** - SUBROUTINE cube_to_pw(grid, filename, scaling) + SUBROUTINE cube_to_pw(grid, filename, scaling, parallel_read) TYPE(pw_type), POINTER :: grid CHARACTER(len=*), INTENT(in) :: filename REAL(kind=dp), INTENT(in) :: scaling + LOGICAL, INTENT(in) :: parallel_read CHARACTER(len=*), PARAMETER :: routineN = 'cube_to_pw', routineP = moduleN//':'//routineN INTEGER, PARAMETER :: entry_len = 13, num_entries_line = 6 @@ -239,7 +241,6 @@ CONTAINS size_of_z, tag INTEGER, DIMENSION(3) :: lbounds, lbounds_local, npoints, & npoints_local, ubounds, ubounds_local - LOGICAL :: parallel_read REAL(kind=dp), ALLOCATABLE, DIMENSION(:) :: buffer REAL(kind=dp), DIMENSION(3) :: dr, rdum @@ -247,19 +248,12 @@ CONTAINS CALL timeset(routineN, handle) - parallel_read = .TRUE. - !get rs grids and parallel envnment gid = grid%pw_grid%para%group my_rank = grid%pw_grid%para%my_pos num_pe = grid%pw_grid%para%group_size tag = 1 - ! Parallel routine falls back to stream read in serial mode, - ! but it has slight overhead compared to sequential read - ! Therefore, we use sequential version in serial mode - IF (num_pe == 1) parallel_read = .FALSE. - lbounds_local = grid%pw_grid%bounds_local(1, :) ubounds_local = grid%pw_grid%bounds_local(2, :) size_of_z = ubounds_local(3)-lbounds_local(3)+1 diff --git a/src/qs_cdft_methods.F b/src/qs_cdft_methods.F index e7be41b779..cfe721da54 100644 --- a/src/qs_cdft_methods.F +++ b/src/qs_cdft_methods.F @@ -21,7 +21,8 @@ MODULE qs_cdft_methods USE cp_output_handling, ONLY: cp_print_key_finished_output,& cp_print_key_unit_nr USE cp_para_types, ONLY: cp_para_env_type - USE cp_realspace_grid_cube, ONLY: cp_pw_to_cube + USE cp_realspace_grid_cube, ONLY: cp_cube_to_pw,& + cp_pw_to_cube USE cube_utils, ONLY: cube_info_type USE hirshfeld_methods, ONLY: create_shape_function USE hirshfeld_types, ONLY: get_hirshfeld_info,& @@ -76,7 +77,6 @@ MODULE qs_cdft_methods qs_rho_type USE qs_subsys_types, ONLY: qs_subsys_get,& qs_subsys_type - USE realspace_grid_cube, ONLY: cube_to_pw USE realspace_grid_types, ONLY: & pw2rs, realspace_grid_desc_type, realspace_grid_type, rs2pw, rs_grid_create, & rs_grid_release, rs_grid_retain, rs_grid_zero, rs_pw_transfer @@ -923,22 +923,22 @@ CONTAINS ! Total density (rho_alpha + rho_beta) CALL pw_pool_create_pw(auxbas_pw_pool, becke_control%fragments(1, 1)%pw, & use_data=REALDATA3D, in_space=REALSPACE) - CALL cube_to_pw(becke_control%fragments(1, 1)%pw, & - becke_control%fragment_a_fname, 1.0_dp) + CALL cp_cube_to_pw(becke_control%fragments(1, 1)%pw, & + becke_control%fragment_a_fname, 1.0_dp) CALL pw_pool_create_pw(auxbas_pw_pool, becke_control%fragments(1, 2)%pw, & use_data=REALDATA3D, in_space=REALSPACE) - CALL cube_to_pw(becke_control%fragments(1, 2)%pw, & - becke_control%fragment_b_fname, 1.0_dp) + CALL cp_cube_to_pw(becke_control%fragments(1, 2)%pw, & + becke_control%fragment_b_fname, 1.0_dp) ! Spin difference density (rho_alpha - rho_beta) if needed IF (needs_spin_density) THEN CALL pw_pool_create_pw(auxbas_pw_pool, becke_control%fragments(2, 1)%pw, & use_data=REALDATA3D, in_space=REALSPACE) - CALL cube_to_pw(becke_control%fragments(2, 1)%pw, & - becke_control%fragment_a_spin_fname, multiplier(1)) + CALL cp_cube_to_pw(becke_control%fragments(2, 1)%pw, & + becke_control%fragment_a_spin_fname, multiplier(1)) CALL pw_pool_create_pw(auxbas_pw_pool, becke_control%fragments(2, 2)%pw, & use_data=REALDATA3D, in_space=REALSPACE) - CALL cube_to_pw(becke_control%fragments(2, 2)%pw, & - becke_control%fragment_b_spin_fname, multiplier(2)) + CALL cp_cube_to_pw(becke_control%fragments(2, 2)%pw, & + becke_control%fragment_b_spin_fname, multiplier(2)) END IF ! Sum up fragments DO i = 1, nfrag_spins diff --git a/src/qs_external_potential.F b/src/qs_external_potential.F index 315019f4c8..98b65596ae 100644 --- a/src/qs_external_potential.F +++ b/src/qs_external_potential.F @@ -14,6 +14,7 @@ MODULE qs_external_potential pbc USE cp_control_types, ONLY: dft_control_type USE cp_log_handling, ONLY: cp_to_string + USE cp_realspace_grid_cube, ONLY: cp_cube_to_pw USE force_fields_util, ONLY: get_generic_info USE fparser, ONLY: evalf,& evalfd,& @@ -37,7 +38,6 @@ MODULE qs_external_potential USE qs_force_types, ONLY: qs_force_type USE qs_kind_types, ONLY: get_qs_kind,& qs_kind_type - USE realspace_grid_cube, ONLY: cube_to_pw USE string_utilities, ONLY: compress #include "./base/base_uses.f90" @@ -89,7 +89,7 @@ CONTAINS IF ((.NOT. static_potential) .OR. dft_control%eval_external_potential) THEN IF (read_from_cube) THEN CALL section_vals_val_get(ext_pot_section, "SCALING_FACTOR", r_val=scaling_factor) - CALL cube_to_pw(v_ee%pw, 'pot.cube', scaling_factor) + CALL cp_cube_to_pw(v_ee%pw, 'pot.cube', scaling_factor) dft_control%eval_external_potential = .FALSE. ELSE dr = v_ee%pw%pw_grid%dr