From b7b3bfa8495d3c336743a0a2ee398f2a49ff7e2a Mon Sep 17 00:00:00 2001 From: Hans Pabst Date: Mon, 6 Oct 2025 13:05:23 +0200 Subject: [PATCH] Adjusted print_func similar to Offload and DBM library - Print grid statistics (only if non-zero counter exists). - Removed dependency from string_utilities (strlcpy_c2f). - No copy of the message is necessary. - Reduced PACKAGE dependencies. - Print offload memory stats. --- src/grid/common/PACKAGE | 2 +- src/grid/common/grid_library.c | 91 ++++++++++++++++++++++------------ src/grid/common/grid_library.h | 5 +- src/grid/grid_api.F | 57 +++++---------------- src/grid/grid_miniapp.c | 35 +++++++------ src/grid/grid_unittest.c | 32 +++++------- src/mpiwrap/cp_mpi.c | 20 ++++++++ src/mpiwrap/cp_mpi.h | 6 +++ 8 files changed, 135 insertions(+), 113 deletions(-) diff --git a/src/grid/common/PACKAGE b/src/grid/common/PACKAGE index 018ed21192..4fc10ea5d1 100644 --- a/src/grid/common/PACKAGE +++ b/src/grid/common/PACKAGE @@ -1,5 +1,5 @@ { "description": "Common parts shared by the grid backends", - "requires": ["../../offload"], + "requires": ["../../offload", "../../mpiwrap"], "archive": "libcp2kgridcommon", } diff --git a/src/grid/common/grid_library.c b/src/grid/common/grid_library.c index 99dc871be5..05742cede8 100644 --- a/src/grid/common/grid_library.c +++ b/src/grid/common/grid_library.c @@ -4,6 +4,12 @@ /* */ /* SPDX-License-Identifier: BSD-3-Clause */ /*----------------------------------------------------------------------------*/ +#include "grid_library.h" +#include "grid_common.h" +#include "grid_constants.h" + +#include "../../mpiwrap/cp_mpi.h" +#include "../../offload/offload_runtime.h" #include #include @@ -12,10 +18,8 @@ #include #include -#include "../../offload/offload_runtime.h" -#include "grid_common.h" -#include "grid_constants.h" -#include "grid_library.h" +#define GRID_LIBRARY_PRINT(FN, MSG, OUTPUT_UNIT) \ + ((FN)(MSG, (int)strlen(MSG), OUTPUT_UNIT)) // counter dimensions #define GRID_NBACKENDS 5 @@ -154,15 +158,15 @@ static int compare_counters(const void *a, const void *b) { * \brief Prints statistics gathered by the grid library. * \author Ole Schuett ******************************************************************************/ -void grid_library_print_stats(void (*mpi_sum_func)(long *, int), - const int mpi_comm, - void (*print_func)(char *, int), +void grid_library_print_stats(const int fortran_comm, + void (*print_func)(const char *, int, int), const int output_unit) { if (!library_initialized) { printf("Error: Grid library is not initialized.\n"); abort(); } + const cp_mpi_comm_t comm = cp_mpi_comm_f2c(fortran_comm); // Sum all counters across threads and mpi ranks. const int ncounters = GRID_NBACKENDS * GRID_NKERNELS * GRID_MAX_LP; long counters[ncounters][2]; @@ -173,33 +177,56 @@ void grid_library_print_stats(void (*mpi_sum_func)(long *, int), for (int j = 0; j < max_threads; j++) { counters[i][0] += per_thread_globals[j]->counters[i]; } - mpi_sum_func(&counters[i][0], mpi_comm); + cp_mpi_sum_long(&counters[i][0], 1, comm); total += counters[i][0]; } // Sort counters. qsort(counters, ncounters, 2 * sizeof(long), &compare_counters); + // Determine if anything needs to be printed. + bool print = false; + for (int i = 0; i < ncounters && !print; i++) { + if (counters[i][0] != 0) { + print = true; + } + } + if (!print) { + return; // nothing to be printed + } + // Print counters. - print_func("\n", output_unit); - print_func(" ----------------------------------------------------------------" - "---------------\n", - output_unit); - print_func(" - " - " -\n", - output_unit); - print_func(" - GRID STATISTICS " - " -\n", - output_unit); - print_func(" - " - " -\n", - output_unit); - print_func(" ----------------------------------------------------------------" - "---------------\n", - output_unit); - print_func(" LP KERNEL BACKEND " - "COUNT PERCENT\n", - output_unit); + GRID_LIBRARY_PRINT(print_func, "\n", output_unit); + GRID_LIBRARY_PRINT( + print_func, + " ----------------------------------------------------------------" + "---------------\n", + output_unit); + GRID_LIBRARY_PRINT( + print_func, + " - " + " -\n", + output_unit); + GRID_LIBRARY_PRINT( + print_func, + " - GRID STATISTICS " + " -\n", + output_unit); + GRID_LIBRARY_PRINT( + print_func, + " - " + " -\n", + output_unit); + GRID_LIBRARY_PRINT( + print_func, + " ----------------------------------------------------------------" + "---------------\n", + output_unit); + GRID_LIBRARY_PRINT( + print_func, + " LP KERNEL BACKEND " + "COUNT PERCENT\n", + output_unit); const char *kernel_names[] = {"collocate ortho", "integrate ortho", "collocate general", "integrate general"}; @@ -217,12 +244,14 @@ void grid_library_print_stats(void (*mpi_sum_func)(long *, int), char buffer[100]; snprintf(buffer, sizeof(buffer), " %-5i %-17s %-6s %34li %10.2f%%\n", lp, kernel_names[kern], backend_names[back], counters[i][0], percent); - print_func(buffer, output_unit); + GRID_LIBRARY_PRINT(print_func, buffer, output_unit); } - print_func(" ----------------------------------------------------------------" - "---------------\n", - output_unit); + GRID_LIBRARY_PRINT( + print_func, + " ----------------------------------------------------------------" + "---------------\n", + output_unit); } // EOF diff --git a/src/grid/common/grid_library.h b/src/grid/common/grid_library.h index c94b694dd0..27b04809de 100644 --- a/src/grid/common/grid_library.h +++ b/src/grid/common/grid_library.h @@ -55,8 +55,9 @@ grid_library_config grid_library_get_config(void); * \brief Prints statistics gathered by the grid library. * \author Ole Schuett ******************************************************************************/ -void grid_library_print_stats(void (*mpi_sum_func)(long *, int), int mpi_comm, - void (*print_func)(char *, int), int output_unit); +void grid_library_print_stats(const int fortran_comm, + void (*print_func)(const char *, int, int), + int output_unit); /******************************************************************************* * \brief Various kernels provided by the grid library. diff --git a/src/grid/grid_api.F b/src/grid/grid_api.F index cb8c9d20ad..35bf279289 100644 --- a/src/grid/grid_api.F +++ b/src/grid/grid_api.F @@ -11,13 +11,11 @@ ! ************************************************************************************************** MODULE grid_api USE ISO_C_BINDING, ONLY: & - C_ASSOCIATED, C_BOOL, C_CHAR, C_DOUBLE, C_FUNLOC, C_FUNPTR, C_INT, C_LOC, C_LONG, & - C_NULL_PTR, C_PTR + C_ASSOCIATED, C_BOOL, C_CHAR, C_DOUBLE, C_FUNLOC, C_FUNPTR, C_INT, C_LOC, C_NULL_PTR, C_PTR USE kinds, ONLY: dp USE message_passing, ONLY: mp_comm_type USE offload_api, ONLY: offload_buffer_type USE realspace_grid_types, ONLY: realspace_grid_type - USE string_utilities, ONLY: strlcpy_c2f #include "../base/base_uses.f90" IMPLICIT NONE @@ -1193,10 +1191,9 @@ CONTAINS INTEGER, INTENT(IN) :: output_unit INTERFACE - SUBROUTINE grid_library_print_stats_c(mpi_sum_func, mpi_comm, print_func, output_unit) & + SUBROUTINE grid_library_print_stats_c(mpi_comm, print_func, output_unit) & BIND(C, name="grid_library_print_stats") IMPORT :: C_FUNPTR, C_INT - TYPE(C_FUNPTR), VALUE :: mpi_sum_func INTEGER(KIND=C_INT), VALUE :: mpi_comm TYPE(C_FUNPTR), VALUE :: print_func INTEGER(KIND=C_INT), VALUE :: output_unit @@ -1204,52 +1201,24 @@ CONTAINS END INTERFACE ! Since Fortran units and mpi groups can't be used from C, we pass function pointers instead. - CALL grid_library_print_stats_c(mpi_sum_func=C_FUNLOC(mpi_sum_func), & - mpi_comm=mpi_comm%get_handle(), & + CALL grid_library_print_stats_c(mpi_comm=mpi_comm%get_handle(), & print_func=C_FUNLOC(print_func), & output_unit=output_unit) END SUBROUTINE grid_library_print_stats ! ************************************************************************************************** -!> \brief Callback to run mpi_sum on a Fortran MPI communicator. -!> \param number ... -!> \param mpi_comm ... -!> \author Ole Schuett +!> \brief Callback to write to a Fortran output unit (called by C-side). +!> \param msg to be printed. +!> \param msglen number of characters excluding the terminating character. +!> \param output_unit used for output. +!> \author Ole Schuett and Hans Pabst ! ************************************************************************************************** - SUBROUTINE mpi_sum_func(number, mpi_comm) BIND(C, name="grid_api_mpi_sum_func") - INTEGER(KIND=C_LONG), INTENT(INOUT) :: number - INTEGER(KIND=C_INT), INTENT(IN), VALUE :: mpi_comm + SUBROUTINE print_func(msg, msglen, output_unit) BIND(C, name="grid_api_print_func") + CHARACTER(KIND=C_CHAR), INTENT(IN) :: msg(*) + INTEGER(KIND=C_INT), INTENT(IN), VALUE :: msglen, output_unit - TYPE(mp_comm_type) :: my_mpi_comm - - ! Convert the handle to the default integer kind and convert it to the communicator type - CALL my_mpi_comm%set_handle(INT(mpi_comm)) - - CALL my_mpi_comm%sum(number) - END SUBROUTINE mpi_sum_func - -! ************************************************************************************************** -!> \brief Callback to write to a Fortran output unit. -!> \param message ... -!> \param output_unit ... -!> \author Ole Schuett -! ************************************************************************************************** - SUBROUTINE print_func(message, output_unit) BIND(C, name="grid_api_print_func") - CHARACTER(LEN=1, KIND=C_CHAR), INTENT(IN) :: message(*) - INTEGER(KIND=C_INT), INTENT(IN), VALUE :: output_unit - - CHARACTER(LEN=1000) :: buffer - INTEGER :: nchars - - IF (output_unit <= 0) & - RETURN - - ! Convert C char array into Fortran string. - nchars = strlcpy_c2f(buffer, message) - - ! Print the message. - WRITE (output_unit, FMT="(A)", ADVANCE="NO") buffer(1:nchars) + IF (output_unit <= 0) RETURN ! Omit to print the message. + WRITE (output_unit, FMT="(100A)", ADVANCE="NO") msg(1:msglen) END SUBROUTINE print_func - END MODULE grid_api diff --git a/src/grid/grid_miniapp.c b/src/grid/grid_miniapp.c index a3e329ceee..8d01c93979 100644 --- a/src/grid/grid_miniapp.c +++ b/src/grid/grid_miniapp.c @@ -4,23 +4,24 @@ /* */ /* SPDX-License-Identifier: BSD-3-Clause */ /*----------------------------------------------------------------------------*/ +#include "../offload/offload_library.h" +#include "../offload/offload_mempool.h" +#include "common/grid_library.h" +#include "grid_replay.h" #include #include #include -#include "../offload/offload_library.h" -#include "common/grid_library.h" -#include "grid_replay.h" - -void mpi_sum_func(long *number, int mpi_comm) { - *number += 0; // Nothing todo without MPI, pretend arguments are used anyways. - mpi_comm += 0; -} - -void print_func(char *message, int output_unit) { - output_unit += 0; // Pretend argument is used. - printf("%s", message); +/******************************************************************************* + * \brief Wrapper for printf, passed to grid_library_print_stats. + * \author Ole Schuett + ******************************************************************************/ +static void print_func(const char *msg, int msglen, int output_unit) { + (void)msglen; // mark used + if (output_unit == 0) { // i.e. my_rank == 0 + printf("%s", msg); + } } /******************************************************************************* @@ -71,11 +72,13 @@ int main(int argc, char *argv[]) { const bool success = grid_replay(argv[iarg++], cycles, collocate, batch, cycles_per_block, tolerance); - grid_library_print_stats(&mpi_sum_func, 0, &print_func, 0); - grid_library_finalize(); - - if (!success) { + if (success) { + grid_library_print_stats(0 /*fortran_comm*/, &print_func, 0 /*rank*/); + offload_mempool_stats_print(0 /*fortran_comm*/, &print_func, 0 /*rank*/); + grid_library_finalize(); + } else { fprintf(stderr, "Error: Maximal difference is too large.\n"); + grid_library_finalize(); return 2; } diff --git a/src/grid/grid_unittest.c b/src/grid/grid_unittest.c index 24f27845a0..ab61002c88 100644 --- a/src/grid/grid_unittest.c +++ b/src/grid/grid_unittest.c @@ -4,36 +4,29 @@ /* */ /* SPDX-License-Identifier: BSD-3-Clause */ /*----------------------------------------------------------------------------*/ +#include "../offload/offload_library.h" +#include "../offload/offload_mempool.h" +#include "common/grid_library.h" +#include "grid_replay.h" #include #include #include -#include "../offload/offload_library.h" -#include "common/grid_library.h" -#include "grid_replay.h" - // Only used to call MPI_Init and MPI_Finalize to avoid spurious MPI error. #if defined(__parallel) #include #endif -/******************************************************************************* - * \brief Standin for mpi_sum, passed to grid_library_print_stats. - * \author Ole Schuett - ******************************************************************************/ -static void mpi_sum_func(long *number, int mpi_comm) { - (void)number; // mark used - (void)mpi_comm; -} - /******************************************************************************* * \brief Wrapper for printf, passed to grid_library_print_stats. * \author Ole Schuett ******************************************************************************/ -static void print_func(char *message, int output_unit) { - (void)output_unit; // mark used - printf("%s", message); +static void print_func(const char *msg, int msglen, int output_unit) { + (void)msglen; // mark used + if (output_unit == 0) { // i.e. my_rank == 0 + printf("%s", msg); + } } /******************************************************************************* @@ -98,12 +91,13 @@ int main(int argc, char *argv[]) { errors += run_test(argv[1], "general_subpatch16.task"); errors += run_test(argv[1], "general_overflow.task"); - grid_library_print_stats(&mpi_sum_func, 0, &print_func, 0); - grid_library_finalize(); - if (errors == 0) { + grid_library_print_stats(0 /*fortran_comm*/, &print_func, 0 /*rank*/); + offload_mempool_stats_print(0 /*fortran_comm*/, &print_func, 0 /*rank*/); + grid_library_finalize(); printf("\nAll tests have passed :-)\n"); } else { + grid_library_finalize(); printf("\nFound %i errors :-(\n", errors); } diff --git a/src/mpiwrap/cp_mpi.c b/src/mpiwrap/cp_mpi.c index 8d5535fffd..42610fb454 100644 --- a/src/mpiwrap/cp_mpi.c +++ b/src/mpiwrap/cp_mpi.c @@ -318,6 +318,26 @@ void cp_mpi_sum_int(int *values, const int count, const cp_mpi_comm_t comm) { #endif } +/******************************************************************************* + * \brief Wrapper around MPI_Allreduce for op MPI_SUM and datatype MPI_LONG. + * \author Hans Pabst + ******************************************************************************/ +void cp_mpi_sum_long(long *values, const int count, const cp_mpi_comm_t comm) { +#if defined(__parallel) + long value = 0; + void *recvbuf = (1 < count ? cp_mpi_alloc_mem(count * sizeof(long)) : &value); + CHECK(MPI_Allreduce(values, recvbuf, count, MPI_LONG, MPI_SUM, comm)); + memcpy(values, recvbuf, count * sizeof(long)); + if (1 < count) { + cp_mpi_free_mem(recvbuf); + } +#else + (void)comm; // mark used + (void)values; + (void)count; +#endif +} + /******************************************************************************* * \brief Wrapper around MPI_Allreduce for op MPI_SUM and datatype MPI_INT64_T. * \author Ole Schuett diff --git a/src/mpiwrap/cp_mpi.h b/src/mpiwrap/cp_mpi.h index 56d6ea9850..5dbe03875e 100644 --- a/src/mpiwrap/cp_mpi.h +++ b/src/mpiwrap/cp_mpi.h @@ -133,6 +133,12 @@ void cp_mpi_max_double(double *values, const int count, ******************************************************************************/ void cp_mpi_sum_int(int *values, const int count, const cp_mpi_comm_t comm); +/******************************************************************************* + * \brief Wrapper around MPI_Allreduce for op MPI_SUM and datatype MPI_LONG. + * \author Hans Pabst + ******************************************************************************/ +void cp_mpi_sum_long(long *values, const int count, const cp_mpi_comm_t comm); + /******************************************************************************* * \brief Wrapper around MPI_Allreduce for op MPI_SUM and datatype MPI_INT64_T. * \author Ole Schuett