From 2eb13e25da2bd313fcf7ea48b839b5eca8a8a76d Mon Sep 17 00:00:00 2001 From: Hans Pabst Date: Wed, 29 Jan 2025 21:04:46 +0100 Subject: [PATCH] DBM: statistics for pooled memory (#3917) - Print number of allocations, and memory consumption. - MPI: avoid memory allocation for reductions (count=1). - MPI: dbm_mpi_max_uint64, dbm_mpi_sum_uint64 (unused). - Stats are based on the max-metric over all ranks. --- src/dbm/dbm_library.c | 41 +++++++++++++++++-- src/dbm/dbm_mempool.c | 35 ++++++++++++++-- src/dbm/dbm_mempool.h | 21 +++++++++- src/dbm/dbm_mpi.c | 82 +++++++++++++++++++++++++++++++++----- src/dbm/dbm_mpi.h | 14 +++++++ src/dbm/dbm_multiply_cpu.h | 2 - 6 files changed, 173 insertions(+), 22 deletions(-) diff --git a/src/dbm/dbm_library.c b/src/dbm/dbm_library.c index 99b2417b83..3bde9e1b0f 100644 --- a/src/dbm/dbm_library.c +++ b/src/dbm/dbm_library.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -131,8 +130,7 @@ void dbm_library_print_stats(const int fortran_comm, const dbm_mpi_comm_t comm = dbm_mpi_comm_f2c(fortran_comm); // Sum all counters across threads and mpi ranks. - int64_t counters[DBM_NUM_COUNTERS][2]; - memset(counters, 0, DBM_NUM_COUNTERS * 2 * sizeof(int64_t)); + int64_t counters[DBM_NUM_COUNTERS][2] = {{0}}; double total = 0.0; for (int i = 0; i < DBM_NUM_COUNTERS; i++) { counters[i][1] = i; // needed as inverse index after qsort @@ -168,6 +166,7 @@ void dbm_library_print_stats(const int fortran_comm, output_unit); const char *labels[] = {"?", "??", "???", ">999"}; + char buffer[100]; for (int i = 0; i < DBM_NUM_COUNTERS; i++) { if (counters[i][0] == 0) { continue; // skip empty counters @@ -177,7 +176,6 @@ void dbm_library_print_stats(const int fortran_comm, const int m = (idx % 64) / 16; const int n = (idx % 16) / 4; const int k = (idx % 4) / 1; - char buffer[100]; snprintf(buffer, sizeof(buffer), " %4s x %4s x %4s %46" PRId64 " %10.2f%%\n", labels[m], labels[n], labels[k], counters[i][0], percent); @@ -187,6 +185,41 @@ void dbm_library_print_stats(const int fortran_comm, print_func(" ----------------------------------------------------------------" "---------------\n", output_unit); + + dbm_memstats_t memstats; + dbm_mempool_statistics(&memstats); + dbm_mpi_max_uint64(&memstats.device_mallocs, 1, comm); + dbm_mpi_max_uint64(&memstats.host_mallocs, 1, comm); + + if (0 != memstats.device_mallocs || 0 != memstats.host_mallocs) { + print_func(" Memory consumption " + " Number of allocations Size [MiB]\n", + output_unit); + } + if (0 < memstats.device_mallocs) { + dbm_mpi_max_uint64(&memstats.device_size, 1, comm); + snprintf(buffer, sizeof(buffer), + " Device " + " %20" PRIuPTR " %10" PRIuPTR "\n", + (uintptr_t)memstats.device_mallocs, + (uintptr_t)((memstats.device_size + (512U << 10)) >> 20)); + print_func(buffer, output_unit); + } + if (0 < memstats.host_mallocs) { + dbm_mpi_max_uint64(&memstats.host_size, 1, comm); + snprintf(buffer, sizeof(buffer), + " Host " + " %20" PRIuPTR " %10" PRIuPTR "\n", + (uintptr_t)memstats.host_mallocs, + (uintptr_t)((memstats.host_size + (512U << 10)) >> 20)); + print_func(buffer, output_unit); + } + if (0 < memstats.device_mallocs || 0 < memstats.host_mallocs) { + print_func( + " ----------------------------------------------------------------" + "---------------\n", + output_unit); + } } // EOF diff --git a/src/dbm/dbm_mempool.c b/src/dbm/dbm_mempool.c index 7e4c4b6362..58619ada4c 100644 --- a/src/dbm/dbm_mempool.c +++ b/src/dbm/dbm_mempool.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -87,6 +88,12 @@ static dbm_memchunk_t *mempool_available_head = NULL; ******************************************************************************/ static dbm_memchunk_t *mempool_allocated_head = NULL; +/******************************************************************************* + * \brief Private statistics (survives dbm_mempool_clear). + * \author Hans Pabst + ******************************************************************************/ +static dbm_memstats_t mempool_stats = {0}; + /******************************************************************************* * \brief Private routine for allocating host or device memory from the pool. * \author Ole Schuett @@ -104,7 +111,7 @@ static void *internal_mempool_malloc(const size_t size, const bool on_device) { dbm_memchunk_t **indirect = &mempool_available_head, **hit = NULL; while (*indirect != NULL) { if ((*indirect)->on_device == on_device) { - const size_t max_size = (size_t)(ALLOCATION_FACTOR * size + 0.5); + const size_t max_size = (size_t)(ALLOCATION_FACTOR * size /*+ 0.5*/); const size_t hit_size = (*indirect)->size; if (NULL == hit) { // Fallback hit = indirect; @@ -134,13 +141,24 @@ static void *internal_mempool_malloc(const size_t size, const bool on_device) { // Insert chunk into mempool_allocated. chunk->next = mempool_allocated_head; mempool_allocated_head = chunk; + + // Update statistics + if (chunk->size < size) { + if (on_device) { + mempool_stats.device_size += size - chunk->size; + ++mempool_stats.device_mallocs; + } else { + mempool_stats.host_size += size - chunk->size; + ++mempool_stats.host_mallocs; + } + } } // Resize chunk if needed (outside of critical section). if (chunk->size < size) { actual_free(chunk->mem, chunk->on_device); chunk->mem = actual_malloc(size, chunk->on_device); - chunk->size = size; + chunk->size = size; // update } return chunk->mem; @@ -191,7 +209,7 @@ void dbm_mempool_free(void *mem) { } /******************************************************************************* - * \brief Internal routine for freeing all memory in the pool. + * \brief Internal routine for freeing all memory in the pool (not thread-safe). * \author Ole Schuett ******************************************************************************/ void dbm_mempool_clear(void) { @@ -206,7 +224,7 @@ void dbm_mempool_clear(void) { // free(chunk); //} - // Free chunks in mempool_avavailable. + // Free chunks in mempool_available. while (mempool_available_head != NULL) { dbm_memchunk_t *chunk = mempool_available_head; mempool_available_head = chunk->next; @@ -215,4 +233,13 @@ void dbm_mempool_clear(void) { } } +/******************************************************************************* + * \brief Internal routine to query statistics (not thread-safe). + * \author Hans Pabst + ******************************************************************************/ +void dbm_mempool_statistics(dbm_memstats_t *memstats) { + assert(NULL != memstats); + *memstats = mempool_stats; +} + // EOF diff --git a/src/dbm/dbm_mempool.h b/src/dbm/dbm_mempool.h index a0d4e41158..b03566251d 100644 --- a/src/dbm/dbm_mempool.h +++ b/src/dbm/dbm_mempool.h @@ -11,7 +11,7 @@ extern "C" { #endif -#include +#include /******************************************************************************* * \brief Internal routine for allocating host memory from the pool. @@ -32,11 +32,28 @@ void *dbm_mempool_device_malloc(const size_t size); void dbm_mempool_free(void *memory); /******************************************************************************* - * \brief Internal routine for freeing all memory in the pool. + * \brief Internal routine for freeing all memory in the pool (not thread-safe). * \author Ole Schuett ******************************************************************************/ void dbm_mempool_clear(void); +/******************************************************************************* + * \brief Internal struct for pool statistics. + * \author Hans Pabst + ******************************************************************************/ +typedef struct dbm_memstats_t { + // Memory consumption (maximum). + uint64_t host_size, device_size; + // Number of allocations. + uint64_t host_mallocs, device_mallocs; +} dbm_memstats_t; + +/******************************************************************************* + * \brief Internal routine to query statistics (not thread-safe). + * \author Hans Pabst + ******************************************************************************/ +void dbm_mempool_statistics(dbm_memstats_t *memstats); + #ifdef __cplusplus } #endif diff --git a/src/dbm/dbm_mpi.c b/src/dbm/dbm_mpi.c index 72ab7c54e9..91ac3910b1 100644 --- a/src/dbm/dbm_mpi.c +++ b/src/dbm/dbm_mpi.c @@ -237,10 +237,35 @@ bool dbm_mpi_comms_are_similar(const dbm_mpi_comm_t comm1, ******************************************************************************/ void dbm_mpi_max_int(int *values, const int count, const dbm_mpi_comm_t comm) { #if defined(__parallel) - void *recvbuf = dbm_mpi_alloc_mem(count * sizeof(int)); + int value = 0; + void *recvbuf = (0 < count ? dbm_mpi_alloc_mem(count * sizeof(int)) : &value); CHECK(MPI_Allreduce(values, recvbuf, count, MPI_INT, MPI_MAX, comm)); memcpy(values, recvbuf, count * sizeof(int)); - dbm_mpi_free_mem(recvbuf); + if (0 < count) { + dbm_mpi_free_mem(recvbuf); + } +#else + (void)comm; // mark used + (void)values; + (void)count; +#endif +} + +/******************************************************************************* + * \brief Wrapper around MPI_Allreduce for op MPI_MAX and datatype MPI_UINT64_T. + * \author Ole Schuett + ******************************************************************************/ +void dbm_mpi_max_uint64(uint64_t *values, const int count, + const dbm_mpi_comm_t comm) { +#if defined(__parallel) + uint64_t value = 0; + void *recvbuf = + (0 < count ? dbm_mpi_alloc_mem(count * sizeof(uint64_t)) : &value); + CHECK(MPI_Allreduce(values, recvbuf, count, MPI_UINT64_T, MPI_MAX, comm)); + memcpy(values, recvbuf, count * sizeof(uint64_t)); + if (0 < count) { + dbm_mpi_free_mem(recvbuf); + } #else (void)comm; // mark used (void)values; @@ -255,10 +280,14 @@ void dbm_mpi_max_int(int *values, const int count, const dbm_mpi_comm_t comm) { void dbm_mpi_max_double(double *values, const int count, const dbm_mpi_comm_t comm) { #if defined(__parallel) - void *recvbuf = dbm_mpi_alloc_mem(count * sizeof(double)); + double value = 0; + void *recvbuf = + (0 < count ? dbm_mpi_alloc_mem(count * sizeof(double)) : &value); CHECK(MPI_Allreduce(values, recvbuf, count, MPI_DOUBLE, MPI_MAX, comm)); memcpy(values, recvbuf, count * sizeof(double)); - dbm_mpi_free_mem(recvbuf); + if (0 < count) { + dbm_mpi_free_mem(recvbuf); + } #else (void)comm; // mark used (void)values; @@ -272,10 +301,13 @@ void dbm_mpi_max_double(double *values, const int count, ******************************************************************************/ void dbm_mpi_sum_int(int *values, const int count, const dbm_mpi_comm_t comm) { #if defined(__parallel) - void *recvbuf = dbm_mpi_alloc_mem(count * sizeof(int)); + int value = 0; + void *recvbuf = (0 < count ? dbm_mpi_alloc_mem(count * sizeof(int)) : &value); CHECK(MPI_Allreduce(values, recvbuf, count, MPI_INT, MPI_SUM, comm)); memcpy(values, recvbuf, count * sizeof(int)); - dbm_mpi_free_mem(recvbuf); + if (0 < count) { + dbm_mpi_free_mem(recvbuf); + } #else (void)comm; // mark used (void)values; @@ -290,10 +322,36 @@ void dbm_mpi_sum_int(int *values, const int count, const dbm_mpi_comm_t comm) { void dbm_mpi_sum_int64(int64_t *values, const int count, const dbm_mpi_comm_t comm) { #if defined(__parallel) - void *recvbuf = dbm_mpi_alloc_mem(count * sizeof(int64_t)); + int64_t value = 0; + void *recvbuf = + (0 < count ? dbm_mpi_alloc_mem(count * sizeof(int64_t)) : &value); CHECK(MPI_Allreduce(values, recvbuf, count, MPI_INT64_T, MPI_SUM, comm)); memcpy(values, recvbuf, count * sizeof(int64_t)); - dbm_mpi_free_mem(recvbuf); + if (0 < count) { + dbm_mpi_free_mem(recvbuf); + } +#else + (void)comm; // mark used + (void)values; + (void)count; +#endif +} + +/******************************************************************************* + * \brief Wrapper around MPI_Allreduce for op MPI_SUM and datatype MPI_UINT64_T. + * \author Hans Pabst + ******************************************************************************/ +void dbm_mpi_sum_uint64(uint64_t *values, const int count, + const dbm_mpi_comm_t comm) { +#if defined(__parallel) + uint64_t value = 0; + void *recvbuf = + (0 < count ? dbm_mpi_alloc_mem(count * sizeof(uint64_t)) : &value); + CHECK(MPI_Allreduce(values, recvbuf, count, MPI_UINT64_T, MPI_SUM, comm)); + memcpy(values, recvbuf, count * sizeof(uint64_t)); + if (0 < count) { + dbm_mpi_free_mem(recvbuf); + } #else (void)comm; // mark used (void)values; @@ -308,10 +366,14 @@ void dbm_mpi_sum_int64(int64_t *values, const int count, void dbm_mpi_sum_double(double *values, const int count, const dbm_mpi_comm_t comm) { #if defined(__parallel) - void *recvbuf = dbm_mpi_alloc_mem(count * sizeof(double)); + double value = 0; + void *recvbuf = + (0 < count ? dbm_mpi_alloc_mem(count * sizeof(double)) : &value); CHECK(MPI_Allreduce(values, recvbuf, count, MPI_DOUBLE, MPI_SUM, comm)); memcpy(values, recvbuf, count * sizeof(double)); - dbm_mpi_free_mem(recvbuf); + if (0 < count) { + dbm_mpi_free_mem(recvbuf); + } #else (void)comm; // mark used (void)values; diff --git a/src/dbm/dbm_mpi.h b/src/dbm/dbm_mpi.h index f63982d216..c9b502a627 100644 --- a/src/dbm/dbm_mpi.h +++ b/src/dbm/dbm_mpi.h @@ -114,6 +114,13 @@ bool dbm_mpi_comms_are_similar(const dbm_mpi_comm_t comm1, ******************************************************************************/ void dbm_mpi_max_int(int *values, const int count, const dbm_mpi_comm_t comm); +/******************************************************************************* + * \brief Wrapper around MPI_Allreduce for op MPI_MAX and datatype MPI_UINT64_T. + * \author Hans Pabst + ******************************************************************************/ +void dbm_mpi_max_uint64(uint64_t *values, const int count, + const dbm_mpi_comm_t comm); + /******************************************************************************* * \brief Wrapper around MPI_Allreduce for op MPI_MAX and datatype MPI_DOUBLE. * \author Ole Schuett @@ -134,6 +141,13 @@ void dbm_mpi_sum_int(int *values, const int count, const dbm_mpi_comm_t comm); void dbm_mpi_sum_int64(int64_t *values, const int count, const dbm_mpi_comm_t comm); +/******************************************************************************* + * \brief Wrapper around MPI_Allreduce for op MPI_SUM and datatype MPI_UINT64_T. + * \author Hans Pabst + ******************************************************************************/ +void dbm_mpi_sum_uint64(uint64_t *values, const int count, + const dbm_mpi_comm_t comm); + /******************************************************************************* * \brief Wrapper around MPI_Allreduce for op MPI_SUM and datatype MPI_DOUBLE. * \author Ole Schuett diff --git a/src/dbm/dbm_multiply_cpu.h b/src/dbm/dbm_multiply_cpu.h index de43a9af56..1e9244f622 100644 --- a/src/dbm/dbm_multiply_cpu.h +++ b/src/dbm/dbm_multiply_cpu.h @@ -7,8 +7,6 @@ #ifndef DBM_MULTIPLY_CPU_H #define DBM_MULTIPLY_CPU_H -#include - #include "dbm_multiply_internal.h" #include "dbm_shard.h"