diff --git a/src/dbm/dbm_mempool.c b/src/dbm/dbm_mempool.c index ad1a87b010..9825c46326 100644 --- a/src/dbm/dbm_mempool.c +++ b/src/dbm/dbm_mempool.c @@ -19,60 +19,6 @@ #include "dbm_mempool.h" #include "dbm_mpi.h" -/******************************************************************************* - * \brief Private routine for actually allocating system memory. - * \author Ole Schuett - ******************************************************************************/ -static void *actual_malloc(const size_t size, const bool on_device) { - void *memory = NULL; - - if (0 != size) { -#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_DBM) - if (on_device) { - offload_activate_chosen_device(); - offloadMalloc(&memory, size); - } else { -#if (0 != DBM_OFFLOAD_ALLOC) - offloadMallocHost(&memory, size); -#else - memory = dbm_mpi_alloc_mem(size); -#endif - } -#else - (void)on_device; // mark used - memory = dbm_mpi_alloc_mem(size); -#endif - assert(memory != NULL); - } - - return memory; -} - -/******************************************************************************* - * \brief Private routine for actually freeing system memory. - * \author Ole Schuett - ******************************************************************************/ -static void actual_free(const void *memory, const bool on_device) { - if (NULL != memory) { - void *mem = (void *)(uintptr_t)memory; -#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_DBM) - if (on_device) { - offload_activate_chosen_device(); - offloadFree(mem); - } else { -#if (0 != DBM_OFFLOAD_ALLOC) - offloadFreeHost(mem); -#else - dbm_mpi_free_mem(mem); -#endif - } -#else - (void)on_device; // mark used - dbm_mpi_free_mem(mem); -#endif - } -} - /******************************************************************************* * \brief Private struct for storing a chunk of memory. * \author Ole Schuett @@ -97,6 +43,75 @@ static dbm_memchunk_t *mempool_host_available_head = NULL; static dbm_memchunk_t *mempool_device_allocated_head = NULL; static dbm_memchunk_t *mempool_host_allocated_head = NULL; +/******************************************************************************* + * \brief Private statistics. + * \author Hans Pabst + ******************************************************************************/ +static dbm_memstats_t mempool_stats = {0}; + +/******************************************************************************* + * \brief Private routine for actually allocating system memory. + * \author Ole Schuett + ******************************************************************************/ +static void *actual_malloc(size_t size, bool on_device) { + void *memory = NULL; + + if (0 != size) { +#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_DBM) + if (on_device) { + offload_activate_chosen_device(); + offloadMalloc(&memory, size); + } else { +#if (0 != DBM_OFFLOAD_ALLOC) + offloadMallocHost(&memory, size); +#else + memory = dbm_mpi_alloc_mem(size); +#endif + } +#else + (void)on_device; // mark used + memory = dbm_mpi_alloc_mem(size); +#endif + assert(memory != NULL); + + // Update statistics. + if (on_device) { +#pragma omp atomic + ++mempool_stats.device_mallocs; + } else { +#pragma omp atomic + ++mempool_stats.host_mallocs; + } + } + + return memory; +} + +/******************************************************************************* + * \brief Private routine for actually freeing system memory. + * \author Ole Schuett + ******************************************************************************/ +static void actual_free(const void *memory, bool on_device) { + if (NULL != memory) { + void *mem = (void *)(uintptr_t)memory; +#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_DBM) + if (on_device) { + offload_activate_chosen_device(); + offloadFree(mem); + } else { +#if (0 != DBM_OFFLOAD_ALLOC) + offloadFreeHost(mem); +#else + dbm_mpi_free_mem(mem); +#endif + } +#else + (void)on_device; // mark used + dbm_mpi_free_mem(mem); +#endif + } +} + /******************************************************************************* * \brief Private routine for allocating host or device memory from the pool. * \author Ole Schuett @@ -104,7 +119,7 @@ static dbm_memchunk_t *mempool_host_allocated_head = NULL; #if (0 != DBM_MEMPOOL_DEVICE) || (0 != DBM_MEMPOOL_HOST) static void *internal_mempool_malloc(dbm_memchunk_t **available_head, dbm_memchunk_t **allocated_head, - const size_t size) { + size_t size) { if (size == 0) { return NULL; } @@ -119,23 +134,25 @@ static void *internal_mempool_malloc(dbm_memchunk_t **available_head, // Find a suitable chunk in mempool_available. dbm_memchunk_t **reuse = NULL, **reclaim = NULL; for (; NULL != *available_head; available_head = &(*available_head)->next) { - const size_t cur_size = (*available_head)->size; - if (cur_size < size) { // needs reallocation - if (NULL == reclaim || (*reclaim)->size < cur_size) { - reclaim = available_head; - } - } else if (NULL == reuse || cur_size < (*reuse)->size) { + const size_t s = (*available_head)->size; + if (size <= s && (NULL == reuse || s < (*reuse)->size)) { reuse = available_head; if (size == (*reuse)->size) { break; // exact match } + } else if (NULL != reclaim) { + if (s > (*reclaim)->size) { + reclaim = available_head; + } + } else { + reclaim = available_head; } } if (NULL == reuse) { reuse = reclaim; } - // If a chunck was found, remove it from mempool_available. + // Remove chunk from mempool_available. if (NULL != reuse) { chunk = *reuse; *reuse = chunk->next; @@ -149,11 +166,13 @@ static void *internal_mempool_malloc(dbm_memchunk_t **available_head, *allocated_head = chunk; } - // Resize chunk if needed (not in critical section) + // Resize chunk (not in critical section). if (chunk->size < size) { - actual_free(chunk->mem, on_device); + void *memory = chunk->mem; + chunk->mem = NULL; // race ok (free and stats) + actual_free(memory, on_device); chunk->mem = actual_malloc(size, on_device); - chunk->size = size; // update + chunk->size = size; } chunk->used = size; // stats @@ -165,7 +184,7 @@ static void *internal_mempool_malloc(dbm_memchunk_t **available_head, * \brief Internal routine for allocating host memory from the pool. * \author Ole Schuett ******************************************************************************/ -void *dbm_mempool_host_malloc(const size_t size) { +void *dbm_mempool_host_malloc(size_t size) { #if (0 != DBM_MEMPOOL_HOST) return internal_mempool_malloc(&mempool_host_available_head, &mempool_host_allocated_head, size); @@ -178,7 +197,7 @@ void *dbm_mempool_host_malloc(const size_t size) { * \brief Internal routine for allocating device memory from the pool * \author Ole Schuett ******************************************************************************/ -void *dbm_mempool_device_malloc(const size_t size) { +void *dbm_mempool_device_malloc(size_t size) { #if (0 != DBM_MEMPOOL_DEVICE) #if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_DBM) return internal_mempool_malloc(&mempool_device_available_head, @@ -250,29 +269,36 @@ void dbm_mempool_device_free(const void *memory) { #endif } +/******************************************************************************* + * \brief Private routine for freeing all memory in the pool. + * \author Ole Schuett + ******************************************************************************/ +static void internal_mempool_clear(dbm_memchunk_t **available_head) { + const bool on_device = (&mempool_device_available_head == available_head); + assert(on_device || &mempool_host_available_head == available_head); + + // Free chunks in mempool_available. + while (NULL != *available_head) { + dbm_memchunk_t *chunk = *available_head; + *available_head = chunk->next; // remove chunk + actual_free(chunk->mem, on_device); + free(chunk); + } +} + /******************************************************************************* * \brief Internal routine for freeing all memory in the pool. * \author Ole Schuett ******************************************************************************/ void dbm_mempool_clear(void) { - assert(omp_get_num_threads() == 1); - // check for memory leak assert(mempool_device_allocated_head == NULL); assert(mempool_host_allocated_head == NULL); - // Free chunks in mempool_available. - while (mempool_device_available_head != NULL) { - dbm_memchunk_t *chunk = mempool_device_available_head; - mempool_device_available_head = chunk->next; - actual_free(chunk->mem, true); - free(chunk); - } - while (mempool_host_available_head != NULL) { - dbm_memchunk_t *chunk = mempool_host_available_head; - mempool_host_available_head = chunk->next; - actual_free(chunk->mem, false); - free(chunk); +#pragma omp critical(dbm_mempool_modify) + { + internal_mempool_clear(&mempool_device_available_head); + internal_mempool_clear(&mempool_host_available_head); } } @@ -281,35 +307,44 @@ void dbm_mempool_clear(void) { * \author Hans Pabst ******************************************************************************/ void dbm_mempool_statistics(dbm_memstats_t *memstats) { - dbm_memchunk_t *chunk; assert(NULL != memstats); - memset(memstats, 0, sizeof(*memstats)); + memset(memstats, 0, sizeof(dbm_memstats_t)); #pragma omp critical(dbm_mempool_modify) { + dbm_memchunk_t *chunk; for (chunk = mempool_device_available_head; NULL != chunk; chunk = chunk->next) { memstats->device_used += chunk->used; memstats->device_size += chunk->size; - ++memstats->device_mallocs; } for (chunk = mempool_device_allocated_head; NULL != chunk; chunk = chunk->next) { memstats->device_used += chunk->used; memstats->device_size += chunk->size; - ++memstats->device_mallocs; } for (chunk = mempool_host_available_head; NULL != chunk; chunk = chunk->next) { memstats->host_used += chunk->used; memstats->host_size += chunk->size; - ++memstats->host_mallocs; } for (chunk = mempool_host_allocated_head; NULL != chunk; chunk = chunk->next) { memstats->host_used += chunk->used; memstats->host_size += chunk->size; - ++memstats->host_mallocs; } + if (mempool_stats.device_used < memstats->device_used) { + mempool_stats.device_used = memstats->device_used; + } + if (mempool_stats.device_size < memstats->device_size) { + mempool_stats.device_size = memstats->device_size; + } + if (mempool_stats.host_used < memstats->host_used) { + mempool_stats.host_used = memstats->host_used; + } + if (mempool_stats.host_size < memstats->host_size) { + mempool_stats.host_size = memstats->host_size; + } + memcpy(memstats, &mempool_stats, sizeof(dbm_memstats_t)); } } diff --git a/src/dbm/dbm_mempool.h b/src/dbm/dbm_mempool.h index 965955d025..d5deefcc87 100644 --- a/src/dbm/dbm_mempool.h +++ b/src/dbm/dbm_mempool.h @@ -17,13 +17,13 @@ extern "C" { * \brief Internal routine for allocating host memory from the pool. * \author Ole Schuett ******************************************************************************/ -void *dbm_mempool_host_malloc(const size_t size); +void *dbm_mempool_host_malloc(size_t size); /******************************************************************************* * \brief Internal routine for allocating device memory from the pool. * \author Ole Schuett ******************************************************************************/ -void *dbm_mempool_device_malloc(const size_t size); +void *dbm_mempool_device_malloc(size_t size); /******************************************************************************* * \brief Internal routine for releasing memory back to the pool. diff --git a/src/dbm/dbm_mpi.c b/src/dbm/dbm_mpi.c index e3fe4faf49..63a2eba04f 100644 --- a/src/dbm/dbm_mpi.c +++ b/src/dbm/dbm_mpi.c @@ -479,7 +479,7 @@ void dbm_mpi_alltoallv_double(const double *sendbuf, const int *sendcounts, ******************************************************************************/ void *dbm_mpi_alloc_mem(size_t size) { void *result = NULL; -#if defined(__parallel) && !defined(OPEN_MPI) +#if defined(__parallel) CHECK(MPI_Alloc_mem((MPI_Aint)size, MPI_INFO_NULL, &result)); #else result = malloc(size); @@ -492,7 +492,7 @@ void *dbm_mpi_alloc_mem(size_t size) { * \author Hans Pabst ******************************************************************************/ void dbm_mpi_free_mem(void *mem) { -#if defined(__parallel) && !defined(OPEN_MPI) +#if defined(__parallel) CHECK(MPI_Free_mem(mem)); #else free(mem); diff --git a/src/dbm/dbm_multiply_opencl.c b/src/dbm/dbm_multiply_opencl.c index 0beb085b0d..0464e81535 100644 --- a/src/dbm/dbm_multiply_opencl.c +++ b/src/dbm/dbm_multiply_opencl.c @@ -169,10 +169,8 @@ void dbm_multiply_gpu_launch_kernel(const offloadStream_t stream, double alpha, fprintf(stderr, "INFO ACC/LIBDBM: DBM-kernel failed to generate\n"); } } - kernel = kernel_global; - } else { - kernel = clCloneKernel(kernel_global, &result); } + kernel = clCloneKernel(kernel_global, &result); /* always clone */ ACC_OPENCL_RELEASE(config->lock_main); } else if (NULL == kernel) { kernel = clCloneKernel(kernel_global, &result);