DBM: allow actual allocation outside of critical section (#4088)

- Reverted "DBM: adjusted MPI-based memory allocation".
- More accurate statistics (counted allocations).
- Made stats valid for non-pooled allocations.
- Check leaks in dbm_mempool_clear (internal_mempool_clear).
- Removed thread-safety limitation (mempool_clear).
- OpenCL: fixed an issue cloning the initial kernel.

src/dbm/dbm_mempool.c

@@ -19,60 +19,6 @@
 #include "dbm_mempool.h"
 #include "dbm_mpi.h"
 
-/*******************************************************************************
- * \brief Private routine for actually allocating system memory.
- * \author Ole Schuett
- ******************************************************************************/
-static void *actual_malloc(const size_t size, const bool on_device) {
-  void *memory = NULL;
-
-  if (0 != size) {
-#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_DBM)
-    if (on_device) {
-      offload_activate_chosen_device();
-      offloadMalloc(&memory, size);
-    } else {
-#if (0 != DBM_OFFLOAD_ALLOC)
-      offloadMallocHost(&memory, size);
-#else
-      memory = dbm_mpi_alloc_mem(size);
-#endif
-    }
-#else
-    (void)on_device; // mark used
-    memory = dbm_mpi_alloc_mem(size);
-#endif
-    assert(memory != NULL);
-  }
-
-  return memory;
-}
-
-/*******************************************************************************
- * \brief Private routine for actually freeing system memory.
- * \author Ole Schuett
- ******************************************************************************/
-static void actual_free(const void *memory, const bool on_device) {
-  if (NULL != memory) {
-    void *mem = (void *)(uintptr_t)memory;
-#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_DBM)
-    if (on_device) {
-      offload_activate_chosen_device();
-      offloadFree(mem);
-    } else {
-#if (0 != DBM_OFFLOAD_ALLOC)
-      offloadFreeHost(mem);
-#else
-      dbm_mpi_free_mem(mem);
-#endif
-    }
-#else
-    (void)on_device; // mark used
-    dbm_mpi_free_mem(mem);
-#endif
-  }
-}
-
 /*******************************************************************************
  * \brief Private struct for storing a chunk of memory.
  * \author Ole Schuett
@@ -97,6 +43,75 @@ static dbm_memchunk_t *mempool_host_available_head = NULL;
 static dbm_memchunk_t *mempool_device_allocated_head = NULL;
 static dbm_memchunk_t *mempool_host_allocated_head = NULL;
 
+/*******************************************************************************
+ * \brief Private statistics.
+ * \author Hans Pabst
+ ******************************************************************************/
+static dbm_memstats_t mempool_stats = {0};
+
+/*******************************************************************************
+ * \brief Private routine for actually allocating system memory.
+ * \author Ole Schuett
+ ******************************************************************************/
+static void *actual_malloc(size_t size, bool on_device) {
+  void *memory = NULL;
+
+  if (0 != size) {
+#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_DBM)
+    if (on_device) {
+      offload_activate_chosen_device();
+      offloadMalloc(&memory, size);
+    } else {
+#if (0 != DBM_OFFLOAD_ALLOC)
+      offloadMallocHost(&memory, size);
+#else
+      memory = dbm_mpi_alloc_mem(size);
+#endif
+    }
+#else
+    (void)on_device; // mark used
+    memory = dbm_mpi_alloc_mem(size);
+#endif
+    assert(memory != NULL);
+
+    // Update statistics.
+    if (on_device) {
+#pragma omp atomic
+      ++mempool_stats.device_mallocs;
+    } else {
+#pragma omp atomic
+      ++mempool_stats.host_mallocs;
+    }
+  }
+
+  return memory;
+}
+
+/*******************************************************************************
+ * \brief Private routine for actually freeing system memory.
+ * \author Ole Schuett
+ ******************************************************************************/
+static void actual_free(const void *memory, bool on_device) {
+  if (NULL != memory) {
+    void *mem = (void *)(uintptr_t)memory;
+#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_DBM)
+    if (on_device) {
+      offload_activate_chosen_device();
+      offloadFree(mem);
+    } else {
+#if (0 != DBM_OFFLOAD_ALLOC)
+      offloadFreeHost(mem);
+#else
+      dbm_mpi_free_mem(mem);
+#endif
+    }
+#else
+    (void)on_device; // mark used
+    dbm_mpi_free_mem(mem);
+#endif
+  }
+}
+
 /*******************************************************************************
  * \brief Private routine for allocating host or device memory from the pool.
  * \author Ole Schuett
@@ -104,7 +119,7 @@ static dbm_memchunk_t *mempool_host_allocated_head = NULL;
 #if (0 != DBM_MEMPOOL_DEVICE) || (0 != DBM_MEMPOOL_HOST)
 static void *internal_mempool_malloc(dbm_memchunk_t **available_head,
                                      dbm_memchunk_t **allocated_head,
-                                     const size_t size) {
+                                     size_t size) {
   if (size == 0) {
     return NULL;
   }
@@ -119,23 +134,25 @@ static void *internal_mempool_malloc(dbm_memchunk_t **available_head,
     // Find a suitable chunk in mempool_available.
     dbm_memchunk_t **reuse = NULL, **reclaim = NULL;
     for (; NULL != *available_head; available_head = &(*available_head)->next) {
-      const size_t cur_size = (*available_head)->size;
-      if (cur_size < size) { // needs reallocation
-        if (NULL == reclaim || (*reclaim)->size < cur_size) {
-          reclaim = available_head;
-        }
-      } else if (NULL == reuse || cur_size < (*reuse)->size) {
+      const size_t s = (*available_head)->size;
+      if (size <= s && (NULL == reuse || s < (*reuse)->size)) {
         reuse = available_head;
         if (size == (*reuse)->size) {
           break; // exact match
         }
+      } else if (NULL != reclaim) {
+        if (s > (*reclaim)->size) {
+          reclaim = available_head;
+        }
+      } else {
+        reclaim = available_head;
       }
     }
     if (NULL == reuse) {
       reuse = reclaim;
     }
 
-    // If a chunck was found, remove it from mempool_available.
+    // Remove chunk from mempool_available.
     if (NULL != reuse) {
       chunk = *reuse;
       *reuse = chunk->next;
@@ -149,11 +166,13 @@ static void *internal_mempool_malloc(dbm_memchunk_t **available_head,
     *allocated_head = chunk;
   }
 
-  // Resize chunk if needed (not in critical section)
+  // Resize chunk (not in critical section).
   if (chunk->size < size) {
-    actual_free(chunk->mem, on_device);
+    void *memory = chunk->mem;
+    chunk->mem = NULL; // race ok (free and stats)
+    actual_free(memory, on_device);
     chunk->mem = actual_malloc(size, on_device);
-    chunk->size = size; // update
+    chunk->size = size;
   }
   chunk->used = size; // stats
 
@@ -165,7 +184,7 @@ static void *internal_mempool_malloc(dbm_memchunk_t **available_head,
  * \brief Internal routine for allocating host memory from the pool.
  * \author Ole Schuett
  ******************************************************************************/
-void *dbm_mempool_host_malloc(const size_t size) {
+void *dbm_mempool_host_malloc(size_t size) {
 #if (0 != DBM_MEMPOOL_HOST)
   return internal_mempool_malloc(&mempool_host_available_head,
                                  &mempool_host_allocated_head, size);
@@ -178,7 +197,7 @@ void *dbm_mempool_host_malloc(const size_t size) {
  * \brief Internal routine for allocating device memory from the pool
  * \author Ole Schuett
  ******************************************************************************/
-void *dbm_mempool_device_malloc(const size_t size) {
+void *dbm_mempool_device_malloc(size_t size) {
 #if (0 != DBM_MEMPOOL_DEVICE)
 #if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_DBM)
   return internal_mempool_malloc(&mempool_device_available_head,
@@ -250,29 +269,36 @@ void dbm_mempool_device_free(const void *memory) {
 #endif
 }
 
+/*******************************************************************************
+ * \brief Private routine for freeing all memory in the pool.
+ * \author Ole Schuett
+ ******************************************************************************/
+static void internal_mempool_clear(dbm_memchunk_t **available_head) {
+  const bool on_device = (&mempool_device_available_head == available_head);
+  assert(on_device || &mempool_host_available_head == available_head);
+
+  // Free chunks in mempool_available.
+  while (NULL != *available_head) {
+    dbm_memchunk_t *chunk = *available_head;
+    *available_head = chunk->next; // remove chunk
+    actual_free(chunk->mem, on_device);
+    free(chunk);
+  }
+}
+
 /*******************************************************************************
  * \brief Internal routine for freeing all memory in the pool.
  * \author Ole Schuett
  ******************************************************************************/
 void dbm_mempool_clear(void) {
-  assert(omp_get_num_threads() == 1);
-
   // check for memory leak
   assert(mempool_device_allocated_head == NULL);
   assert(mempool_host_allocated_head == NULL);
 
-  // Free chunks in mempool_available.
-  while (mempool_device_available_head != NULL) {
-    dbm_memchunk_t *chunk = mempool_device_available_head;
-    mempool_device_available_head = chunk->next;
-    actual_free(chunk->mem, true);
-    free(chunk);
-  }
-  while (mempool_host_available_head != NULL) {
-    dbm_memchunk_t *chunk = mempool_host_available_head;
-    mempool_host_available_head = chunk->next;
-    actual_free(chunk->mem, false);
-    free(chunk);
+#pragma omp critical(dbm_mempool_modify)
+  {
+    internal_mempool_clear(&mempool_device_available_head);
+    internal_mempool_clear(&mempool_host_available_head);
   }
 }
 
@@ -281,35 +307,44 @@ void dbm_mempool_clear(void) {
  * \author Hans Pabst
  ******************************************************************************/
 void dbm_mempool_statistics(dbm_memstats_t *memstats) {
-  dbm_memchunk_t *chunk;
   assert(NULL != memstats);
-  memset(memstats, 0, sizeof(*memstats));
+  memset(memstats, 0, sizeof(dbm_memstats_t));
 #pragma omp critical(dbm_mempool_modify)
   {
+    dbm_memchunk_t *chunk;
     for (chunk = mempool_device_available_head; NULL != chunk;
          chunk = chunk->next) {
       memstats->device_used += chunk->used;
       memstats->device_size += chunk->size;
-      ++memstats->device_mallocs;
     }
     for (chunk = mempool_device_allocated_head; NULL != chunk;
          chunk = chunk->next) {
       memstats->device_used += chunk->used;
       memstats->device_size += chunk->size;
-      ++memstats->device_mallocs;
     }
     for (chunk = mempool_host_available_head; NULL != chunk;
          chunk = chunk->next) {
       memstats->host_used += chunk->used;
       memstats->host_size += chunk->size;
-      ++memstats->host_mallocs;
     }
     for (chunk = mempool_host_allocated_head; NULL != chunk;
          chunk = chunk->next) {
       memstats->host_used += chunk->used;
       memstats->host_size += chunk->size;
-      ++memstats->host_mallocs;
     }
+    if (mempool_stats.device_used < memstats->device_used) {
+      mempool_stats.device_used = memstats->device_used;
+    }
+    if (mempool_stats.device_size < memstats->device_size) {
+      mempool_stats.device_size = memstats->device_size;
+    }
+    if (mempool_stats.host_used < memstats->host_used) {
+      mempool_stats.host_used = memstats->host_used;
+    }
+    if (mempool_stats.host_size < memstats->host_size) {
+      mempool_stats.host_size = memstats->host_size;
+    }
+    memcpy(memstats, &mempool_stats, sizeof(dbm_memstats_t));
   }
 }
 

src/dbm/dbm_mempool.h

@@ -17,13 +17,13 @@ extern "C" {
  * \brief Internal routine for allocating host memory from the pool.
  * \author Ole Schuett
  ******************************************************************************/
-void *dbm_mempool_host_malloc(const size_t size);
+void *dbm_mempool_host_malloc(size_t size);
 
 /*******************************************************************************
  * \brief Internal routine for allocating device memory from the pool.
  * \author Ole Schuett
  ******************************************************************************/
-void *dbm_mempool_device_malloc(const size_t size);
+void *dbm_mempool_device_malloc(size_t size);
 
 /*******************************************************************************
  * \brief Internal routine for releasing memory back to the pool.

src/dbm/dbm_mpi.c

@@ -479,7 +479,7 @@ void dbm_mpi_alltoallv_double(const double *sendbuf, const int *sendcounts,
  ******************************************************************************/
 void *dbm_mpi_alloc_mem(size_t size) {
   void *result = NULL;
-#if defined(__parallel) && !defined(OPEN_MPI)
+#if defined(__parallel)
   CHECK(MPI_Alloc_mem((MPI_Aint)size, MPI_INFO_NULL, &result));
 #else
   result = malloc(size);
@@ -492,7 +492,7 @@ void *dbm_mpi_alloc_mem(size_t size) {
  * \author Hans Pabst
  ******************************************************************************/
 void dbm_mpi_free_mem(void *mem) {
-#if defined(__parallel) && !defined(OPEN_MPI)
+#if defined(__parallel)
   CHECK(MPI_Free_mem(mem));
 #else
   free(mem);

src/dbm/dbm_multiply_opencl.c

@@ -169,10 +169,8 @@ void dbm_multiply_gpu_launch_kernel(const offloadStream_t stream, double alpha,
           fprintf(stderr, "INFO ACC/LIBDBM: DBM-kernel failed to generate\n");
         }
       }
-      kernel = kernel_global;
-    } else {
-      kernel = clCloneKernel(kernel_global, &result);
     }
+    kernel = clCloneKernel(kernel_global, &result); /* always clone */
     ACC_OPENCL_RELEASE(config->lock_main);
   } else if (NULL == kernel) {
     kernel = clCloneKernel(kernel_global, &result);