From 5091418d717fc18e2b5e716c9d62ac614d838459 Mon Sep 17 00:00:00 2001 From: YoungHui Park Date: Wed, 29 Jul 2020 11:33:27 -0500 Subject: [PATCH] Add a new variable to store maximum bank size --- include/openmc/simulation.h | 1 + src/simulation.cpp | 5 +++++ src/state_point.cpp | 21 ++++++++------------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/include/openmc/simulation.h b/include/openmc/simulation.h index 8c4f044ce3..9422590cf8 100644 --- a/include/openmc/simulation.h +++ b/include/openmc/simulation.h @@ -46,6 +46,7 @@ extern std::vector k_generation; extern std::vector work_index; extern int64_t total_surf_banks; //!< Total number of surface source banks +extern int64_t max_bank_size; //!< Maximum bank size for MPI extern std::vector surf_src_index; } // namespace simulation diff --git a/src/simulation.cpp b/src/simulation.cpp index a58dcb535b..f56a543bca 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -271,6 +271,7 @@ std::vector k_generation; std::vector work_index; int64_t total_surf_banks {0}; +int64_t max_bank_size {0}; std::vector surf_src_index; } // namespace simulation @@ -557,6 +558,9 @@ void calculate_work() // Number of particles for rank i int64_t work_i = i < remainder ? min_work + 1 : min_work; + // Set maximum bank size + if (mpi::master) simulation::max_bank_size = work_i; + // Set number of particles if (mpi::rank == i) simulation::work_per_rank = work_i; @@ -656,6 +660,7 @@ void query_surf_src_size() for (int i = 1; i < mpi::n_procs + 1; ++i) { simulation::surf_src_index[i] = simulation::surf_src_index[i - 1] + bank_size[i - 1]; } + simulation::max_bank_size = *std::max_element(bank_size.begin(), bank_size.end()); total = simulation::surf_src_index[mpi::n_procs]; } #else diff --git a/src/state_point.cpp b/src/state_point.cpp index 14a61d81da..62058423e6 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -580,8 +580,8 @@ write_source_bank(hid_t group_id, bool surf_src_bank) src_bank.clear(); src_bank.assign(simulation::surf_src_bank.data(), - simulation::surf_src_bank.data() - + simulation::surf_src_bank.size()); + simulation::surf_src_bank.data() + + simulation::surf_src_bank.size()); } #ifdef PHDF5 @@ -621,11 +621,10 @@ write_source_bank(hid_t group_id, bool surf_src_bank) hid_t dset = H5Dcreate(group_id, "source_bank", banktype, dspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - // Save source bank sites since the souce_bank array is overwritten below -#ifdef OPENMC_MPI - std::vector temp_source {src_bank.begin(), - src_bank.begin() + count_size}; -#endif + // Set new bank sites to avoid the src_bank being overwritten on MPI_Recv + std::vector src_to_save; + src_to_save.reserve(simulation::max_bank_size); + std::copy(src_bank.begin(), src_bank.end(), src_to_save.begin()); for (int i = 0; i < mpi::n_procs; ++i) { // Create memory space @@ -635,7 +634,7 @@ write_source_bank(hid_t group_id, bool surf_src_bank) #ifdef OPENMC_MPI // Receive source sites from other processes if (i > 0) - MPI_Recv(src_bank.data(), count[0], mpi::bank, i, i, + MPI_Recv(src_to_save.data(), count[0], mpi::bank, i, i, mpi::intracomm, MPI_STATUS_IGNORE); #endif @@ -645,7 +644,7 @@ write_source_bank(hid_t group_id, bool surf_src_bank) H5Sselect_hyperslab(dspace, H5S_SELECT_SET, start, nullptr, count, nullptr); // Write data to hyperslab - H5Dwrite(dset, banktype, memspace, dspace, H5P_DEFAULT, src_bank.data()); + H5Dwrite(dset, banktype, memspace, dspace, H5P_DEFAULT, src_to_save.data()); H5Sclose(memspace); H5Sclose(dspace); @@ -654,10 +653,6 @@ write_source_bank(hid_t group_id, bool surf_src_bank) // Close all ids H5Dclose(dset); -#ifdef OPENMC_MPI - // Restore state of source bank - std::copy(temp_source.begin(), temp_source.end(), src_bank.begin()); -#endif } else { #ifdef OPENMC_MPI MPI_Send(src_bank.data(), count_size, mpi::bank,