From f6020279c997d28b7da986b9d210b68b87c09557 Mon Sep 17 00:00:00 2001 From: John Tramm Date: Tue, 14 Jan 2020 15:39:20 +0000 Subject: [PATCH] Refactored to remove the non-shared fission bank. This involved some minor changes to the MPI bank synchronization code and the site counting code, but overall makes the code much simpler. Since there is now only one fission bank, the name was changed from shared_fission_bank to just fission_bank throughout. --- include/openmc/bank.h | 9 ++++----- include/openmc/mesh.h | 2 +- src/bank.cpp | 32 +++++++++++++++----------------- src/eigenvalue.cpp | 21 ++++++++++++--------- src/mesh.cpp | 6 ++++-- src/physics.cpp | 8 ++++---- src/physics_mg.cpp | 8 ++++---- src/simulation.cpp | 19 ++++++++----------- 8 files changed, 52 insertions(+), 53 deletions(-) diff --git a/include/openmc/bank.h b/include/openmc/bank.h index b3c058ef2..dea32c655 100644 --- a/include/openmc/bank.h +++ b/include/openmc/bank.h @@ -16,11 +16,10 @@ namespace openmc { namespace simulation { extern std::vector source_bank; -extern std::vector fission_bank; -extern Particle::Bank* shared_fission_bank; -extern int shared_fission_bank_length; -extern int shared_fission_bank_max; +extern Particle::Bank* fission_bank; +extern int64_t fission_bank_length; +extern int64_t fission_bank_max; } // namespace simulation @@ -30,7 +29,7 @@ extern int shared_fission_bank_max; void free_memory_bank(); -void init_shared_fission_bank(int max); +void init_fission_bank(int64_t max); } // namespace openmc diff --git a/include/openmc/mesh.h b/include/openmc/mesh.h index 34782b5e3..974464d4e 100644 --- a/include/openmc/mesh.h +++ b/include/openmc/mesh.h @@ -161,7 +161,7 @@ public: //! \param[in] bank Array of bank sites //! \param[out] Whether any bank sites are outside the mesh //! \return Array indicating number of sites in each mesh/energy bin - xt::xtensor count_sites(const std::vector& bank, + xt::xtensor count_sites(const Particle::Bank* bank, uint64_t length, bool* outside) const; // Data members diff --git a/src/bank.cpp b/src/bank.cpp index 0576ea2ec..0d8cde5f1 100644 --- a/src/bank.cpp +++ b/src/bank.cpp @@ -6,7 +6,7 @@ #include // Explicit template instantiation definition -template class std::vector; +//template class std::vector; namespace openmc { @@ -17,11 +17,10 @@ namespace openmc { namespace simulation { std::vector source_bank; -std::vector fission_bank; -Particle::Bank* shared_fission_bank {nullptr}; -int shared_fission_bank_length {0}; -int shared_fission_bank_max; +Particle::Bank* fission_bank {nullptr}; +int64_t fission_bank_length {0}; +int64_t fission_bank_max; } // namespace simulation @@ -32,18 +31,17 @@ int shared_fission_bank_max; void free_memory_bank() { simulation::source_bank.clear(); - simulation::fission_bank.clear(); - if( simulation::shared_fission_bank != nullptr ) - delete[] simulation::shared_fission_bank; - simulation::shared_fission_bank = nullptr; - simulation::shared_fission_bank_length = 0; + if( simulation::fission_bank != nullptr ) + delete[] simulation::fission_bank; + simulation::fission_bank = nullptr; + simulation::fission_bank_length = 0; } -void init_shared_fission_bank(int max) +void init_fission_bank(int64_t max) { - simulation::shared_fission_bank_max = max; - simulation::shared_fission_bank = new Particle::Bank[max]; - simulation::shared_fission_bank_length = 0; + simulation::fission_bank_max = max; + simulation::fission_bank = new Particle::Bank[max]; + simulation::fission_bank_length = 0; } //============================================================================== @@ -64,12 +62,12 @@ extern "C" int openmc_source_bank(void** ptr, int64_t* n) extern "C" int openmc_fission_bank(void** ptr, int64_t* n) { - if (simulation::fission_bank.size() == 0) { + if (simulation::fission_bank_length == 0) { set_errmsg("Fission bank has not been allocated."); return OPENMC_E_ALLOCATE; } else { - *ptr = simulation::fission_bank.data(); - *n = simulation::fission_bank.size(); + *ptr = simulation::fission_bank; + *n = simulation::fission_bank_length; return 0; } } diff --git a/src/eigenvalue.cpp b/src/eigenvalue.cpp index d958b33b8..f61ca813a 100644 --- a/src/eigenvalue.cpp +++ b/src/eigenvalue.cpp @@ -83,7 +83,7 @@ void synchronize_bank() #ifdef OPENMC_MPI int64_t start = 0; - int64_t n_bank = simulation::fission_bank.size(); + int64_t n_bank = simulation::fission_bank_length; MPI_Exscan(&n_bank, &start, 1, MPI_INT64_T, MPI_SUM, mpi::intracomm); // While we would expect the value of start on rank 0 to be 0, the MPI @@ -91,14 +91,14 @@ void synchronize_bank() // significant if (mpi::rank == 0) start = 0; - int64_t finish = start + simulation::fission_bank.size(); + int64_t finish = start + simulation::fission_bank_length; int64_t total = finish; MPI_Bcast(&total, 1, MPI_INT64_T, mpi::n_procs - 1, mpi::intracomm); #else int64_t start = 0; - int64_t finish = simulation::fission_bank.size(); - int64_t total = simulation::fission_bank.size(); + int64_t finish = simulation::fission_bank_length; + int64_t total = finish; #endif // If there are not that many particles per generation, it's possible that no @@ -106,7 +106,7 @@ void synchronize_bank() // extra logic to treat this circumstance, we really want to ensure the user // runs enough particles to avoid this in the first place. - if (simulation::fission_bank.empty()) { + if (simulation::fission_bank_length == 0) { fatal_error("No fission sites banked on MPI rank " + std::to_string(mpi::rank)); } @@ -138,7 +138,9 @@ void synchronize_bank() int64_t index_temp = 0; std::vector temp_sites(3*simulation::work_per_rank); - for (const auto& site : simulation::fission_bank) { + for (uint64_t i = 0; i < simulation::fission_bank_length; i++ ) { + const auto& site = simulation::fission_bank[i]; + // If there are less than n_particles particles banked, automatically add // int(n_particles/total) sites to temp_sites. For example, if you need // 1000 and 300 were banked, this would add 3 source sites per banked site @@ -193,7 +195,7 @@ void synchronize_bank() // fission bank sites_needed = settings::n_particles - finish; for (int i = 0; i < sites_needed; ++i) { - int i_bank = simulation::fission_bank.size() - sites_needed + i; + int i_bank = simulation::fission_bank_length - sites_needed + i; temp_sites[index_temp] = simulation::fission_bank[i_bank]; ++index_temp; } @@ -504,7 +506,8 @@ void shannon_entropy() // Get source weight in each mesh bin bool sites_outside; xt::xtensor p = simulation::entropy_mesh->count_sites( - simulation::fission_bank, &sites_outside); + simulation::fission_bank, simulation::fission_bank_length, + &sites_outside); // display warning message if there were sites outside entropy box if (sites_outside) { @@ -543,7 +546,7 @@ void ufs_count_sites() // count number of source sites in each ufs mesh cell bool sites_outside; simulation::source_frac = simulation::ufs_mesh->count_sites( - simulation::source_bank, &sites_outside); + simulation::source_bank.data(), simulation::source_bank.size(), &sites_outside); // Check for sites outside of the mesh if (mpi::master && sites_outside) { diff --git a/src/mesh.cpp b/src/mesh.cpp index f679c402a..490596cd1 100644 --- a/src/mesh.cpp +++ b/src/mesh.cpp @@ -790,7 +790,7 @@ void RegularMesh::to_hdf5(hid_t group) const } xt::xtensor -RegularMesh::count_sites(const std::vector& bank, +RegularMesh::count_sites(const Particle::Bank* bank, uint64_t length, bool* outside) const { // Determine shape of array for counts @@ -801,7 +801,9 @@ RegularMesh::count_sites(const std::vector& bank, xt::xarray cnt {shape, 0.0}; bool outside_ = false; - for (const auto& site : bank) { + for (uint64_t i = 0; i < length; i++) { + const auto& site = bank[i]; + // determine scoring bin for entropy mesh int mesh_bin = get_bin(site.r); diff --git a/src/physics.cpp b/src/physics.cpp index 1d509fd46..436533bb8 100644 --- a/src/physics.cpp +++ b/src/physics.cpp @@ -186,17 +186,17 @@ create_fission_sites(Particle* p, int i_nuclide, const Reaction* rx) { int idx; #pragma omp atomic capture - idx = simulation::shared_fission_bank_length++; - if( idx >= simulation::shared_fission_bank_max ) + idx = simulation::fission_bank_length++; + if( idx >= simulation::fission_bank_max ) { warning("The shared fission bank is full. Additional fission sites created " "in this generation will not be banked."); #pragma omp atomic write - simulation::shared_fission_bank_length = simulation::shared_fission_bank_max; + simulation::fission_bank_length = simulation::fission_bank_max; skipped++; break; } - site = simulation::shared_fission_bank + idx; + site = simulation::fission_bank + idx; } else { diff --git a/src/physics_mg.cpp b/src/physics_mg.cpp index 18bbbe6df..d14a15781 100644 --- a/src/physics_mg.cpp +++ b/src/physics_mg.cpp @@ -133,17 +133,17 @@ create_fission_sites(Particle* p) { int idx; #pragma omp atomic capture - idx = simulation::shared_fission_bank_length++; - if( idx >= simulation::shared_fission_bank_max ) + idx = simulation::fission_bank_length++; + if( idx >= simulation::fission_bank_max ) { warning("The shared fission bank is full. Additional fission sites created " "in this generation will not be banked."); #pragma omp atomic write - simulation::shared_fission_bank_length = simulation::shared_fission_bank_max; + simulation::fission_bank_length = simulation::fission_bank_max; skipped++; break; } - site = simulation::shared_fission_bank + idx; + site = simulation::fission_bank + idx; } else { diff --git a/src/simulation.cpp b/src/simulation.cpp index 6ac776639..7295f68c0 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -264,11 +264,9 @@ void allocate_banks() simulation::source_bank.resize(simulation::work_per_rank); if (settings::run_mode == RUN_MODE_EIGENVALUE) { - simulation::fission_bank.reserve(3*simulation::work_per_rank); + init_fission_bank(3*simulation::work_per_rank); } - // Allocate shared bank - init_shared_fission_bank(simulation::work_per_rank * 3); } void initialize_batch() @@ -369,7 +367,7 @@ void initialize_generation() { if (settings::run_mode == RUN_MODE_EIGENVALUE) { // Clear out the fission bank - simulation::fission_bank.clear(); + simulation::fission_bank_length = 0; // Count source sites if using uniform fission source weighting if (settings::ufs_on) ufs_count_sites(); @@ -401,13 +399,11 @@ void finalize_generation() global_tally_leakage = 0.0; if (settings::run_mode == RUN_MODE_EIGENVALUE) { - // Copy shared fission bank into regular bank for use in MPI synchronization - for( int i = 0; i < simulation::shared_fission_bank_length; i++ ) - simulation::fission_bank.push_back(simulation::shared_fission_bank[i]); - simulation::shared_fission_bank_length = 0; - - // Sorts the fission bank so as to allow for reproducibility - std::stable_sort(simulation::fission_bank.begin(), simulation::fission_bank.end()); + // If using shared memory, sort the fission bank so as to allow for reproducibility + #ifdef _OPENMP + std::stable_sort(simulation::fission_bank, + simulation::fission_bank + simulation::fission_bank_length); + #endif // Distribute fission bank across processors evenly synchronize_bank(); @@ -430,6 +426,7 @@ void finalize_generation() // For fixed-source mode, we need to sample the external source fill_source_bank_fixedsource(); } + } void initialize_history(Particle* p, int64_t index_source)