diff --git a/include/openmc/bank.h b/include/openmc/bank.h index b3c058ef2..dea32c655 100644 --- a/include/openmc/bank.h +++ b/include/openmc/bank.h @@ -16,11 +16,10 @@ namespace openmc { namespace simulation { extern std::vector source_bank; -extern std::vector fission_bank; -extern Particle::Bank* shared_fission_bank; -extern int shared_fission_bank_length; -extern int shared_fission_bank_max; +extern Particle::Bank* fission_bank; +extern int64_t fission_bank_length; +extern int64_t fission_bank_max; } // namespace simulation @@ -30,7 +29,7 @@ extern int shared_fission_bank_max; void free_memory_bank(); -void init_shared_fission_bank(int max); +void init_fission_bank(int64_t max); } // namespace openmc diff --git a/include/openmc/mesh.h b/include/openmc/mesh.h index 34782b5e3..974464d4e 100644 --- a/include/openmc/mesh.h +++ b/include/openmc/mesh.h @@ -161,7 +161,7 @@ public: //! \param[in] bank Array of bank sites //! \param[out] Whether any bank sites are outside the mesh //! \return Array indicating number of sites in each mesh/energy bin - xt::xtensor count_sites(const std::vector& bank, + xt::xtensor count_sites(const Particle::Bank* bank, uint64_t length, bool* outside) const; // Data members diff --git a/src/bank.cpp b/src/bank.cpp index 0576ea2ec..0d8cde5f1 100644 --- a/src/bank.cpp +++ b/src/bank.cpp @@ -6,7 +6,7 @@ #include // Explicit template instantiation definition -template class std::vector; +//template class std::vector; namespace openmc { @@ -17,11 +17,10 @@ namespace openmc { namespace simulation { std::vector source_bank; -std::vector fission_bank; -Particle::Bank* shared_fission_bank {nullptr}; -int shared_fission_bank_length {0}; -int shared_fission_bank_max; +Particle::Bank* fission_bank {nullptr}; +int64_t fission_bank_length {0}; +int64_t fission_bank_max; } // namespace simulation @@ -32,18 +31,17 @@ int shared_fission_bank_max; void free_memory_bank() { simulation::source_bank.clear(); - simulation::fission_bank.clear(); - if( simulation::shared_fission_bank != nullptr ) - delete[] simulation::shared_fission_bank; - simulation::shared_fission_bank = nullptr; - simulation::shared_fission_bank_length = 0; + if( simulation::fission_bank != nullptr ) + delete[] simulation::fission_bank; + simulation::fission_bank = nullptr; + simulation::fission_bank_length = 0; } -void init_shared_fission_bank(int max) +void init_fission_bank(int64_t max) { - simulation::shared_fission_bank_max = max; - simulation::shared_fission_bank = new Particle::Bank[max]; - simulation::shared_fission_bank_length = 0; + simulation::fission_bank_max = max; + simulation::fission_bank = new Particle::Bank[max]; + simulation::fission_bank_length = 0; } //============================================================================== @@ -64,12 +62,12 @@ extern "C" int openmc_source_bank(void** ptr, int64_t* n) extern "C" int openmc_fission_bank(void** ptr, int64_t* n) { - if (simulation::fission_bank.size() == 0) { + if (simulation::fission_bank_length == 0) { set_errmsg("Fission bank has not been allocated."); return OPENMC_E_ALLOCATE; } else { - *ptr = simulation::fission_bank.data(); - *n = simulation::fission_bank.size(); + *ptr = simulation::fission_bank; + *n = simulation::fission_bank_length; return 0; } } diff --git a/src/eigenvalue.cpp b/src/eigenvalue.cpp index d958b33b8..f61ca813a 100644 --- a/src/eigenvalue.cpp +++ b/src/eigenvalue.cpp @@ -83,7 +83,7 @@ void synchronize_bank() #ifdef OPENMC_MPI int64_t start = 0; - int64_t n_bank = simulation::fission_bank.size(); + int64_t n_bank = simulation::fission_bank_length; MPI_Exscan(&n_bank, &start, 1, MPI_INT64_T, MPI_SUM, mpi::intracomm); // While we would expect the value of start on rank 0 to be 0, the MPI @@ -91,14 +91,14 @@ void synchronize_bank() // significant if (mpi::rank == 0) start = 0; - int64_t finish = start + simulation::fission_bank.size(); + int64_t finish = start + simulation::fission_bank_length; int64_t total = finish; MPI_Bcast(&total, 1, MPI_INT64_T, mpi::n_procs - 1, mpi::intracomm); #else int64_t start = 0; - int64_t finish = simulation::fission_bank.size(); - int64_t total = simulation::fission_bank.size(); + int64_t finish = simulation::fission_bank_length; + int64_t total = finish; #endif // If there are not that many particles per generation, it's possible that no @@ -106,7 +106,7 @@ void synchronize_bank() // extra logic to treat this circumstance, we really want to ensure the user // runs enough particles to avoid this in the first place. - if (simulation::fission_bank.empty()) { + if (simulation::fission_bank_length == 0) { fatal_error("No fission sites banked on MPI rank " + std::to_string(mpi::rank)); } @@ -138,7 +138,9 @@ void synchronize_bank() int64_t index_temp = 0; std::vector temp_sites(3*simulation::work_per_rank); - for (const auto& site : simulation::fission_bank) { + for (uint64_t i = 0; i < simulation::fission_bank_length; i++ ) { + const auto& site = simulation::fission_bank[i]; + // If there are less than n_particles particles banked, automatically add // int(n_particles/total) sites to temp_sites. For example, if you need // 1000 and 300 were banked, this would add 3 source sites per banked site @@ -193,7 +195,7 @@ void synchronize_bank() // fission bank sites_needed = settings::n_particles - finish; for (int i = 0; i < sites_needed; ++i) { - int i_bank = simulation::fission_bank.size() - sites_needed + i; + int i_bank = simulation::fission_bank_length - sites_needed + i; temp_sites[index_temp] = simulation::fission_bank[i_bank]; ++index_temp; } @@ -504,7 +506,8 @@ void shannon_entropy() // Get source weight in each mesh bin bool sites_outside; xt::xtensor p = simulation::entropy_mesh->count_sites( - simulation::fission_bank, &sites_outside); + simulation::fission_bank, simulation::fission_bank_length, + &sites_outside); // display warning message if there were sites outside entropy box if (sites_outside) { @@ -543,7 +546,7 @@ void ufs_count_sites() // count number of source sites in each ufs mesh cell bool sites_outside; simulation::source_frac = simulation::ufs_mesh->count_sites( - simulation::source_bank, &sites_outside); + simulation::source_bank.data(), simulation::source_bank.size(), &sites_outside); // Check for sites outside of the mesh if (mpi::master && sites_outside) { diff --git a/src/mesh.cpp b/src/mesh.cpp index f679c402a..490596cd1 100644 --- a/src/mesh.cpp +++ b/src/mesh.cpp @@ -790,7 +790,7 @@ void RegularMesh::to_hdf5(hid_t group) const } xt::xtensor -RegularMesh::count_sites(const std::vector& bank, +RegularMesh::count_sites(const Particle::Bank* bank, uint64_t length, bool* outside) const { // Determine shape of array for counts @@ -801,7 +801,9 @@ RegularMesh::count_sites(const std::vector& bank, xt::xarray cnt {shape, 0.0}; bool outside_ = false; - for (const auto& site : bank) { + for (uint64_t i = 0; i < length; i++) { + const auto& site = bank[i]; + // determine scoring bin for entropy mesh int mesh_bin = get_bin(site.r); diff --git a/src/physics.cpp b/src/physics.cpp index 1d509fd46..436533bb8 100644 --- a/src/physics.cpp +++ b/src/physics.cpp @@ -186,17 +186,17 @@ create_fission_sites(Particle* p, int i_nuclide, const Reaction* rx) { int idx; #pragma omp atomic capture - idx = simulation::shared_fission_bank_length++; - if( idx >= simulation::shared_fission_bank_max ) + idx = simulation::fission_bank_length++; + if( idx >= simulation::fission_bank_max ) { warning("The shared fission bank is full. Additional fission sites created " "in this generation will not be banked."); #pragma omp atomic write - simulation::shared_fission_bank_length = simulation::shared_fission_bank_max; + simulation::fission_bank_length = simulation::fission_bank_max; skipped++; break; } - site = simulation::shared_fission_bank + idx; + site = simulation::fission_bank + idx; } else { diff --git a/src/physics_mg.cpp b/src/physics_mg.cpp index 18bbbe6df..d14a15781 100644 --- a/src/physics_mg.cpp +++ b/src/physics_mg.cpp @@ -133,17 +133,17 @@ create_fission_sites(Particle* p) { int idx; #pragma omp atomic capture - idx = simulation::shared_fission_bank_length++; - if( idx >= simulation::shared_fission_bank_max ) + idx = simulation::fission_bank_length++; + if( idx >= simulation::fission_bank_max ) { warning("The shared fission bank is full. Additional fission sites created " "in this generation will not be banked."); #pragma omp atomic write - simulation::shared_fission_bank_length = simulation::shared_fission_bank_max; + simulation::fission_bank_length = simulation::fission_bank_max; skipped++; break; } - site = simulation::shared_fission_bank + idx; + site = simulation::fission_bank + idx; } else { diff --git a/src/simulation.cpp b/src/simulation.cpp index 6ac776639..7295f68c0 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -264,11 +264,9 @@ void allocate_banks() simulation::source_bank.resize(simulation::work_per_rank); if (settings::run_mode == RUN_MODE_EIGENVALUE) { - simulation::fission_bank.reserve(3*simulation::work_per_rank); + init_fission_bank(3*simulation::work_per_rank); } - // Allocate shared bank - init_shared_fission_bank(simulation::work_per_rank * 3); } void initialize_batch() @@ -369,7 +367,7 @@ void initialize_generation() { if (settings::run_mode == RUN_MODE_EIGENVALUE) { // Clear out the fission bank - simulation::fission_bank.clear(); + simulation::fission_bank_length = 0; // Count source sites if using uniform fission source weighting if (settings::ufs_on) ufs_count_sites(); @@ -401,13 +399,11 @@ void finalize_generation() global_tally_leakage = 0.0; if (settings::run_mode == RUN_MODE_EIGENVALUE) { - // Copy shared fission bank into regular bank for use in MPI synchronization - for( int i = 0; i < simulation::shared_fission_bank_length; i++ ) - simulation::fission_bank.push_back(simulation::shared_fission_bank[i]); - simulation::shared_fission_bank_length = 0; - - // Sorts the fission bank so as to allow for reproducibility - std::stable_sort(simulation::fission_bank.begin(), simulation::fission_bank.end()); + // If using shared memory, sort the fission bank so as to allow for reproducibility + #ifdef _OPENMP + std::stable_sort(simulation::fission_bank, + simulation::fission_bank + simulation::fission_bank_length); + #endif // Distribute fission bank across processors evenly synchronize_bank(); @@ -430,6 +426,7 @@ void finalize_generation() // For fixed-source mode, we need to sample the external source fill_source_bank_fixedsource(); } + } void initialize_history(Particle* p, int64_t index_source)