Refactored to remove the non-shared fission bank. This involved some minor changes to the MPI bank synchronization code and the site counting code, but overall makes the code much simpler. Since there is now only one fission bank, the name was changed from shared_fission_bank to just fission_bank throughout.

include/openmc/bank.h

@@ -16,11 +16,10 @@ namespace openmc {
 namespace simulation {
 
 extern std::vector<Particle::Bank> source_bank;
-extern std::vector<Particle::Bank> fission_bank;
 
-extern Particle::Bank* shared_fission_bank;
-extern int shared_fission_bank_length;
-extern int shared_fission_bank_max;
+extern Particle::Bank* fission_bank;
+extern int64_t fission_bank_length;
+extern int64_t fission_bank_max;
 
 } // namespace simulation
 
@@ -30,7 +29,7 @@ extern int shared_fission_bank_max;
 
 void free_memory_bank();
 
-void init_shared_fission_bank(int max);
+void init_fission_bank(int64_t max);
 
 } // namespace openmc
 

include/openmc/mesh.h

@@ -161,7 +161,7 @@ public:
   //! \param[in] bank Array of bank sites
   //! \param[out] Whether any bank sites are outside the mesh
   //! \return Array indicating number of sites in each mesh/energy bin
-  xt::xtensor<double, 1> count_sites(const std::vector<Particle::Bank>& bank,
+  xt::xtensor<double, 1> count_sites(const Particle::Bank* bank, uint64_t length,
     bool* outside) const;
 
   // Data members

src/bank.cpp

@@ -6,7 +6,7 @@
 #include <cstdint>
 
 // Explicit template instantiation definition
-template class std::vector<openmc::Particle::Bank>;
+//template class std::vector<openmc::Particle::Bank>;
 
 namespace openmc {
 
@@ -17,11 +17,10 @@ namespace openmc {
 namespace simulation {
 
 std::vector<Particle::Bank> source_bank;
-std::vector<Particle::Bank> fission_bank;
 
-Particle::Bank* shared_fission_bank {nullptr};
-int shared_fission_bank_length {0};
-int shared_fission_bank_max;
+Particle::Bank* fission_bank {nullptr};
+int64_t fission_bank_length {0};
+int64_t fission_bank_max;
 
 } // namespace simulation
 
@@ -32,18 +31,17 @@ int shared_fission_bank_max;
 void free_memory_bank()
 {
   simulation::source_bank.clear();
-  simulation::fission_bank.clear();
-  if( simulation::shared_fission_bank != nullptr )
-    delete[] simulation::shared_fission_bank;
-  simulation::shared_fission_bank = nullptr;
-  simulation::shared_fission_bank_length = 0;
+  if( simulation::fission_bank != nullptr )
+    delete[] simulation::fission_bank;
+  simulation::fission_bank = nullptr;
+  simulation::fission_bank_length = 0;
 }
 
-void init_shared_fission_bank(int max)
+void init_fission_bank(int64_t max)
 {
-  simulation::shared_fission_bank_max = max;
-  simulation::shared_fission_bank = new Particle::Bank[max];
-  simulation::shared_fission_bank_length = 0;
+  simulation::fission_bank_max = max;
+  simulation::fission_bank = new Particle::Bank[max];
+  simulation::fission_bank_length = 0;
 }
 
 //==============================================================================
@@ -64,12 +62,12 @@ extern "C" int openmc_source_bank(void** ptr, int64_t* n)
 
 extern "C" int openmc_fission_bank(void** ptr, int64_t* n)
 {
-  if (simulation::fission_bank.size() == 0) {
+  if (simulation::fission_bank_length == 0) {
     set_errmsg("Fission bank has not been allocated.");
     return OPENMC_E_ALLOCATE;
   } else {
-    *ptr = simulation::fission_bank.data();
-    *n = simulation::fission_bank.size();
+    *ptr = simulation::fission_bank;
+    *n =   simulation::fission_bank_length;
     return 0;
   }
 }

src/eigenvalue.cpp

@@ -83,7 +83,7 @@ void synchronize_bank()
 
 #ifdef OPENMC_MPI
   int64_t start = 0;
-  int64_t n_bank = simulation::fission_bank.size();
+  int64_t n_bank = simulation::fission_bank_length;
   MPI_Exscan(&n_bank, &start, 1, MPI_INT64_T, MPI_SUM, mpi::intracomm);
 
   // While we would expect the value of start on rank 0 to be 0, the MPI
@@ -91,14 +91,14 @@ void synchronize_bank()
   // significant
   if (mpi::rank == 0) start = 0;
 
-  int64_t finish = start + simulation::fission_bank.size();
+  int64_t finish = start + simulation::fission_bank_length;
   int64_t total = finish;
   MPI_Bcast(&total, 1, MPI_INT64_T, mpi::n_procs - 1, mpi::intracomm);
 
 #else
   int64_t start  = 0;
-  int64_t finish = simulation::fission_bank.size();
-  int64_t total  = simulation::fission_bank.size();
+  int64_t finish = simulation::fission_bank_length;
+  int64_t total  = finish;
 #endif
 
   // If there are not that many particles per generation, it's possible that no
@@ -106,7 +106,7 @@ void synchronize_bank()
   // extra logic to treat this circumstance, we really want to ensure the user
   // runs enough particles to avoid this in the first place.
 
-  if (simulation::fission_bank.empty()) {
+  if (simulation::fission_bank_length == 0) {
     fatal_error("No fission sites banked on MPI rank " + std::to_string(mpi::rank));
   }
 
@@ -138,7 +138,9 @@ void synchronize_bank()
   int64_t index_temp = 0;
   std::vector<Particle::Bank> temp_sites(3*simulation::work_per_rank);
 
-  for (const auto& site : simulation::fission_bank) {
+  for (uint64_t i = 0; i < simulation::fission_bank_length; i++ ) {
+    const auto& site = simulation::fission_bank[i];
+
     // If there are less than n_particles particles banked, automatically add
     // int(n_particles/total) sites to temp_sites. For example, if you need
     // 1000 and 300 were banked, this would add 3 source sites per banked site
@@ -193,7 +195,7 @@ void synchronize_bank()
       // fission bank
       sites_needed = settings::n_particles - finish;
       for (int i = 0; i < sites_needed; ++i) {
-        int i_bank = simulation::fission_bank.size() - sites_needed + i;
+        int i_bank = simulation::fission_bank_length - sites_needed + i;
         temp_sites[index_temp] = simulation::fission_bank[i_bank];
         ++index_temp;
       }
@@ -504,7 +506,8 @@ void shannon_entropy()
   // Get source weight in each mesh bin
   bool sites_outside;
   xt::xtensor<double, 1> p = simulation::entropy_mesh->count_sites(
-    simulation::fission_bank, &sites_outside);
+    simulation::fission_bank, simulation::fission_bank_length,
+    &sites_outside);
 
   // display warning message if there were sites outside entropy box
   if (sites_outside) {
@@ -543,7 +546,7 @@ void ufs_count_sites()
     // count number of source sites in each ufs mesh cell
     bool sites_outside;
     simulation::source_frac = simulation::ufs_mesh->count_sites(
-      simulation::source_bank, &sites_outside);
+      simulation::source_bank.data(), simulation::source_bank.size(), &sites_outside);
 
     // Check for sites outside of the mesh
     if (mpi::master && sites_outside) {

src/mesh.cpp

@@ -790,7 +790,7 @@ void RegularMesh::to_hdf5(hid_t group) const
 }
 
 xt::xtensor<double, 1>
-RegularMesh::count_sites(const std::vector<Particle::Bank>& bank,
+RegularMesh::count_sites(const Particle::Bank* bank, uint64_t length,
   bool* outside) const
 {
   // Determine shape of array for counts
@@ -801,7 +801,9 @@ RegularMesh::count_sites(const std::vector<Particle::Bank>& bank,
   xt::xarray<double> cnt {shape, 0.0};
   bool outside_ = false;
 
-  for (const auto& site : bank) {
+  for (uint64_t i = 0; i < length; i++) {
+    const auto& site = bank[i];
+
     // determine scoring bin for entropy mesh
     int mesh_bin = get_bin(site.r);
 

src/physics.cpp

@@ -186,17 +186,17 @@ create_fission_sites(Particle* p, int i_nuclide, const Reaction* rx)
     {
       int idx;
       #pragma omp atomic capture
-      idx = simulation::shared_fission_bank_length++;
-      if( idx >= simulation::shared_fission_bank_max )
+      idx = simulation::fission_bank_length++;
+      if( idx >= simulation::fission_bank_max )
       {
         warning("The shared fission bank is full. Additional fission sites created "
             "in this generation will not be banked.");
         #pragma omp atomic write
-        simulation::shared_fission_bank_length = simulation::shared_fission_bank_max;
+        simulation::fission_bank_length = simulation::fission_bank_max;
         skipped++;
         break;
       }
-      site = simulation::shared_fission_bank + idx;
+      site = simulation::fission_bank + idx;
     }
     else
     {

src/physics_mg.cpp

@@ -133,17 +133,17 @@ create_fission_sites(Particle* p)
     {
       int idx;
       #pragma omp atomic capture
-      idx = simulation::shared_fission_bank_length++;
-      if( idx >= simulation::shared_fission_bank_max )
+      idx = simulation::fission_bank_length++;
+      if( idx >= simulation::fission_bank_max )
       {
         warning("The shared fission bank is full. Additional fission sites created "
             "in this generation will not be banked.");
         #pragma omp atomic write
-        simulation::shared_fission_bank_length = simulation::shared_fission_bank_max;
+        simulation::fission_bank_length = simulation::fission_bank_max;
         skipped++;
         break;
       }
-      site = simulation::shared_fission_bank + idx;
+      site = simulation::fission_bank + idx;
     }
     else
     {

src/simulation.cpp

@@ -264,11 +264,9 @@ void allocate_banks()
   simulation::source_bank.resize(simulation::work_per_rank);
 
   if (settings::run_mode == RUN_MODE_EIGENVALUE) {
-    simulation::fission_bank.reserve(3*simulation::work_per_rank);
+    init_fission_bank(3*simulation::work_per_rank);
   }
   
-  // Allocate shared bank
-  init_shared_fission_bank(simulation::work_per_rank * 3);
 }
 
 void initialize_batch()
@@ -369,7 +367,7 @@ void initialize_generation()
 {
   if (settings::run_mode == RUN_MODE_EIGENVALUE) {
     // Clear out the fission bank
-    simulation::fission_bank.clear();
+    simulation::fission_bank_length = 0;
 
     // Count source sites if using uniform fission source weighting
     if (settings::ufs_on) ufs_count_sites();
@@ -401,13 +399,11 @@ void finalize_generation()
   global_tally_leakage = 0.0;
 
   if (settings::run_mode == RUN_MODE_EIGENVALUE) {	
-    // Copy shared fission bank into regular bank for use in MPI synchronization
-    for( int i = 0; i < simulation::shared_fission_bank_length; i++ )
-      simulation::fission_bank.push_back(simulation::shared_fission_bank[i]);
-    simulation::shared_fission_bank_length = 0;
-
-    // Sorts the fission bank so as to allow for reproducibility
-    std::stable_sort(simulation::fission_bank.begin(), simulation::fission_bank.end());
+    // If using shared memory, sort the fission bank so as to allow for reproducibility
+    #ifdef _OPENMP
+    std::stable_sort(simulation::fission_bank,
+        simulation::fission_bank + simulation::fission_bank_length);
+    #endif
 
     // Distribute fission bank across processors evenly
     synchronize_bank();
@@ -430,6 +426,7 @@ void finalize_generation()
     // For fixed-source mode, we need to sample the external source
     fill_source_bank_fixedsource();
   }
+    
 }
 
 void initialize_history(Particle* p, int64_t index_source)