From 732a780366797aa832c6b23932fcf97b5eac9287 Mon Sep 17 00:00:00 2001
From: John Tramm <john.tramm@gmail.com>
Date: Fri, 17 Jan 2020 15:03:34 +0000
Subject: [PATCH] fixed indexing error, though things are still going haywire
 with MPI. Going to test if this is due to sort or unique_ptr.

---
 src/bank.cpp       | 8 +++++---
 src/particle.cpp   | 7 +++++--
 src/simulation.cpp | 6 ++----
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/bank.cpp b/src/bank.cpp
index b9bf9056d..c4f1511b9 100644
--- a/src/bank.cpp
+++ b/src/bank.cpp
@@ -2,6 +2,7 @@
 #include "openmc/capi.h"
 #include "openmc/error.h"
 #include "openmc/simulation.h"
+#include "openmc/message_passing.h"
 
 #include <cstdint>
 
@@ -74,7 +75,7 @@ void sort_fission_bank()
     tmp = simulation::progeny_per_particle[i];
     simulation::progeny_per_particle[i] = value;
   }
-  
+
   // TODO: C++17 introduces the exclusive_scan() function which could be
   // used to replace everything above this point in this function.
 
@@ -83,8 +84,9 @@ void sort_fission_bank()
 
   // Use parent and progeny indices to sort fission bank
   for (int64_t i = 0; i < simulation::fission_bank_length; i++) {
-    Particle::Bank& site = simulation::fission_bank[i];
-    int64_t idx = simulation::progeny_per_particle[site.parent_id-1] + site.progeny_id;
+    const auto& site = simulation::fission_bank[i];
+    int64_t offset = site.parent_id - 1 - simulation::work_index[mpi::rank];
+    int64_t idx = simulation::progeny_per_particle[offset] + site.progeny_id;
     sorted_bank[idx] = site;
   }
 
diff --git a/src/particle.cpp b/src/particle.cpp
index b9858b3c3..89eeb4372 100644
--- a/src/particle.cpp
+++ b/src/particle.cpp
@@ -168,7 +168,7 @@ Particle::event_calculate_xs()
   if (write_track_) write_particle_track(*this);
 
   if (settings::check_overlaps) check_cell_overlap(this);
-  
+
   // Calculate microscopic and macroscopic cross sections
   if (material_ != MATERIAL_VOID) {
     if (settings::run_CE) {
@@ -394,7 +394,10 @@ Particle::event_death()
 
   // Record the number of progeny created by this particle.
   // This data will be used to efficiently sort the fission bank.
-  simulation::progeny_per_particle[id_-1] = n_progeny_;
+  int64_t offset = id_ - 1 - simulation::work_index[mpi::rank];
+  assert(offset >= 0);
+  assert(offset < simulation::work_per_rank);
+  simulation::progeny_per_particle[offset] = n_progeny_;
 }
 
 
diff --git a/src/simulation.cpp b/src/simulation.cpp
index 178621db0..40819d1b0 100644
--- a/src/simulation.cpp
+++ b/src/simulation.cpp
@@ -397,11 +397,9 @@ void finalize_generation()
     // If using shared memory, stable sort the fission bank (by parent IDs) 
     // so as to allow for reproducibility regardless of which order particles
     // are run in.
-    #ifdef _OPENMP
-    //std::stable_sort(simulation::fission_bank,
-    //    simulation::fission_bank + simulation::fission_bank_length);
+    //#ifdef _OPENMP
     sort_fission_bank();
-    #endif
+    //#endif
 
     // Distribute fission bank across processors evenly
     synchronize_bank();