Merge branch 'develop' into add_enums2

include/openmc/bank.h

@@ -7,11 +7,6 @@
 #include "openmc/particle.h"
 #include "openmc/position.h"
 
-// Without an explicit instantiation of vector<Bank>, the Intel compiler
-// will complain about the threadprivate directive on filter_matches. Note that
-// this has to happen *outside* of the openmc namespace
-extern template class std::vector<openmc::Particle::Bank>;
-
 namespace openmc {
 
 //==============================================================================
@@ -21,13 +16,11 @@ namespace openmc {
 namespace simulation {
 
 extern std::vector<Particle::Bank> source_bank;
-extern std::vector<Particle::Bank> fission_bank;
-extern std::vector<Particle::Bank> secondary_bank;
-#ifdef _OPENMP
-extern std::vector<Particle::Bank> master_fission_bank;
-#endif
 
-#pragma omp threadprivate(fission_bank, secondary_bank)
+extern std::unique_ptr<Particle::Bank[]> fission_bank;
+extern int64_t fission_bank_length;
+extern int64_t fission_bank_max;
+extern std::vector<int64_t> progeny_per_particle;
 
 } // namespace simulation
 
@@ -35,8 +28,12 @@ extern std::vector<Particle::Bank> master_fission_bank;
 // Non-member functions
 //==============================================================================
 
+void sort_fission_bank();
+
 void free_memory_bank();
 
+void init_fission_bank(int64_t max);
+
 } // namespace openmc
 
 #endif // OPENMC_BANK_H

include/openmc/cell.h

@@ -113,7 +113,7 @@ public:
 
   //! Find the oncoming boundary of this cell.
   virtual std::pair<double, int32_t>
-  distance(Position r, Direction u, int32_t on_surface) const = 0;
+  distance(Position r, Direction u, int32_t on_surface, Particle* p) const = 0;
 
   //! Write all information needed to reconstruct the cell to an HDF5 group.
   //! \param group_id An HDF5 group id.
@@ -204,7 +204,7 @@ public:
   contains(Position r, Direction u, int32_t on_surface) const;
 
   std::pair<double, int32_t>
-  distance(Position r, Direction u, int32_t on_surface) const;
+  distance(Position r, Direction u, int32_t on_surface, Particle* p) const;
 
   void to_hdf5(hid_t group_id) const;
 
@@ -248,7 +248,7 @@ public:
   bool contains(Position r, Direction u, int32_t on_surface) const;
 
   std::pair<double, int32_t>
-  distance(Position r, Direction u, int32_t on_surface) const;
+  distance(Position r, Direction u, int32_t on_surface, Particle* p) const;
 
   BoundingBox bounding_box() const;
 

include/openmc/dagmc.h

@@ -14,14 +14,6 @@ extern "C" const bool dagmc_enabled;
 
 namespace openmc {
 
-namespace simulation {
-
-extern moab::DagMC::RayHistory history; //!< facet history for DagMC particles
-extern Direction last_dir; //!< last direction passed to DagMC's ray_fire
-#pragma omp threadprivate(history, last_dir)
-
-}
-
 namespace model {
   extern moab::DagMC* DAG;
 }

include/openmc/eigenvalue.h

@@ -42,14 +42,6 @@ void calculate_generation_keff();
 //! generations. It also broadcasts the value from the master process.
 void calculate_average_keff();
 
-#ifdef _OPENMP
-//! Join threadprivate fission banks into a single fission bank
-//!
-//! Note that this operation is necessarily sequential to preserve the order of
-//! the bank when using varying numbers of threads.
-void join_bank_from_threads();
-#endif
-
 //! Calculates a minimum variance estimate of k-effective
 //!
 //! The minimum variance estimate is based on a linear combination of the

include/openmc/geometry.h

@@ -24,17 +24,6 @@ extern std::vector<int64_t> overlap_check_count;
 
 } // namespace model
 
-//==============================================================================
-// Information about nearest boundary crossing
-//==============================================================================
-
-struct BoundaryInfo {
-  double distance {INFINITY};   //!< distance to nearest boundary
-  int surface_index {0}; //!< if boundary is surface, index in surfaces vector
-  int coord_level;   //!< coordinate level after crossing boundary
-  std::array<int, 3> lattice_translation {}; //!< which way lattice indices will change
-};
-
 //==============================================================================
 //! Check two distances by coincidence tolerance
 //==============================================================================

include/openmc/mesh.h

@@ -161,7 +161,7 @@ public:
   //! \param[in] bank Array of bank sites
   //! \param[out] Whether any bank sites are outside the mesh
   //! \return Array indicating number of sites in each mesh/energy bin
-  xt::xtensor<double, 1> count_sites(const std::vector<Particle::Bank>& bank,
+  xt::xtensor<double, 1> count_sites(const Particle::Bank* bank, int64_t length,
     bool* outside) const;
 
   // Data members

include/openmc/particle.h

@@ -14,6 +14,12 @@
 #include "openmc/constants.h"
 #include "openmc/position.h"
 #include "openmc/random_lcg.h"
+#include "openmc/tallies/filter_match.h"
+
+#ifdef DAGMC
+#include "DagMC.hpp"
+#endif
+
 
 namespace openmc {
 
@@ -131,6 +137,17 @@ struct MacroXS {
   double pair_production; //!< macroscopic pair production xs
 };
 
+//==============================================================================
+// Information about nearest boundary crossing
+//==============================================================================
+
+struct BoundaryInfo {
+  double distance {INFINITY};   //!< distance to nearest boundary
+  int surface_index {0}; //!< if boundary is surface, index in surfaces vector
+  int coord_level;   //!< coordinate level after crossing boundary
+  std::array<int, 3> lattice_translation {}; //!< which way lattice indices will change
+};
+
 //============================================================================
 //! State of a particle being transported through geometry
 //============================================================================
@@ -146,6 +163,12 @@ public:
   };
 
   //! Saved ("banked") state of a particle
+  //! NOTE: This structure's MPI type is built in initialize_mpi() of 
+  //! initialize.cpp. Any changes made to the struct here must also be
+  //! made when building the Bank MPI type in initialize_mpi().
+  //! NOTE: This structure is also used on the python side, and is defined
+  //! in lib/core.py. Changes made to the type here must also be made to the
+  //! python defintion.
   struct Bank {
     Position r;
     Direction u;
@@ -153,6 +176,15 @@ public:
     double wgt;
     int delayed_group;
     Type particle;
+    int64_t parent_id;
+    int64_t progeny_id;
+  };
+  
+  //! Saved ("banked") state of a particle, for nu-fission tallying
+  struct NuBank {
+    double E;  //!< particle energy
+    double wgt; //!< particle weight
+    int delayed_group; //!< particle delayed group
   };
 
   //==========================================================================
@@ -199,8 +231,13 @@ public:
   //! \param src Source site data
   void from_source(const Bank* src);
 
-  //! Transport a particle from birth to death
-  void transport();
+  // Coarse-grained particle events
+  void event_calculate_xs();
+  void event_advance();
+  void event_cross_surface();
+  void event_collide();
+  void event_revive_from_secondary();
+  void event_death();
 
   //! Cross a surface and handle boundary conditions
   void cross_surface();
@@ -276,10 +313,13 @@ public:
                                             //!< sites banked
 
   // Indices for various arrays
-  int surface_ {0};             //!< index for surface particle is on
+  int surface_ {0};         //!< index for surface particle is on
   int cell_born_ {-1};      //!< index for cell particle was born in
   int material_ {-1};       //!< index for current material
   int material_last_ {-1};  //!< index for last material
+  
+  // Boundary information
+  BoundaryInfo boundary_;
 
   // Temperature of current cell
   double sqrtkT_ {-1.0};      //!< sqrt(k_Boltzmann * temperature) in eV
@@ -294,6 +334,39 @@ public:
   // Current PRNG state
   uint64_t seeds_[N_STREAMS]; // current seeds
   int      stream_;           // current RNG stream
+  
+  // Secondary particle bank
+  std::vector<Particle::Bank> secondary_bank_;
+
+  int64_t current_work_; // current work index
+
+  std::vector<double> flux_derivs_;  // for derivatives for this particle
+
+  std::vector<FilterMatch> filter_matches_; // tally filter matches
+
+  std::vector<std::vector<Position>> tracks_; // tracks for outputting to file
+
+  std::vector<NuBank> nu_bank_; // bank of most recently fissioned particles
+
+  // Global tally accumulators
+  double keff_tally_absorption_ {0.0};
+  double keff_tally_collision_ {0.0};
+  double keff_tally_tracklength_ {0.0};
+  double keff_tally_leakage_ {0.0};
+
+  bool trace_ {false};     //!< flag to show debug information
+
+  double collision_distance_; // distance to particle's next closest collision
+
+  int n_event_ {0}; // number of events executed in this particle's history
+
+  // DagMC state variables
+  #ifdef DAGMC
+  moab::DagMC::RayHistory history_;
+  Direction last_dir_;
+  #endif
+
+  int64_t n_progeny_ {0}; // Number of progeny produced by this particle
 };
 
 } // namespace openmc

include/openmc/physics.h

@@ -48,8 +48,7 @@ int sample_nuclide(Particle* p);
 
 //! Determine the average total, prompt, and delayed neutrons produced from
 //! fission and creates appropriate bank sites.
-void create_fission_sites(Particle* p, int i_nuclide, const Reaction* rx,
-  std::vector<Particle::Bank>& bank);
+void create_fission_sites(Particle* p, int i_nuclide, const Reaction* rx);
 
 int sample_element(Particle* p);
 

include/openmc/physics_mg.h

@@ -33,9 +33,8 @@ scatter(Particle* p);
 //! \brief Determines the average total, prompt and delayed neutrons produced
 //! from fission and creates the appropriate bank sites.
 //! \param p Particle to operate on
-//! \param bank The particle bank to populate
 void
-create_fission_sites(Particle* p, std::vector<Particle::Bank>& bank);
+create_fission_sites(Particle* p);
 
 //! \brief Handles an absorption event
 //! \param p Particle to operate on

include/openmc/simulation.h

@@ -24,7 +24,6 @@ namespace simulation {
 
 extern "C" int current_batch;    //!< current batch
 extern "C" int current_gen;      //!< current fission generation
-extern "C" int64_t current_work; //!< index in source back of current particle
 extern "C" bool initialized;     //!< has simulation been initialized?
 extern "C" double keff;          //!< average k over batches
 extern "C" double keff_std;      //!< standard deviation of average k
@@ -46,11 +45,6 @@ extern const RegularMesh* ufs_mesh;
 extern std::vector<double> k_generation;
 extern std::vector<int64_t> work_index;
 
-// Threadprivate variables
-extern "C" bool trace;     //!< flag to show debug information
-
-#pragma omp threadprivate(current_work, trace)
-
 } // namespace simulation
 
 //==============================================================================
@@ -69,8 +63,12 @@ void initialize_batch();
 //! Initialize a fission generation
 void initialize_generation();
 
+//! Full initialization of a particle history
 void initialize_history(Particle* p, int64_t index_source);
 
+//! Helper function for initialize_history() that is called independently elsewhere
+void initialize_history_partial(Particle* p);
+
 //! Finalize a batch
 //!
 //! Handles synchronization and accumulation of tallies, calculation of Shannon
@@ -90,6 +88,13 @@ void broadcast_results();
 
 void free_memory_simulation();
 
+//! Simulate a single particle history (and all generated secondary particles,
+//!  if enabled), from birth to death
+void transport_history_based_single_particle(Particle& p);
+
+//! Simulate all particle histories using history-based parallelism
+void transport_history_based();
+
 } // namespace openmc
 
 #endif // OPENMC_SIMULATION_H

include/openmc/surface.h

@@ -11,6 +11,7 @@
 #include "pugixml.hpp"
 
 #include "openmc/constants.h"
+#include "openmc/particle.h"
 #include "openmc/position.h"
 #include "dagmc.h"
 
@@ -112,8 +113,9 @@ public:
   //! Determine the direction of a ray reflected from the surface.
   //! \param[in] r The point at which the ray is incident.
   //! \param[in] u Incident direction of the ray
+  //! \param[inout] p Pointer to the particle
   //! \return Outgoing direction of the ray
-  virtual Direction reflect(Position r, Direction u) const;
+  virtual Direction reflect(Position r, Direction u, Particle* p) const;
 
   virtual Direction diffuse_reflect(Position r, Direction u,
     uint64_t* seed) const;
@@ -170,7 +172,7 @@ public:
   double evaluate(Position r) const;
   double distance(Position r, Direction u, bool coincident) const;
   Direction normal(Position r) const;
-  Direction reflect(Position r, Direction u) const;
+  Direction reflect(Position r, Direction u, Particle* p) const;
 
   void to_hdf5(hid_t group_id) const;
 

include/openmc/tallies/derivative.h

@@ -27,7 +27,6 @@ struct TallyDerivative {
   int id;  //!< User-defined identifier
   int diff_material;  //!< Material this derivative is applied to
   int diff_nuclide;  //!< Nuclide this material is applied to
-  double flux_deriv;  //!< Derivative of the current particle's weight
 
   TallyDerivative() {}
   explicit TallyDerivative(pugi::xml_node node);
@@ -54,16 +53,13 @@ apply_derivative_to_score(const Particle* p, int i_tally, int i_nuclide,
 //! further tallies are scored.
 //
 //! \param p The particle being tracked
-void score_collision_derivative(const Particle* p);
+void score_collision_derivative(Particle* p);
 
 //! Adjust diff tally flux derivatives for a particle tracking event.
 //
 //! \param p The particle being tracked
 //! \param distance The distance in [cm] traveled by the particle
-void score_track_derivative(const Particle* p, double distance);
-
-//! Set the flux derivatives on differential tallies to zero.
-void zero_flux_derivs();
+void score_track_derivative(Particle* p, double distance);
 
 } // namespace openmc
 
@@ -71,15 +67,10 @@ void zero_flux_derivs();
 // Global variables
 //==============================================================================
 
-// Explicit vector template specialization declaration of threadprivate variable
-// outside of the openmc namespace for the picky Intel compiler.
-extern template class std::vector<openmc::TallyDerivative>;
-
 namespace openmc {
 
 namespace model {
 extern std::vector<TallyDerivative> tally_derivs;
-#pragma omp threadprivate(tally_derivs)
 extern std::unordered_map<int, int> tally_deriv_map;
 } // namespace model
 

include/openmc/tallies/filter.h

@@ -12,31 +12,10 @@
 #include "openmc/constants.h"
 #include "openmc/hdf5_interface.h"
 #include "openmc/particle.h"
+#include "openmc/tallies/filter_match.h"
 #include "pugixml.hpp"
 
 
-namespace openmc {
-
-//==============================================================================
-//! Stores bins and weights for filtered tally events.
-//==============================================================================
-
-class FilterMatch
-{
-public:
-  std::vector<int> bins_;
-  std::vector<double> weights_;
-  int i_bin_;
-  bool bins_present_ {false};
-};
-
-} // namespace openmc
-
-// Without an explicit instantiation of vector<FilterMatch>, the Intel compiler
-// will complain about the threadprivate directive on filter_matches. Note that
-// this has to happen *outside* of the openmc namespace
-extern template class std::vector<openmc::FilterMatch>;
-
 namespace openmc {
 
 //==============================================================================
@@ -127,13 +106,6 @@ private:
 // Global variables
 //==============================================================================
 
-namespace simulation {
-
-extern std::vector<FilterMatch> filter_matches;
-#pragma omp threadprivate(filter_matches)
-
-} // namespace simulation
-
 namespace model {
   extern "C" int32_t n_filters;
   extern std::vector<std::unique_ptr<Filter>> tally_filters;

include/openmc/tallies/filter_match.h

@@ -0,0 +1,21 @@
+#ifndef OPENMC_TALLIES_FILTERMATCH_H
+#define OPENMC_TALLIES_FILTERMATCH_H
+
+
+namespace openmc {
+
+//==============================================================================
+//! Stores bins and weights for filtered tally events.
+//==============================================================================
+
+class FilterMatch
+{
+public:
+  std::vector<int> bins_;
+  std::vector<double> weights_;
+  int i_bin_;
+  bool bins_present_ {false};
+};
+
+} // namespace openmc
+#endif // OPENMC_TALLIES_FILTERMATCH_H

include/openmc/tallies/tally.h

@@ -160,17 +160,10 @@ namespace simulation {
   extern "C" int32_t n_realizations;
 }
 
-// It is possible to protect accumulate operations on global tallies by using an
-// atomic update. However, when multiple threads accumulate to the same global
-// tally, it can cause a higher cache miss rate due to invalidation. Thus, we
-// use threadprivate variables to accumulate global tallies and then reduce at
-// the end of a generation.
 extern double global_tally_absorption;
 extern double global_tally_collision;
 extern double global_tally_tracklength;
 extern double global_tally_leakage;
-#pragma omp threadprivate(global_tally_absorption, global_tally_collision, \
-  global_tally_tracklength, global_tally_leakage)
 
 //==============================================================================
 // Non-member functions

include/openmc/tallies/tally_scoring.h

@@ -2,6 +2,7 @@
 #define OPENMC_TALLIES_TALLY_SCORING_H
 
 #include "openmc/particle.h"
+#include "openmc/tallies/filter.h"
 #include "openmc/tallies/tally.h"
 
 namespace openmc {
@@ -22,12 +23,13 @@ class FilterBinIter
 public:
 
   //! Construct an iterator over bins that match a given particle's state.
-  FilterBinIter(const Tally& tally, const Particle* p);
+  FilterBinIter(const Tally& tally, Particle* p);
 
   //! Construct an iterator over all filter bin combinations.
   //
   //! \param end if true, the returned iterator indicates the end of a loop.
-  FilterBinIter(const Tally& tally, bool end);
+  FilterBinIter(const Tally& tally, bool end,
+      std::vector<FilterMatch>* particle_filter_matches);
 
   bool operator==(const FilterBinIter& other) const
   {return index_ == other.index_;}
@@ -39,6 +41,8 @@ public:
 
   int index_ {1};
   double weight_ {1.};
+  
+  std::vector<FilterMatch>& filter_matches_;
 
 private:
   void compute_index_weight();
@@ -73,7 +77,7 @@ void score_analog_tally_ce(Particle* p);
 //! Analog tallies are triggered at every collision, not every event.
 //
 //! \param p The particle being tracked
-void score_analog_tally_mg(const Particle* p);
+void score_analog_tally_mg(Particle* p);
 
 //! Score tallies using a tracklength estimate of the flux.
 //
@@ -89,7 +93,7 @@ void score_tracklength_tally(Particle* p, double distance);
 //
 //! \param p The particle being tracked
 //! \param tallies A vector of tallies to score to
-void score_surface_tally(const Particle* p, const std::vector<int>& tallies);
+void score_surface_tally(Particle* p, const std::vector<int>& tallies);
 
 } // namespace openmc
 

include/openmc/timer.h

@@ -21,6 +21,7 @@ extern Timer time_finalize;
 extern Timer time_inactive;
 extern Timer time_initialize;
 extern Timer time_read_xs;
+extern Timer time_sample_source;
 extern Timer time_tallies;
 extern Timer time_total;
 extern Timer time_transport;

include/openmc/track_output.h

@@ -9,9 +9,9 @@ namespace openmc {
 // Non-member functions
 //==============================================================================
 
-void add_particle_track();
-void write_particle_track(const Particle& p);
-void finalize_particle_track(const Particle& p);
+void add_particle_track(Particle& p);
+void write_particle_track(Particle& p);
+void finalize_particle_track(Particle& p);
 
 } // namespace openmc