first pass on adding event_based and in_flight_particles as xml and python options.

.travis.yml

@@ -47,8 +47,6 @@ matrix:
       env: OMP=y MPI=y PHDF5=y DAGMC=y
     - python: "3.7"
       env: OMP=y MPI=n PHDF5=n EVENT=y
-    - python: "3.7"
-      env: OMP=n MPI=y PHDF5=n EVENT=y
 notifications:
   webhooks: https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN
 install:

CMakeLists.txt

@@ -19,7 +19,6 @@ option(debug    "Compile with debug flags"                       OFF)
 option(optimize "Turn on all compiler optimization flags"        OFF)
 option(coverage "Compile with coverage analysis flags"           OFF)
 option(dagmc    "Enable support for DAGMC (CAD) geometry"        OFF)
-option(event    "Enable event-based transport mode"              OFF)
 
 #===============================================================================
 # MPI for distributed-memory parallelism
@@ -354,10 +353,6 @@ if(dagmc)
   target_include_directories(libopenmc PRIVATE ${DAGMC_INCLUDE_DIRS})
 endif()
 
-if(event)
-  target_compile_definitions(libopenmc PRIVATE EVENT_BASED)
-endif()
-
 #===============================================================================
 # openmc executable
 #===============================================================================

docs/source/io_formats/settings.rst

@@ -137,6 +137,15 @@ The ``<entropy_mesh>`` element indicates the ID of a mesh that is to be used for
 calculating Shannon entropy. The mesh should cover all possible fissionable
 materials in the problem and is specified using a :ref:`mesh_element`.
 
+----------------------------
+``<event_based>``
+----------------------------
+
+Determines whether to use event-based parallelism instead of the default
+history based parallelism. 
+
+  *Default*: false
+
 -----------------------------------
 ``<generations_per_batch>`` Element
 -----------------------------------
@@ -218,6 +227,16 @@ to false.
 
   *Default*: true
 
+-----------------------
+``<max_in_flight_particles>`` Element
+-----------------------
+
+This element indicates the number of neutrons to run in flight concurrently
+when using event-based parallelism. A higher value uses more memory, but
+may be more efficient computationally.
+
+  *Default*: 100000
+
 ---------------------------
 ``<max_order>`` Element
 ---------------------------

include/openmc/settings.h

@@ -52,6 +52,7 @@ extern bool ufs_on;                   //!< uniform fission site method on?
 extern bool urr_ptables_on;           //!< use unresolved resonance prob. tables?
 extern bool write_all_tracks;         //!< write track files for every particle?
 extern bool write_initial_source;     //!< write out initial source file?
+extern bool event_based;              //!< use event-based mode (instead of history-based)
 
 // Paths to various files
 extern std::string path_cross_sections;   //!< path to cross_sections.xml

openmc/settings.py

@@ -53,6 +53,9 @@ class Settings(object):
         Mesh to be used to calculate Shannon entropy. If the mesh dimensions are
         not specified. OpenMC assigns a mesh such that 20 source sites per mesh
         cell are to be expected on average.
+    event_based : bool
+        Indicate whether to use event-based parallelism instead of the default
+        history-based parallelism.
     generations_per_batch : int
         Number of generations per batch
     inactive : int
@@ -68,6 +71,9 @@ class Settings(object):
     material_cell_offsets : bool
         Generate an "offset table" for material cells by default. These tables
         are necessary when a particular instance of a cell needs to be tallied.
+    max_in_flight_particles : int
+        Number of neutrons to run concurrently when using event-based
+        parallelism.
     max_order : None or int
         Maximum scattering order to apply globally when in multi-group mode.
     no_reduce : bool
@@ -229,6 +235,9 @@ class Settings(object):
 
         self._dagmc = False
 
+        self._event_based = None
+        self._max_in_flight_particles = None
+
     @property
     def run_mode(self):
         return self._run_mode
@@ -376,6 +385,14 @@ class Settings(object):
     @property
     def dagmc(self):
         return self._dagmc
+    
+    @property
+    def event(self):
+        return self._event
+    
+    @property
+    def max_in_flight_particles(self):
+        return self._max_in_flight_particles
 
     @run_mode.setter
     def run_mode(self, run_mode):
@@ -704,6 +721,17 @@ class Settings(object):
     def delayed_photon_scaling(self, value):
         cv.check_type('delayed photon scaling', value, bool)
         self._delayed_photon_scaling = value
+    
+    @event_based.setter
+    def event_based(self, value):
+        cv.check_type('event based', value, bool)
+        self._event_based = value
+    
+    @max_in_flight_particles.setter
+    def max_in_flight_particles(self, value):
+        cv.check_type('max in flight particles', value, Integral)
+        cv.check_greater_than('max in flight particles', value, 0)
+        self._max_in_flight_particles = value
 
     @material_cell_offsets.setter
     def material_cell_offsets(self, value):
@@ -948,6 +976,16 @@ class Settings(object):
         if self._delayed_photon_scaling is not None:
             elem = ET.SubElement(root, "delayed_photon_scaling")
             elem.text = str(self._delayed_photon_scaling).lower()
+    
+    def _create_event_based_subelement(self, root):
+        if self._event_based is not None:
+            elem = ET.SubElement(root, "event_based")
+            elem.text = str(self._event_based).lower()
+    
+    def _create_max_in_flight_particles_subelement(self, root):
+        if self._max_in_flight_particles is not None:
+            elem = ET.SubElement(root, "max_in_flight_particles")
+            elem.text = str(self._max_in_flight_particles).lower()
 
     def _create_material_cell_offsets_subelement(self, root):
         if self._material_cell_offsets is not None:
@@ -1189,6 +1227,16 @@ class Settings(object):
         text = get_text(root, 'delayed_photon_scaling')
         if text is not None:
             self.delayed_photon_scaling = text in ('true', '1')
+    
+    def _event_based_from_xml_element(self, root):
+        text = get_text(root, 'event_based')
+        if text is not None:
+            self.event_based = text in ('true', '1')
+    
+    def _max_in_flight_particles_from_xml_element(self, root):
+        text = get_text(root, 'max_in_flight_particles')
+        if text is not None:
+            self.max_in_flight_particles = int(text)
 
     def _material_cell_offsets_from_xml_element(self, root):
         text = get_text(root, 'material_cell_offsets')
@@ -1250,6 +1298,8 @@ class Settings(object):
         self._create_volume_calcs_subelement(root_element)
         self._create_create_fission_neutrons_subelement(root_element)
         self._create_delayed_photon_scaling_subelement(root_element)
+        self._create_event_based_subelement(root_element)
+        self._create_max_in_flight_particles_subelement(root_element)
         self._create_material_cell_offsets_subelement(root_element)
         self._create_log_grid_bins_subelement(root_element)
         self._create_dagmc_subelement(root_element)
@@ -1317,6 +1367,8 @@ class Settings(object):
         settings._resonance_scattering_from_xml_element(root)
         settings._create_fission_neutrons_from_xml_element(root)
         settings._delayed_photon_scaling_from_xml_element(root)
+        settings._event_based_from_xml_element(root)
+        settings._max_in_flight_particles_from_xml_element(root)
         settings._material_cell_offsets_from_xml_element(root)
         settings._log_grid_bins_from_xml_element(root)
         settings._dagmc_from_xml_element(root)

src/finalize.cpp

@@ -48,12 +48,12 @@ void free_memory()
   if (mpi::master) {
     free_memory_cmfd();
   }
+  if (settings::event_based) {
+    free_event_queues();
+  }
 #ifdef DAGMC
   free_memory_dagmc();
 #endif
-#ifdef EVENT_BASED
-  free_event_queues();
-#endif
 }
 
 }

src/output.cpp

@@ -466,14 +466,14 @@ void print_runtime()
   show_time("Total time in simulation", time_inactive.elapsed() +
     time_active.elapsed());
   show_time("Time in transport only", time_transport.elapsed(), 1);
-  #ifdef EVENT_BASED
-  show_time("Particle initialization", time_event_init.elapsed(), 2);
-  show_time("XS lookups", time_event_calculate_xs.elapsed(), 2);
-  show_time("Advancing", time_event_advance_particle.elapsed(), 2);
-  show_time("Surface crossings", time_event_surface_crossing.elapsed(), 2);
-  show_time("Collisions", time_event_collision.elapsed(), 2);
-  show_time("Particle death", time_event_death.elapsed(), 2);
-  #endif
+  if (settings::event_based) {
+    show_time("Particle initialization", time_event_init.elapsed(), 2);
+    show_time("XS lookups", time_event_calculate_xs.elapsed(), 2);
+    show_time("Advancing", time_event_advance_particle.elapsed(), 2);
+    show_time("Surface crossings", time_event_surface_crossing.elapsed(), 2);
+    show_time("Collisions", time_event_collision.elapsed(), 2);
+    show_time("Particle death", time_event_death.elapsed(), 2);
+  }
   if (settings::run_mode == RunMode::EIGENVALUE) {
     show_time("Time in inactive batches", time_inactive.elapsed(), 1);
   }

src/relaxng/settings.rnc

@@ -23,7 +23,9 @@ element settings {
   element energy_mode { ( "continuous-energy" | "ce" | "CE" | "multi-group" | "mg" | "MG" ) }? &
 
   element entropy_mesh { xsd:positiveInteger }? &
-
+  
+  element event_based { xsd:boolean }? &
+  
   element generations_per_batch { xsd:positiveInteger }? &
 
   element inactive { xsd:nonNegativeInteger }? &
@@ -36,6 +38,8 @@ element settings {
   element log_grid_bins { xsd:positiveInteger }? &
 
   element material_cell_offsets { xsd:boolean }? &
+  
+  element max_in_flight_particles { xsd:positiveInteger }? &
 
   element max_order { xsd:nonNegativeInteger }? &
 

src/relaxng/settings.rng

@@ -87,6 +87,16 @@
         <data type="boolean"/>
       </element>
     </optional>
+    <optional>
+      <element name="event_based">
+        <data type="boolean"/>
+      </element>
+    </optional>
+    <optional>
+      <element name="max_in_flight_particles">
+        <data type="positiveInteger"/>
+      </element>
+    </optional>
     <optional>
       <element name="electron_treatment">
         <choice>

src/settings.cpp

@@ -67,6 +67,7 @@ bool ufs_on                  {false};
 bool urr_ptables_on          {true};
 bool write_all_tracks        {false};
 bool write_initial_source    {false};
+bool event_based;            {false};
 
 std::string path_cross_sections;
 std::string path_input;
@@ -791,6 +792,11 @@ void read_settings_xml()
   if (check_for_node(root, "delayed_photon_scaling")) {
     delayed_photon_scaling = get_node_value_bool(root, "delayed_photon_scaling");
   }
+  
+  // Check whether to use event-based parallelism
+  if (check_for_node(root, "event_based")) {
+    event_based = get_node_value_bool(root, "event_based");
+  }
 
   // Check whether material cell offsets should be generated
   if (check_for_node(root, "material_cell_offsets")) {

src/simulation.cpp

@@ -75,11 +75,11 @@ int openmc_simulation_init()
 
   // If doing an event-based simulation, intialize the particle buffer
   // and event queues
-  #ifdef EVENT_BASED
-  int64_t event_buffer_length = std::min(simulation::work_per_rank,
+  if (settings::event_based) {
+    int64_t event_buffer_length = std::min(simulation::work_per_rank,
       settings::max_in_flight_particles);
-  init_event_queues(event_buffer_length);
-  #endif
+    init_event_queues(event_buffer_length);
+  }
 
   // Allocate tally results arrays if they're not allocated yet
   for (auto& t : model::tallies) {
@@ -192,11 +192,11 @@ int openmc_next_batch(int* status)
     simulation::time_transport.start();
 
     // Transport loop
-    #ifdef EVENT_BASED
-    transport_event_based();
-    #else
-    transport_history_based();
-    #endif
+    if (settings::event_based) {
+      transport_event_based();
+    } else {
+      transport_history_based();
+    }
 
     // Accumulate time for transport
     simulation::time_transport.stop();