diff --git a/.travis.yml b/.travis.yml index a390bd1a0..df3ca21cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,8 +47,6 @@ matrix: env: OMP=y MPI=y PHDF5=y DAGMC=y - python: "3.7" env: OMP=y MPI=n PHDF5=n EVENT=y - - python: "3.7" - env: OMP=n MPI=y PHDF5=n EVENT=y notifications: webhooks: https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN install: diff --git a/CMakeLists.txt b/CMakeLists.txt index 95b334ce4..80909b83c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,6 @@ option(debug "Compile with debug flags" OFF) option(optimize "Turn on all compiler optimization flags" OFF) option(coverage "Compile with coverage analysis flags" OFF) option(dagmc "Enable support for DAGMC (CAD) geometry" OFF) -option(event "Enable event-based transport mode" OFF) #=============================================================================== # MPI for distributed-memory parallelism @@ -354,10 +353,6 @@ if(dagmc) target_include_directories(libopenmc PRIVATE ${DAGMC_INCLUDE_DIRS}) endif() -if(event) - target_compile_definitions(libopenmc PRIVATE EVENT_BASED) -endif() - #=============================================================================== # openmc executable #=============================================================================== diff --git a/docs/source/io_formats/settings.rst b/docs/source/io_formats/settings.rst index a371a3c39..d14a57667 100644 --- a/docs/source/io_formats/settings.rst +++ b/docs/source/io_formats/settings.rst @@ -137,6 +137,15 @@ The ```` element indicates the ID of a mesh that is to be used for calculating Shannon entropy. The mesh should cover all possible fissionable materials in the problem and is specified using a :ref:`mesh_element`. +---------------------------- +```` +---------------------------- + +Determines whether to use event-based parallelism instead of the default +history based parallelism. + + *Default*: false + ----------------------------------- ```` Element ----------------------------------- @@ -218,6 +227,16 @@ to false. *Default*: true +----------------------- +```` Element +----------------------- + +This element indicates the number of neutrons to run in flight concurrently +when using event-based parallelism. A higher value uses more memory, but +may be more efficient computationally. + + *Default*: 100000 + --------------------------- ```` Element --------------------------- diff --git a/include/openmc/settings.h b/include/openmc/settings.h index 670d5a5f9..65b4266bc 100644 --- a/include/openmc/settings.h +++ b/include/openmc/settings.h @@ -52,6 +52,7 @@ extern bool ufs_on; //!< uniform fission site method on? extern bool urr_ptables_on; //!< use unresolved resonance prob. tables? extern bool write_all_tracks; //!< write track files for every particle? extern bool write_initial_source; //!< write out initial source file? +extern bool event_based; //!< use event-based mode (instead of history-based) // Paths to various files extern std::string path_cross_sections; //!< path to cross_sections.xml diff --git a/openmc/settings.py b/openmc/settings.py index 27ca52a38..0ad42f175 100644 --- a/openmc/settings.py +++ b/openmc/settings.py @@ -53,6 +53,9 @@ class Settings(object): Mesh to be used to calculate Shannon entropy. If the mesh dimensions are not specified. OpenMC assigns a mesh such that 20 source sites per mesh cell are to be expected on average. + event_based : bool + Indicate whether to use event-based parallelism instead of the default + history-based parallelism. generations_per_batch : int Number of generations per batch inactive : int @@ -68,6 +71,9 @@ class Settings(object): material_cell_offsets : bool Generate an "offset table" for material cells by default. These tables are necessary when a particular instance of a cell needs to be tallied. + max_in_flight_particles : int + Number of neutrons to run concurrently when using event-based + parallelism. max_order : None or int Maximum scattering order to apply globally when in multi-group mode. no_reduce : bool @@ -229,6 +235,9 @@ class Settings(object): self._dagmc = False + self._event_based = None + self._max_in_flight_particles = None + @property def run_mode(self): return self._run_mode @@ -376,6 +385,14 @@ class Settings(object): @property def dagmc(self): return self._dagmc + + @property + def event(self): + return self._event + + @property + def max_in_flight_particles(self): + return self._max_in_flight_particles @run_mode.setter def run_mode(self, run_mode): @@ -704,6 +721,17 @@ class Settings(object): def delayed_photon_scaling(self, value): cv.check_type('delayed photon scaling', value, bool) self._delayed_photon_scaling = value + + @event_based.setter + def event_based(self, value): + cv.check_type('event based', value, bool) + self._event_based = value + + @max_in_flight_particles.setter + def max_in_flight_particles(self, value): + cv.check_type('max in flight particles', value, Integral) + cv.check_greater_than('max in flight particles', value, 0) + self._max_in_flight_particles = value @material_cell_offsets.setter def material_cell_offsets(self, value): @@ -948,6 +976,16 @@ class Settings(object): if self._delayed_photon_scaling is not None: elem = ET.SubElement(root, "delayed_photon_scaling") elem.text = str(self._delayed_photon_scaling).lower() + + def _create_event_based_subelement(self, root): + if self._event_based is not None: + elem = ET.SubElement(root, "event_based") + elem.text = str(self._event_based).lower() + + def _create_max_in_flight_particles_subelement(self, root): + if self._max_in_flight_particles is not None: + elem = ET.SubElement(root, "max_in_flight_particles") + elem.text = str(self._max_in_flight_particles).lower() def _create_material_cell_offsets_subelement(self, root): if self._material_cell_offsets is not None: @@ -1189,6 +1227,16 @@ class Settings(object): text = get_text(root, 'delayed_photon_scaling') if text is not None: self.delayed_photon_scaling = text in ('true', '1') + + def _event_based_from_xml_element(self, root): + text = get_text(root, 'event_based') + if text is not None: + self.event_based = text in ('true', '1') + + def _max_in_flight_particles_from_xml_element(self, root): + text = get_text(root, 'max_in_flight_particles') + if text is not None: + self.max_in_flight_particles = int(text) def _material_cell_offsets_from_xml_element(self, root): text = get_text(root, 'material_cell_offsets') @@ -1250,6 +1298,8 @@ class Settings(object): self._create_volume_calcs_subelement(root_element) self._create_create_fission_neutrons_subelement(root_element) self._create_delayed_photon_scaling_subelement(root_element) + self._create_event_based_subelement(root_element) + self._create_max_in_flight_particles_subelement(root_element) self._create_material_cell_offsets_subelement(root_element) self._create_log_grid_bins_subelement(root_element) self._create_dagmc_subelement(root_element) @@ -1317,6 +1367,8 @@ class Settings(object): settings._resonance_scattering_from_xml_element(root) settings._create_fission_neutrons_from_xml_element(root) settings._delayed_photon_scaling_from_xml_element(root) + settings._event_based_from_xml_element(root) + settings._max_in_flight_particles_from_xml_element(root) settings._material_cell_offsets_from_xml_element(root) settings._log_grid_bins_from_xml_element(root) settings._dagmc_from_xml_element(root) diff --git a/src/finalize.cpp b/src/finalize.cpp index 4ba1c7825..d2831c860 100644 --- a/src/finalize.cpp +++ b/src/finalize.cpp @@ -48,12 +48,12 @@ void free_memory() if (mpi::master) { free_memory_cmfd(); } + if (settings::event_based) { + free_event_queues(); + } #ifdef DAGMC free_memory_dagmc(); #endif -#ifdef EVENT_BASED - free_event_queues(); -#endif } } diff --git a/src/output.cpp b/src/output.cpp index 3001b8ed4..edcb80391 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -466,14 +466,14 @@ void print_runtime() show_time("Total time in simulation", time_inactive.elapsed() + time_active.elapsed()); show_time("Time in transport only", time_transport.elapsed(), 1); - #ifdef EVENT_BASED - show_time("Particle initialization", time_event_init.elapsed(), 2); - show_time("XS lookups", time_event_calculate_xs.elapsed(), 2); - show_time("Advancing", time_event_advance_particle.elapsed(), 2); - show_time("Surface crossings", time_event_surface_crossing.elapsed(), 2); - show_time("Collisions", time_event_collision.elapsed(), 2); - show_time("Particle death", time_event_death.elapsed(), 2); - #endif + if (settings::event_based) { + show_time("Particle initialization", time_event_init.elapsed(), 2); + show_time("XS lookups", time_event_calculate_xs.elapsed(), 2); + show_time("Advancing", time_event_advance_particle.elapsed(), 2); + show_time("Surface crossings", time_event_surface_crossing.elapsed(), 2); + show_time("Collisions", time_event_collision.elapsed(), 2); + show_time("Particle death", time_event_death.elapsed(), 2); + } if (settings::run_mode == RunMode::EIGENVALUE) { show_time("Time in inactive batches", time_inactive.elapsed(), 1); } diff --git a/src/relaxng/settings.rnc b/src/relaxng/settings.rnc index 3bc95f352..8fcb61670 100644 --- a/src/relaxng/settings.rnc +++ b/src/relaxng/settings.rnc @@ -23,7 +23,9 @@ element settings { element energy_mode { ( "continuous-energy" | "ce" | "CE" | "multi-group" | "mg" | "MG" ) }? & element entropy_mesh { xsd:positiveInteger }? & - + + element event_based { xsd:boolean }? & + element generations_per_batch { xsd:positiveInteger }? & element inactive { xsd:nonNegativeInteger }? & @@ -36,6 +38,8 @@ element settings { element log_grid_bins { xsd:positiveInteger }? & element material_cell_offsets { xsd:boolean }? & + + element max_in_flight_particles { xsd:positiveInteger }? & element max_order { xsd:nonNegativeInteger }? & diff --git a/src/relaxng/settings.rng b/src/relaxng/settings.rng index 4e795419d..6cb080670 100644 --- a/src/relaxng/settings.rng +++ b/src/relaxng/settings.rng @@ -87,6 +87,16 @@ + + + + + + + + + + diff --git a/src/settings.cpp b/src/settings.cpp index 3c8189c46..4d480523d 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -67,6 +67,7 @@ bool ufs_on {false}; bool urr_ptables_on {true}; bool write_all_tracks {false}; bool write_initial_source {false}; +bool event_based; {false}; std::string path_cross_sections; std::string path_input; @@ -791,6 +792,11 @@ void read_settings_xml() if (check_for_node(root, "delayed_photon_scaling")) { delayed_photon_scaling = get_node_value_bool(root, "delayed_photon_scaling"); } + + // Check whether to use event-based parallelism + if (check_for_node(root, "event_based")) { + event_based = get_node_value_bool(root, "event_based"); + } // Check whether material cell offsets should be generated if (check_for_node(root, "material_cell_offsets")) { diff --git a/src/simulation.cpp b/src/simulation.cpp index 6d21b1efc..495080472 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -75,11 +75,11 @@ int openmc_simulation_init() // If doing an event-based simulation, intialize the particle buffer // and event queues - #ifdef EVENT_BASED - int64_t event_buffer_length = std::min(simulation::work_per_rank, + if (settings::event_based) { + int64_t event_buffer_length = std::min(simulation::work_per_rank, settings::max_in_flight_particles); - init_event_queues(event_buffer_length); - #endif + init_event_queues(event_buffer_length); + } // Allocate tally results arrays if they're not allocated yet for (auto& t : model::tallies) { @@ -192,11 +192,11 @@ int openmc_next_batch(int* status) simulation::time_transport.start(); // Transport loop - #ifdef EVENT_BASED - transport_event_based(); - #else - transport_history_based(); - #endif + if (settings::event_based) { + transport_event_based(); + } else { + transport_history_based(); + } // Accumulate time for transport simulation::time_transport.stop();