From 8f71ed2a1dc488f0f99ce4a0aebe8a624ae1ea1f Mon Sep 17 00:00:00 2001 From: Paul Romano Date: Thu, 15 Oct 2020 16:41:13 -0500 Subject: [PATCH 01/10] Store sites from source file in SourceDistribution --- include/openmc/settings.h | 1 - include/openmc/source.h | 1 + include/openmc/state_point.h | 4 +- src/settings.cpp | 1 - src/source.cpp | 76 +++++++++++++++++++----------------- src/state_point.cpp | 24 +++++++----- 6 files changed, 58 insertions(+), 49 deletions(-) diff --git a/include/openmc/settings.h b/include/openmc/settings.h index 85ef1c71ea..c743cf7210 100644 --- a/include/openmc/settings.h +++ b/include/openmc/settings.h @@ -59,7 +59,6 @@ extern std::string path_cross_sections; //!< path to cross_sections.xml extern std::string path_input; //!< directory where main .xml files resides extern std::string path_output; //!< directory where output files are written extern std::string path_particle_restart; //!< path to a particle restart file -extern std::string path_source; extern std::string path_source_library; //!< path to the source shared object extern std::string path_sourcepoint; //!< path to a source file extern "C" std::string path_statepoint; //!< path to a statepoint file diff --git a/include/openmc/source.h b/include/openmc/source.h index 529cb1836e..e47c918516 100644 --- a/include/openmc/source.h +++ b/include/openmc/source.h @@ -57,6 +57,7 @@ private: UPtrSpace space_; //!< Spatial distribution UPtrAngle angle_; //!< Angular distribution UPtrDist energy_; //!< Energy distribution + std::vector sites_; //!< Source sites from a file }; class CustomSource { diff --git a/include/openmc/state_point.h b/include/openmc/state_point.h index 4a901ac2a2..31dffc9ee9 100644 --- a/include/openmc/state_point.h +++ b/include/openmc/state_point.h @@ -2,17 +2,19 @@ #define OPENMC_STATE_POINT_H #include +#include #include "hdf5.h" #include "openmc/capi.h" +#include "openmc/particle.h" namespace openmc { void load_state_point(); void write_source_point(const char* filename); void write_source_bank(hid_t group_id); -void read_source_bank(hid_t group_id); +void read_source_bank(hid_t group_id, std::vector& sites); void write_tally_results_nr(hid_t file_id); void restart_set_keff(); diff --git a/src/settings.cpp b/src/settings.cpp index 8e402b7110..cab7c6c11b 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -73,7 +73,6 @@ std::string path_cross_sections; std::string path_input; std::string path_output; std::string path_particle_restart; -std::string path_source; std::string path_source_library; std::string path_sourcepoint; std::string path_statepoint; diff --git a/src/source.cpp b/src/source.cpp index 4f4ad117bb..91a5f78cc3 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -82,13 +82,35 @@ SourceDistribution::SourceDistribution(pugi::xml_node node) // Check for external source file if (check_for_node(node, "file")) { // Copy path of source file - settings::path_source = get_node_value(node, "file", false, true); + auto path_source = get_node_value(node, "file", false, true); // Check if source file exists - if (!file_exists(settings::path_source)) { - fatal_error(fmt::format("Source file '{}' does not exist.", - settings::path_source)); + if (!file_exists(path_source)) { + fatal_error(fmt::format("Source file '{}' does not exist.", path_source)); } + + // Read the source from a binary file instead of sampling from some + // assumed source distribution + write_message(6, "Reading source file from {}...", path_source); + + // Open the binary file + hid_t file_id = file_open(path_source, 'r', true); + + // Read the file type + std::string filetype; + read_attribute(file_id, "filetype", filetype); + + // Check to make sure this is a source file + if (filetype != "source" && filetype != "statepoint") { + fatal_error("Specified starting source file not a source file type."); + } + + // Read in the source bank + read_source_bank(file_id, sites_); + + // Close file + file_close(file_id); + } else if (check_for_node(node, "library")) { settings::path_source_library = get_node_value(node, "library", false, true); if (!file_exists(settings::path_source_library)) { @@ -180,15 +202,20 @@ Particle::Bank SourceDistribution::sample(uint64_t* seed) const int n_reject = 0; static int n_accept = 0; while (!found) { - // Set particle type - site.particle = particle_; + if (!sites_.empty()) { + size_t i_site = sites_.size()*prn(seed); + site = sites_[i_site]; + } else { + // Set particle type + site.particle = particle_; - // Sample spatial distribution - site.r = space_->sample(seed); - double xyz[] {site.r.x, site.r.y, site.r.z}; + // Sample spatial distribution + site.r = space_->sample(seed); + } // Now search to see if location exists in geometry int32_t cell_index, instance; + double xyz[] {site.r.x, site.r.y, site.r.z}; int err = openmc_find_cell(xyz, &cell_index, &instance); found = (err != OPENMC_E_GEOMETRY); @@ -225,6 +252,8 @@ Particle::Bank SourceDistribution::sample(uint64_t* seed) const // Increment number of accepted samples ++n_accept; + if (!sites_.empty()) return site; + // Sample angle site.u = angle_->sample(seed); @@ -264,33 +293,8 @@ void initialize_source() { write_message("Initializing source particles...", 5); - if (!settings::path_source.empty()) { - // Read the source from a binary file instead of sampling from some - // assumed source distribution - - write_message(6, "Reading source file from {}...", settings::path_source); - - // Open the binary file - hid_t file_id = file_open(settings::path_source, 'r', true); - - // Read the file type - std::string filetype; - read_attribute(file_id, "filetype", filetype); - - // Check to make sure this is a source file - if (filetype != "source" && filetype != "statepoint") { - fatal_error("Specified starting source file not a source file type."); - } - - // Read in the source bank - read_source_bank(file_id); - - // Close file - file_close(file_id); - } else if (!settings::path_source_library.empty()) { - - write_message(6, "Sampling library source {}...", settings::path_source); - + if (!settings::path_source_library.empty()) { + write_message(6, "Sampling library source {}...", settings::path_source_library); fill_source_bank_custom_source(); } else { diff --git a/src/state_point.cpp b/src/state_point.cpp index e3484d1a25..873d32a410 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -485,11 +485,11 @@ void load_state_point() + "...", 5); // Open source file - file_id = file_open(settings::path_source.c_str(), 'r', true); + file_id = file_open(settings::path_sourcepoint.c_str(), 'r', true); } // Read source - read_source_bank(file_id); + read_source_bank(file_id, simulation::source_bank); } @@ -653,7 +653,7 @@ write_source_bank(hid_t group_id) } -void read_source_bank(hid_t group_id) +void read_source_bank(hid_t group_id, std::vector& sites) { hid_t banktype = h5banktype(); @@ -666,11 +666,15 @@ void read_source_bank(hid_t group_id) // Make sure source bank is big enough hid_t dspace = H5Dget_space(dset); - hsize_t dims_all[1]; - H5Sget_simple_extent_dims(dspace, dims_all, nullptr); - if (simulation::work_index[mpi::n_procs] > dims_all[0]) { - fatal_error("Number of source sites in source file is less " - "than number of source particles per generation."); + hsize_t dims_all; + H5Sget_simple_extent_dims(dspace, &dims_all, nullptr); + if (&sites == &simulation::source_bank) { + if (simulation::work_index[mpi::n_procs] > dims_all) { + fatal_error("Number of source sites in source file is less " + "than number of source particles per generation."); + } + } else { + sites.resize(dims_all); } // Select hyperslab for each process @@ -681,10 +685,10 @@ void read_source_bank(hid_t group_id) // Read data in parallel hid_t plist = H5Pcreate(H5P_DATASET_XFER); H5Pset_dxpl_mpio(plist, H5FD_MPIO_COLLECTIVE); - H5Dread(dset, banktype, memspace, dspace, plist, simulation::source_bank.data()); + H5Dread(dset, banktype, memspace, dspace, plist, sites.data()); H5Pclose(plist); #else - H5Dread(dset, banktype, memspace, dspace, H5P_DEFAULT, simulation::source_bank.data()); + H5Dread(dset, banktype, memspace, dspace, H5P_DEFAULT, sites.data()); #endif // Close all ids From 713469bf59143e9b8fae7dfb3e4bf4f1e86b3769 Mon Sep 17 00:00:00 2001 From: Paul Romano Date: Thu, 22 Oct 2020 15:05:43 -0500 Subject: [PATCH 02/10] Make regular source distributions obey the same interface as custom sources --- examples/custom_source/source_ring.cpp | 2 +- .../parameterized_source_ring.cpp | 3 +- include/openmc/source.h | 30 +++++++++++-------- src/settings.cpp | 7 ++--- src/source.cpp | 22 +++++++------- src/tallies/tally.cpp | 2 +- .../source_dlopen/source_sampling.cpp | 6 ++-- .../parameterized_source_sampling.cpp | 2 +- 8 files changed, 38 insertions(+), 36 deletions(-) diff --git a/examples/custom_source/source_ring.cpp b/examples/custom_source/source_ring.cpp index bc8a8a7f8d..a23250db12 100644 --- a/examples/custom_source/source_ring.cpp +++ b/examples/custom_source/source_ring.cpp @@ -5,7 +5,7 @@ #include "openmc/source.h" #include "openmc/particle.h" -class Source : public openmc::CustomSource +class Source : public openmc::SourceDistribution { openmc::Particle::Bank sample(uint64_t* seed) { diff --git a/examples/parameterized_custom_source/parameterized_source_ring.cpp b/examples/parameterized_custom_source/parameterized_source_ring.cpp index c88333e15b..8668fd426d 100644 --- a/examples/parameterized_custom_source/parameterized_source_ring.cpp +++ b/examples/parameterized_custom_source/parameterized_source_ring.cpp @@ -6,8 +6,7 @@ #include "openmc/source.h" #include "openmc/particle.h" -class Source : public openmc::CustomSource -{ +class Source : public openmc::SourceDistribution { public: Source(double radius, double energy) : radius_(radius), energy_(energy) { } diff --git a/include/openmc/source.h b/include/openmc/source.h index e47c918516..f1fe40a391 100644 --- a/include/openmc/source.h +++ b/include/openmc/source.h @@ -23,7 +23,7 @@ class SourceDistribution; namespace model { -extern std::vector external_sources; +extern std::vector> external_sources; } // namespace model @@ -32,19 +32,30 @@ extern std::vector external_sources; //============================================================================== class SourceDistribution { +public: + virtual ~SourceDistribution() = default; + + // Methods that must be implemented + virtual Particle::Bank sample(uint64_t* seed) = 0; + + // Methods that can be overridden + virtual double strength() const { return 1.0; } +}; + +class IndependentSourceDistribution : public SourceDistribution { public: // Constructors - SourceDistribution(UPtrSpace space, UPtrAngle angle, UPtrDist energy); - explicit SourceDistribution(pugi::xml_node node); + IndependentSourceDistribution(UPtrSpace space, UPtrAngle angle, UPtrDist energy); + explicit IndependentSourceDistribution(pugi::xml_node node); //! Sample from the external source distribution //! \param[inout] seed Pseudorandom seed pointer //! \return Sampled site - Particle::Bank sample(uint64_t* seed) const; + Particle::Bank sample(uint64_t* seed) override; // Properties Particle::Type particle_type() const { return particle_; } - double strength() const { return strength_; } + double strength() const override { return strength_; } // Make observing pointers available SpatialDistribution* space() const { return space_.get(); } @@ -60,14 +71,7 @@ private: std::vector sites_; //!< Source sites from a file }; -class CustomSource { - public: - virtual ~CustomSource() {} - - virtual Particle::Bank sample(uint64_t* seed) = 0; -}; - -typedef std::unique_ptr create_custom_source_t(std::string parameters); +typedef std::unique_ptr create_custom_source_t(std::string parameters); //============================================================================== // Functions diff --git a/src/settings.cpp b/src/settings.cpp index cab7c6c11b..8b0cd53008 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -420,17 +420,16 @@ void read_settings_xml() // Get point to list of elements and make sure there is at least one for (pugi::xml_node node : root.children("source")) { - model::external_sources.emplace_back(node); + model::external_sources.push_back(std::make_unique(node)); } // If no source specified, default to isotropic point source at origin with Watt spectrum if (model::external_sources.empty()) { - SourceDistribution source { + model::external_sources.push_back(std::make_unique( UPtrSpace{new SpatialPoint({0.0, 0.0, 0.0})}, UPtrAngle{new Isotropic()}, UPtrDist{new Watt(0.988e6, 2.249e-6)} - }; - model::external_sources.push_back(std::move(source)); + )); } // Check if we want to write out source diff --git a/src/source.cpp b/src/source.cpp index 91a5f78cc3..dc75a0c818 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -39,7 +39,7 @@ namespace openmc { namespace model { -std::vector external_sources; +std::vector> external_sources; } @@ -47,7 +47,7 @@ namespace { void* custom_source_library; std::string custom_source_parameters; -std::unique_ptr custom_source; +std::unique_ptr custom_source; } @@ -56,10 +56,10 @@ std::unique_ptr custom_source; // SourceDistribution implementation //============================================================================== -SourceDistribution::SourceDistribution(UPtrSpace space, UPtrAngle angle, UPtrDist energy) +IndependentSourceDistribution::IndependentSourceDistribution(UPtrSpace space, UPtrAngle angle, UPtrDist energy) : space_{std::move(space)}, angle_{std::move(angle)}, energy_{std::move(energy)} { } -SourceDistribution::SourceDistribution(pugi::xml_node node) +IndependentSourceDistribution::IndependentSourceDistribution(pugi::xml_node node) { // Check for particle type if (check_for_node(node, "particle")) { @@ -190,7 +190,7 @@ SourceDistribution::SourceDistribution(pugi::xml_node node) } -Particle::Bank SourceDistribution::sample(uint64_t* seed) const +Particle::Bank IndependentSourceDistribution::sample(uint64_t* seed) { Particle::Bank site; @@ -331,7 +331,7 @@ Particle::Bank sample_external_source(uint64_t* seed) // Determine total source strength double total_strength = 0.0; for (auto& s : model::external_sources) - total_strength += s.strength(); + total_strength += s->strength(); // Sample from among multiple source distributions int i = 0; @@ -339,13 +339,13 @@ Particle::Bank sample_external_source(uint64_t* seed) double xi = prn(seed)*total_strength; double c = 0.0; for (; i < model::external_sources.size(); ++i) { - c += model::external_sources[i].strength(); + c += model::external_sources[i]->strength(); if (xi < c) break; } } // Sample source site from i-th source distribution - Particle::Bank site {model::external_sources[i].sample(seed)}; + Particle::Bank site {model::external_sources[i]->sample(seed)}; // If running in MG, convert site.E to group if (!settings::run_CE) { @@ -375,7 +375,7 @@ void load_custom_source_library() // reset errors dlerror(); - // get the function to create the CustomSource from the library + // get the function to create the custom source from the library auto create_custom_source = reinterpret_cast( dlsym(custom_source_library, "openmc_create_source")); @@ -387,7 +387,7 @@ void load_custom_source_library() fatal_error(error_msg); } - // create a pointer to an instance of the CustomSource + // create a pointer to an instance of the custom source custom_source = create_custom_source(custom_source_parameters); #else @@ -413,7 +413,7 @@ void close_custom_source_library() Particle::Bank sample_custom_source_library(uint64_t* seed) { - // sample from the instance of the CustomSource + // sample from the instance of the custom source return custom_source->sample(seed); } diff --git a/src/tallies/tally.cpp b/src/tallies/tally.cpp index d5cc7dfb6f..038bcec649 100644 --- a/src/tallies/tally.cpp +++ b/src/tallies/tally.cpp @@ -632,7 +632,7 @@ void Tally::accumulate() double total_source = 0.0; if (settings::run_mode == RunMode::FIXED_SOURCE) { for (const auto& s : model::external_sources) { - total_source += s.strength(); + total_source += s->strength(); } } else { total_source = 1.0; diff --git a/tests/regression_tests/source_dlopen/source_sampling.cpp b/tests/regression_tests/source_dlopen/source_sampling.cpp index ea74afdd97..4425af5a9d 100644 --- a/tests/regression_tests/source_dlopen/source_sampling.cpp +++ b/tests/regression_tests/source_dlopen/source_sampling.cpp @@ -5,7 +5,7 @@ #include "openmc/source.h" #include "openmc/particle.h" -class Source : openmc::CustomSource +class Source : public openmc::SourceDistribution { openmc::Particle::Bank sample(uint64_t *seed) { @@ -14,7 +14,7 @@ class Source : openmc::CustomSource particle.particle = openmc::Particle::Type::neutron; particle.wgt = 1.0; // position - + particle.r.x = 0.; particle.r.y = 0.; particle.r.z = 0.; @@ -22,7 +22,7 @@ class Source : openmc::CustomSource particle.u = {1.0, 0.0, 0.0}; particle.E = 14.08e6; particle.delayed_group = 0; - return particle; + return particle; } }; diff --git a/tests/regression_tests/source_parameterized_dlopen/parameterized_source_sampling.cpp b/tests/regression_tests/source_parameterized_dlopen/parameterized_source_sampling.cpp index 3b0875bb03..3f9e876398 100644 --- a/tests/regression_tests/source_parameterized_dlopen/parameterized_source_sampling.cpp +++ b/tests/regression_tests/source_parameterized_dlopen/parameterized_source_sampling.cpp @@ -1,7 +1,7 @@ #include "openmc/source.h" #include "openmc/particle.h" -class Source : public openmc::CustomSource { +class Source : public openmc::SourceDistribution { public: Source(double energy) : energy_(energy) { } From 3f62fab56558dc1f2fbe135841ccd6220bb78ee0 Mon Sep 17 00:00:00 2001 From: Paul Romano Date: Thu, 22 Oct 2020 15:50:30 -0500 Subject: [PATCH 03/10] Add wrapper class for custom sources --- include/openmc/source.h | 34 +++++--- src/settings.cpp | 14 +++- src/simulation.cpp | 12 --- src/source.cpp | 177 ++++++++++++++-------------------------- 4 files changed, 95 insertions(+), 142 deletions(-) diff --git a/include/openmc/source.h b/include/openmc/source.h index f1fe40a391..59f1151f9e 100644 --- a/include/openmc/source.h +++ b/include/openmc/source.h @@ -71,6 +71,28 @@ private: std::vector sites_; //!< Source sites from a file }; +//============================================================================== +//! Wrapper for custom sources that manages opening/closing shared library +//============================================================================== + +class CustomSourceWrapper : public SourceDistribution { +public: + // Constructors, destructors + CustomSourceWrapper(std::string path, std::string parameters); + ~CustomSourceWrapper(); + + // Defer implementation to custom source library + Particle::Bank sample(uint64_t* seed) override + { + return custom_source_->sample(seed); + } + + double strength() const override { return custom_source_->strength(); } +private: + void* shared_library_; //!< library from dlopen + std::unique_ptr custom_source_; +}; + typedef std::unique_ptr create_custom_source_t(std::string parameters); //============================================================================== @@ -86,18 +108,6 @@ extern "C" void initialize_source(); //! \return Sampled source site Particle::Bank sample_external_source(uint64_t* seed); -//! Sample a site from custom source library -Particle::Bank sample_custom_source_library(uint64_t* seed); - -//! Load custom source library -void load_custom_source_library(); - -//! Release custom source library -void close_custom_source_library(); - -//! Fill source bank at the end of a generation for dlopen based source simulation -void fill_source_bank_custom_source(); - void free_memory_source(); } // namespace openmc diff --git a/src/settings.cpp b/src/settings.cpp index 8b0cd53008..55dc04aa69 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -420,7 +420,19 @@ void read_settings_xml() // Get point to list of elements and make sure there is at least one for (pugi::xml_node node : root.children("source")) { - model::external_sources.push_back(std::make_unique(node)); + if (check_for_node(node, "library")) { + // Get shared library path and parameters + auto path = get_node_value(node, "library", false, true); + std::string parameters; + if (check_for_node(node, "parameters")) { + parameters = get_node_value(node, "parameters", false, true); + } + + // Create custom source + model::external_sources.push_back(std::make_unique(path, parameters)); + } else { + model::external_sources.push_back(std::make_unique(node)); + } } // If no source specified, default to isotropic point source at origin with Watt spectrum diff --git a/src/simulation.cpp b/src/simulation.cpp index c95c0386c0..1e35adeeea 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -121,12 +121,6 @@ int openmc_simulation_init() } } - // If fixed source and using custom source library then need to load - if (settings::run_mode == RunMode::FIXED_SOURCE && - !settings::path_source_library.empty()) { - load_custom_source_library(); - } - // Display header if (mpi::master) { if (settings::run_mode == RunMode::FIXED_SOURCE) { @@ -173,12 +167,6 @@ int openmc_simulation_finalize() t->active_ = false; } - // If fixed source and using custom source library then need to close - if (settings::run_mode == RunMode::FIXED_SOURCE && - !settings::path_source_library.empty()) { - close_custom_source_library(); - } - // Stop timers and show timing statistics simulation::time_finalize.stop(); simulation::time_total.stop(); diff --git a/src/source.cpp b/src/source.cpp index dc75a0c818..e19ccc832c 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -43,17 +43,8 @@ std::vector> external_sources; } -namespace { - -void* custom_source_library; -std::string custom_source_parameters; -std::unique_ptr custom_source; - -} - - //============================================================================== -// SourceDistribution implementation +// IndependentSourceDistribution implementation //============================================================================== IndependentSourceDistribution::IndependentSourceDistribution(UPtrSpace space, UPtrAngle angle, UPtrDist energy) @@ -111,16 +102,6 @@ IndependentSourceDistribution::IndependentSourceDistribution(pugi::xml_node node // Close file file_close(file_id); - } else if (check_for_node(node, "library")) { - settings::path_source_library = get_node_value(node, "library", false, true); - if (!file_exists(settings::path_source_library)) { - fatal_error(fmt::format("Source library '{}' does not exist.", - settings::path_source_library)); - } - - if (check_for_node(node, "parameters")) { - custom_source_parameters = get_node_value(node, "parameters", false, true); - } } else { // Spatial distribution for external source @@ -189,7 +170,6 @@ IndependentSourceDistribution::IndependentSourceDistribution(pugi::xml_node node } } - Particle::Bank IndependentSourceDistribution::sample(uint64_t* seed) { Particle::Bank site; @@ -285,6 +265,56 @@ Particle::Bank IndependentSourceDistribution::sample(uint64_t* seed) return site; } +//============================================================================== +// CustomSourceWrapper implementation +//============================================================================== + +CustomSourceWrapper::CustomSourceWrapper(std::string path, std::string parameters) +{ +#ifdef HAS_DYNAMIC_LINKING + // Open the library + shared_library_ = dlopen(path.c_str(), RTLD_LAZY); + if (!shared_library_) { + fatal_error("Couldn't open source library " + path); + } + + // reset errors + dlerror(); + + // get the function to create the custom source from the library + auto create_custom_source = reinterpret_cast( + dlsym(shared_library_, "openmc_create_source")); + + // check for any dlsym errors + auto dlsym_error = dlerror(); + if (dlsym_error) { + std::string error_msg = fmt::format("Couldn't open the openmc_create_source symbol: {}", dlsym_error); + dlclose(shared_library_); + fatal_error(error_msg); + } + + // create a pointer to an instance of the custom source + custom_source_ = create_custom_source(parameters); + +#else + fatal_error("Custom source libraries have not yet been implemented for " + "non-POSIX systems"); +#endif +} + +CustomSourceWrapper::~CustomSourceWrapper() +{ + // Make sure custom source is cleared before closing shared library + if (custom_source_.get()) custom_source_.reset(); + +#ifdef HAS_DYNAMIC_LINKING + dlclose(shared_library_); +#else + fatal_error("Custom source libraries have not yet been implemented for " + "non-POSIX systems"); +#endif +} + //============================================================================== // Non-member functions //============================================================================== @@ -293,22 +323,16 @@ void initialize_source() { write_message("Initializing source particles...", 5); - if (!settings::path_source_library.empty()) { - write_message(6, "Sampling library source {}...", settings::path_source_library); - fill_source_bank_custom_source(); + // Generation source sites from specified distribution in user input + #pragma omp parallel for + for (int64_t i = 0; i < simulation::work_per_rank; ++i) { + // initialize random number seed + int64_t id = simulation::total_gen*settings::n_particles + + simulation::work_index[mpi::rank] + i + 1; + uint64_t seed = init_seed(id, STREAM_SOURCE); - } else { - // Generation source sites from specified distribution in user input - #pragma omp parallel for - for (int64_t i = 0; i < simulation::work_per_rank; ++i) { - // initialize random number seed - int64_t id = simulation::total_gen*settings::n_particles + - simulation::work_index[mpi::rank] + i + 1; - uint64_t seed = init_seed(id, STREAM_SOURCE); - - // sample external source distribution - simulation::source_bank[i] = sample_external_source(&seed); - } + // sample external source distribution + simulation::source_bank[i] = sample_external_source(&seed); } // Write out initial source @@ -323,11 +347,6 @@ void initialize_source() Particle::Bank sample_external_source(uint64_t* seed) { - // return values from custom source if using - if (!settings::path_source_library.empty()) { - return sample_custom_source_library(seed); - } - // Determine total source strength double total_strength = 0.0; for (auto& s : model::external_sources) @@ -362,80 +381,4 @@ void free_memory_source() model::external_sources.clear(); } -void load_custom_source_library() -{ -#ifdef HAS_DYNAMIC_LINKING - - // Open the library - custom_source_library = dlopen(settings::path_source_library.c_str(), RTLD_LAZY); - if (!custom_source_library) { - fatal_error("Couldn't open source library " + settings::path_source_library); - } - - // reset errors - dlerror(); - - // get the function to create the custom source from the library - auto create_custom_source = reinterpret_cast( - dlsym(custom_source_library, "openmc_create_source")); - - // check for any dlsym errors - auto dlsym_error = dlerror(); - if (dlsym_error) { - std::string error_msg = fmt::format("Couldn't open the openmc_create_source symbol: {}", dlsym_error); - dlclose(custom_source_library); - fatal_error(error_msg); - } - - // create a pointer to an instance of the custom source - custom_source = create_custom_source(custom_source_parameters); - -#else - fatal_error("Custom source libraries have not yet been implemented for " - "non-POSIX systems"); -#endif -} - -void close_custom_source_library() -{ - if (custom_source.get()) { - // Make sure the custom source is destroyed before we close it's libary. - custom_source.reset(); - } - -#ifdef HAS_DYNAMIC_LINKING - dlclose(custom_source_library); -#else - fatal_error("Custom source libraries have not yet been implemented for " - "non-POSIX systems"); -#endif -} - -Particle::Bank sample_custom_source_library(uint64_t* seed) -{ - // sample from the instance of the custom source - return custom_source->sample(seed); -} - -void fill_source_bank_custom_source() -{ - // Load the custom library - load_custom_source_library(); - - // Generation source sites from specified distribution in the - // library source - for (int64_t i = 0; i < simulation::work_per_rank; ++i) { - // initialize random number seed - int64_t id = (simulation::total_gen + overall_generation()) * - settings::n_particles + simulation::work_index[mpi::rank] + i + 1; - uint64_t seed = init_seed(id, STREAM_SOURCE); - - // sample custom library source - simulation::source_bank[i] = sample_custom_source_library(&seed); - } - - // release the library - close_custom_source_library(); -} - } // namespace openmc From 5a60133d5b2cdd0ad5c66642cdfaaac02d98c490 Mon Sep 17 00:00:00 2001 From: Paul Romano Date: Fri, 23 Oct 2020 06:56:14 -0500 Subject: [PATCH 04/10] Implement SourceFile derived from SourceDistribution --- include/openmc/source.h | 12 ++++++ src/settings.cpp | 5 ++- src/source.cpp | 82 +++++++++++++++++++++-------------------- 3 files changed, 58 insertions(+), 41 deletions(-) diff --git a/include/openmc/source.h b/include/openmc/source.h index 59f1151f9e..ff4b3e04fe 100644 --- a/include/openmc/source.h +++ b/include/openmc/source.h @@ -68,6 +68,18 @@ private: UPtrSpace space_; //!< Spatial distribution UPtrAngle angle_; //!< Angular distribution UPtrDist energy_; //!< Energy distribution +}; + + +class SourceFile : public SourceDistribution { +public: + // Constructors + explicit SourceFile(std::string path); + + // Methods + Particle::Bank sample(uint64_t* seed) override; + +private: std::vector sites_; //!< Source sites from a file }; diff --git a/src/settings.cpp b/src/settings.cpp index 55dc04aa69..b692274c64 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -420,7 +420,10 @@ void read_settings_xml() // Get point to list of elements and make sure there is at least one for (pugi::xml_node node : root.children("source")) { - if (check_for_node(node, "library")) { + if (check_for_node(node, "file")) { + auto path = get_node_value(node, "file", false, true); + model::external_sources.push_back(std::make_unique(path)); + } else if (check_for_node(node, "library")) { // Get shared library path and parameters auto path = get_node_value(node, "library", false, true); std::string parameters; diff --git a/src/source.cpp b/src/source.cpp index e19ccc832c..5c96adc20c 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -72,35 +72,6 @@ IndependentSourceDistribution::IndependentSourceDistribution(pugi::xml_node node // Check for external source file if (check_for_node(node, "file")) { - // Copy path of source file - auto path_source = get_node_value(node, "file", false, true); - - // Check if source file exists - if (!file_exists(path_source)) { - fatal_error(fmt::format("Source file '{}' does not exist.", path_source)); - } - - // Read the source from a binary file instead of sampling from some - // assumed source distribution - write_message(6, "Reading source file from {}...", path_source); - - // Open the binary file - hid_t file_id = file_open(path_source, 'r', true); - - // Read the file type - std::string filetype; - read_attribute(file_id, "filetype", filetype); - - // Check to make sure this is a source file - if (filetype != "source" && filetype != "statepoint") { - fatal_error("Specified starting source file not a source file type."); - } - - // Read in the source bank - read_source_bank(file_id, sites_); - - // Close file - file_close(file_id); } else { @@ -182,16 +153,11 @@ Particle::Bank IndependentSourceDistribution::sample(uint64_t* seed) int n_reject = 0; static int n_accept = 0; while (!found) { - if (!sites_.empty()) { - size_t i_site = sites_.size()*prn(seed); - site = sites_[i_site]; - } else { - // Set particle type - site.particle = particle_; + // Set particle type + site.particle = particle_; - // Sample spatial distribution - site.r = space_->sample(seed); - } + // Sample spatial distribution + site.r = space_->sample(seed); // Now search to see if location exists in geometry int32_t cell_index, instance; @@ -232,8 +198,6 @@ Particle::Bank IndependentSourceDistribution::sample(uint64_t* seed) // Increment number of accepted samples ++n_accept; - if (!sites_.empty()) return site; - // Sample angle site.u = angle_->sample(seed); @@ -265,6 +229,44 @@ Particle::Bank IndependentSourceDistribution::sample(uint64_t* seed) return site; } +//============================================================================== +// SourceFile implementation +//============================================================================== + +SourceFile::SourceFile(std::string path) +{ + // Check if source file exists + if (!file_exists(path)) { + fatal_error(fmt::format("Source file '{}' does not exist.", path)); + } + + // Read the source from a binary file instead of sampling from some + // assumed source distribution + write_message(6, "Reading source file from {}...", path); + + // Open the binary file + hid_t file_id = file_open(path, 'r', true); + + // Check to make sure this is a source file + std::string filetype; + read_attribute(file_id, "filetype", filetype); + if (filetype != "source" && filetype != "statepoint") { + fatal_error("Specified starting source file not a source file type."); + } + + // Read in the source bank + read_source_bank(file_id, sites_); + + // Close file + file_close(file_id); +} + +Particle::Bank SourceFile::sample(uint64_t* seed) +{ + size_t i_site = sites_.size()*prn(seed); + return sites_[i_site]; +} + //============================================================================== // CustomSourceWrapper implementation //============================================================================== From f2526ec4391a4dc1496d576a1b2db268f6831c22 Mon Sep 17 00:00:00 2001 From: Paul Romano Date: Fri, 23 Oct 2020 13:26:02 -0500 Subject: [PATCH 05/10] Generalize source bank reading (don't rely on work_index) --- src/state_point.cpp | 33 +++++++++---------- .../source_file/results_true.dat | 2 +- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index 873d32a410..6fcc900a64 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -659,27 +659,26 @@ void read_source_bank(hid_t group_id, std::vector& sites) // Open the dataset hid_t dset = H5Dopen(group_id, "source_bank", H5P_DEFAULT); - - // Create another data space but for each proc individually - hsize_t dims[] {static_cast(simulation::work_per_rank)}; - hid_t memspace = H5Screate_simple(1, dims, nullptr); + hid_t dspace = H5Dget_space(dset); + hsize_t n_sites; + H5Sget_simple_extent_dims(dspace, &n_sites, nullptr); // Make sure source bank is big enough - hid_t dspace = H5Dget_space(dset); - hsize_t dims_all; - H5Sget_simple_extent_dims(dspace, &dims_all, nullptr); - if (&sites == &simulation::source_bank) { - if (simulation::work_index[mpi::n_procs] > dims_all) { - fatal_error("Number of source sites in source file is less " - "than number of source particles per generation."); - } - } else { - sites.resize(dims_all); - } + sites.resize(n_sites); + + // Determine number of source particles to read from each rank and offset + hsize_t min_sites = n_sites / mpi::n_procs; + hsize_t remainder = n_sites % mpi::n_procs; + hsize_t n_sites_local = (mpi::rank < remainder) ? min_sites + 1 : min_sites; + hsize_t offset = (mpi::rank <= remainder) ? + mpi::rank*(min_sites + 1) : + remainder*(min_sites + 1) + (mpi::rank - remainder)*min_sites; + + // Create another data space but for each proc individually + hid_t memspace = H5Screate_simple(1, &n_sites_local, nullptr); // Select hyperslab for each process - hsize_t start[] {static_cast(simulation::work_index[mpi::rank])}; - H5Sselect_hyperslab(dspace, H5S_SELECT_SET, start, nullptr, dims, nullptr); + H5Sselect_hyperslab(dspace, H5S_SELECT_SET, &offset, nullptr, &n_sites_local, nullptr); #ifdef PHDF5 // Read data in parallel diff --git a/tests/regression_tests/source_file/results_true.dat b/tests/regression_tests/source_file/results_true.dat index 4ab7993d88..44c422eeff 100644 --- a/tests/regression_tests/source_file/results_true.dat +++ b/tests/regression_tests/source_file/results_true.dat @@ -1,2 +1,2 @@ k-combined: -2.939525E-01 6.311780E-03 +3.080726E-01 9.554124E-03 From 0bb39005bcf7f9aad4d333ede0715c249229094e Mon Sep 17 00:00:00 2001 From: Paul Romano Date: Fri, 23 Oct 2020 13:45:28 -0500 Subject: [PATCH 06/10] Rename source classes --- examples/custom_source/source_ring.cpp | 6 ++-- .../parameterized_source_ring.cpp | 12 +++---- include/openmc/settings.h | 1 - include/openmc/source.h | 33 +++++++++++-------- src/settings.cpp | 7 ++-- src/source.cpp | 14 ++++---- .../source_dlopen/source_sampling.cpp | 6 ++-- .../parameterized_source_sampling.cpp | 8 ++--- 8 files changed, 46 insertions(+), 41 deletions(-) diff --git a/examples/custom_source/source_ring.cpp b/examples/custom_source/source_ring.cpp index a23250db12..7ff3b03b46 100644 --- a/examples/custom_source/source_ring.cpp +++ b/examples/custom_source/source_ring.cpp @@ -5,7 +5,7 @@ #include "openmc/source.h" #include "openmc/particle.h" -class Source : public openmc::SourceDistribution +class RingSource : public openmc::Source { openmc::Particle::Bank sample(uint64_t* seed) { @@ -30,7 +30,7 @@ class Source : public openmc::SourceDistribution // A function to create a unique pointer to an instance of this class when generated // via a plugin call using dlopen/dlsym. // You must have external C linkage here otherwise dlopen will not find the file -extern "C" std::unique_ptr openmc_create_source(std::string parameters) +extern "C" std::unique_ptr openmc_create_source(std::string parameters) { - return std::make_unique(); + return std::make_unique(); } diff --git a/examples/parameterized_custom_source/parameterized_source_ring.cpp b/examples/parameterized_custom_source/parameterized_source_ring.cpp index 8668fd426d..2176a073ae 100644 --- a/examples/parameterized_custom_source/parameterized_source_ring.cpp +++ b/examples/parameterized_custom_source/parameterized_source_ring.cpp @@ -6,13 +6,13 @@ #include "openmc/source.h" #include "openmc/particle.h" -class Source : public openmc::SourceDistribution { +class RingSource : public openmc::Source { public: - Source(double radius, double energy) : radius_(radius), energy_(energy) { } + RingSource(double radius, double energy) : radius_(radius), energy_(energy) { } // Defines a function that can create a unique pointer to a new instance of this class // by extracting the parameters from the provided string. - static std::unique_ptr from_string(std::string parameters) + static std::unique_ptr from_string(std::string parameters) { std::unordered_map parameter_mapping; @@ -27,7 +27,7 @@ class Source : public openmc::SourceDistribution { double radius = std::stod(parameter_mapping["radius"]); double energy = std::stod(parameter_mapping["energy"]); - return std::make_unique(radius, energy); + return std::make_unique(radius, energy); } // Samples from an instance of this class. @@ -59,7 +59,7 @@ class Source : public openmc::SourceDistribution { // A function to create a unique pointer to an instance of this class when generated // via a plugin call using dlopen/dlsym. // You must have external C linkage here otherwise dlopen will not find the file -extern "C" std::unique_ptr openmc_create_source(std::string parameters) +extern "C" std::unique_ptr openmc_create_source(std::string parameters) { - return Source::from_string(parameters); + return RingSource::from_string(parameters); } diff --git a/include/openmc/settings.h b/include/openmc/settings.h index c743cf7210..74cbbc1328 100644 --- a/include/openmc/settings.h +++ b/include/openmc/settings.h @@ -59,7 +59,6 @@ extern std::string path_cross_sections; //!< path to cross_sections.xml extern std::string path_input; //!< directory where main .xml files resides extern std::string path_output; //!< directory where output files are written extern std::string path_particle_restart; //!< path to a particle restart file -extern std::string path_source_library; //!< path to the source shared object extern std::string path_sourcepoint; //!< path to a source file extern "C" std::string path_statepoint; //!< path to a statepoint file diff --git a/include/openmc/source.h b/include/openmc/source.h index ff4b3e04fe..93e6bcf545 100644 --- a/include/openmc/source.h +++ b/include/openmc/source.h @@ -19,21 +19,21 @@ namespace openmc { // Global variables //============================================================================== -class SourceDistribution; +class Source; namespace model { -extern std::vector> external_sources; +extern std::vector> external_sources; } // namespace model //============================================================================== -//! External source distribution +//! Abstract source interface //============================================================================== -class SourceDistribution { +class Source { public: - virtual ~SourceDistribution() = default; + virtual ~Source() = default; // Methods that must be implemented virtual Particle::Bank sample(uint64_t* seed) = 0; @@ -42,11 +42,15 @@ public: virtual double strength() const { return 1.0; } }; -class IndependentSourceDistribution : public SourceDistribution { +//============================================================================== +//! Source composed of independent spatial, angle, and energy distributions +//============================================================================== + +class IndependentSource : public Source { public: // Constructors - IndependentSourceDistribution(UPtrSpace space, UPtrAngle angle, UPtrDist energy); - explicit IndependentSourceDistribution(pugi::xml_node node); + IndependentSource(UPtrSpace space, UPtrAngle angle, UPtrDist energy); + explicit IndependentSource(pugi::xml_node node); //! Sample from the external source distribution //! \param[inout] seed Pseudorandom seed pointer @@ -70,11 +74,14 @@ private: UPtrDist energy_; //!< Energy distribution }; +//============================================================================== +//! Source composed of particles read from a file +//============================================================================== -class SourceFile : public SourceDistribution { +class FileSource : public Source { public: // Constructors - explicit SourceFile(std::string path); + explicit FileSource(std::string path); // Methods Particle::Bank sample(uint64_t* seed) override; @@ -87,7 +94,7 @@ private: //! Wrapper for custom sources that manages opening/closing shared library //============================================================================== -class CustomSourceWrapper : public SourceDistribution { +class CustomSourceWrapper : public Source { public: // Constructors, destructors CustomSourceWrapper(std::string path, std::string parameters); @@ -102,10 +109,10 @@ public: double strength() const override { return custom_source_->strength(); } private: void* shared_library_; //!< library from dlopen - std::unique_ptr custom_source_; + std::unique_ptr custom_source_; }; -typedef std::unique_ptr create_custom_source_t(std::string parameters); +typedef std::unique_ptr create_custom_source_t(std::string parameters); //============================================================================== // Functions diff --git a/src/settings.cpp b/src/settings.cpp index b692274c64..5a868d445b 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -73,7 +73,6 @@ std::string path_cross_sections; std::string path_input; std::string path_output; std::string path_particle_restart; -std::string path_source_library; std::string path_sourcepoint; std::string path_statepoint; @@ -422,7 +421,7 @@ void read_settings_xml() for (pugi::xml_node node : root.children("source")) { if (check_for_node(node, "file")) { auto path = get_node_value(node, "file", false, true); - model::external_sources.push_back(std::make_unique(path)); + model::external_sources.push_back(std::make_unique(path)); } else if (check_for_node(node, "library")) { // Get shared library path and parameters auto path = get_node_value(node, "library", false, true); @@ -434,13 +433,13 @@ void read_settings_xml() // Create custom source model::external_sources.push_back(std::make_unique(path, parameters)); } else { - model::external_sources.push_back(std::make_unique(node)); + model::external_sources.push_back(std::make_unique(node)); } } // If no source specified, default to isotropic point source at origin with Watt spectrum if (model::external_sources.empty()) { - model::external_sources.push_back(std::make_unique( + model::external_sources.push_back(std::make_unique( UPtrSpace{new SpatialPoint({0.0, 0.0, 0.0})}, UPtrAngle{new Isotropic()}, UPtrDist{new Watt(0.988e6, 2.249e-6)} diff --git a/src/source.cpp b/src/source.cpp index 5c96adc20c..fd2d6cd8f5 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -39,18 +39,18 @@ namespace openmc { namespace model { -std::vector> external_sources; +std::vector> external_sources; } //============================================================================== -// IndependentSourceDistribution implementation +// IndependentSource implementation //============================================================================== -IndependentSourceDistribution::IndependentSourceDistribution(UPtrSpace space, UPtrAngle angle, UPtrDist energy) +IndependentSource::IndependentSource(UPtrSpace space, UPtrAngle angle, UPtrDist energy) : space_{std::move(space)}, angle_{std::move(angle)}, energy_{std::move(energy)} { } -IndependentSourceDistribution::IndependentSourceDistribution(pugi::xml_node node) +IndependentSource::IndependentSource(pugi::xml_node node) { // Check for particle type if (check_for_node(node, "particle")) { @@ -141,7 +141,7 @@ IndependentSourceDistribution::IndependentSourceDistribution(pugi::xml_node node } } -Particle::Bank IndependentSourceDistribution::sample(uint64_t* seed) +Particle::Bank IndependentSource::sample(uint64_t* seed) { Particle::Bank site; @@ -233,7 +233,7 @@ Particle::Bank IndependentSourceDistribution::sample(uint64_t* seed) // SourceFile implementation //============================================================================== -SourceFile::SourceFile(std::string path) +FileSource::FileSource(std::string path) { // Check if source file exists if (!file_exists(path)) { @@ -261,7 +261,7 @@ SourceFile::SourceFile(std::string path) file_close(file_id); } -Particle::Bank SourceFile::sample(uint64_t* seed) +Particle::Bank FileSource::sample(uint64_t* seed) { size_t i_site = sites_.size()*prn(seed); return sites_[i_site]; diff --git a/tests/regression_tests/source_dlopen/source_sampling.cpp b/tests/regression_tests/source_dlopen/source_sampling.cpp index 4425af5a9d..796b504f3c 100644 --- a/tests/regression_tests/source_dlopen/source_sampling.cpp +++ b/tests/regression_tests/source_dlopen/source_sampling.cpp @@ -5,7 +5,7 @@ #include "openmc/source.h" #include "openmc/particle.h" -class Source : public openmc::SourceDistribution +class CustomSource : public openmc::Source { openmc::Particle::Bank sample(uint64_t *seed) { @@ -29,7 +29,7 @@ class Source : public openmc::SourceDistribution // A function to create a unique pointer to an instance of this class when generated // via a plugin call using dlopen/dlsym. // You must have external C linkage here otherwise dlopen will not find the file -extern "C" std::unique_ptr openmc_create_source(std::string parameters) +extern "C" std::unique_ptr openmc_create_source(std::string parameters) { - return std::make_unique(); + return std::make_unique(); } diff --git a/tests/regression_tests/source_parameterized_dlopen/parameterized_source_sampling.cpp b/tests/regression_tests/source_parameterized_dlopen/parameterized_source_sampling.cpp index 3f9e876398..91c165dbc4 100644 --- a/tests/regression_tests/source_parameterized_dlopen/parameterized_source_sampling.cpp +++ b/tests/regression_tests/source_parameterized_dlopen/parameterized_source_sampling.cpp @@ -1,9 +1,9 @@ #include "openmc/source.h" #include "openmc/particle.h" -class Source : public openmc::SourceDistribution { +class CustomSource : public openmc::Source { public: - Source(double energy) : energy_(energy) { } + CustomSource(double energy) : energy_(energy) { } // Samples from an instance of this class. openmc::Particle::Bank sample(uint64_t* seed) @@ -31,8 +31,8 @@ class Source : public openmc::SourceDistribution { // A function to create a unique pointer to an instance of this class when generated // via a plugin call using dlopen/dlsym. // You must have external C linkage here otherwise dlopen will not find the file -extern "C" std::unique_ptr openmc_create_source(std::string parameter) +extern "C" std::unique_ptr openmc_create_source(std::string parameter) { double energy = std::stod(parameter); - return std::make_unique(energy); + return std::make_unique(energy); } From 33b7dcfe22a268ae22773888f50eddcc22ea0031 Mon Sep 17 00:00:00 2001 From: Paul Romano Date: Fri, 23 Oct 2020 13:46:57 -0500 Subject: [PATCH 07/10] Update documentation regarding custom sources --- docs/source/io_formats/settings.rst | 2 +- docs/source/usersguide/settings.rst | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/source/io_formats/settings.rst b/docs/source/io_formats/settings.rst index 09d7db1236..588b58b478 100644 --- a/docs/source/io_formats/settings.rst +++ b/docs/source/io_formats/settings.rst @@ -465,7 +465,7 @@ attributes/sub-elements: as complex as is required to define the source for your problem. The library has a few basic requirements: - * It must contain a class that inherits from ``openmc::CustomSource``; + * It must contain a class that inherits from ``openmc::Source``; * The class must implement a function called ``sample()``; * There must be an ``openmc_create_source()`` function that creates the source as a unique pointer. This function can be used to pass parameters through to diff --git a/docs/source/usersguide/settings.rst b/docs/source/usersguide/settings.rst index 340d4af00c..1d14038749 100644 --- a/docs/source/usersguide/settings.rst +++ b/docs/source/usersguide/settings.rst @@ -194,7 +194,7 @@ below. #include "openmc/source.h" #include "openmc/particle.h" - class Source : public openmc::CustomSource + class CustomSource : public openmc::Source { openmc::Particle::Bank sample(uint64_t* seed) { @@ -216,16 +216,16 @@ below. } }; - extern "C" std::unique_ptr openmc_create_source(std::string parameters) + extern "C" std::unique_ptr openmc_create_source(std::string parameters) { - return std::make_unique(); + return std::make_unique(); } The above source creates monodirectional 14.08 MeV neutrons that are distributed in a ring with a 3 cm radius. This routine is not particularly complex, but should serve as an example upon which to build more complicated sources. - .. note:: The source class must inherit from ``openmc::CustomSource`` and + .. note:: The source class must inherit from ``openmc::Source`` and implement a ``sample()`` function. .. note:: The ``openmc_create_source()`` function signature must be declared @@ -266,9 +266,9 @@ the source class when it is created: #include "openmc/source.h" #include "openmc/particle.h" - class Source : public openmc::CustomSource { + class CustomSource : public openmc::Source { public: - Source(double energy) : energy_{energy} { } + CustomSource(double energy) : energy_{energy} { } // Samples from an instance of this class. openmc::Particle::Bank sample(uint64_t* seed) @@ -293,9 +293,9 @@ the source class when it is created: double energy_; }; - extern "C" std::unique_ptr openmc_create_source(std::string parameter) { + extern "C" std::unique_ptr openmc_create_source(std::string parameter) { double energy = std::stod(parameter); - return std::make_unique(energy); + return std::make_unique(energy); } As with the basic custom source functionality, the custom source library From b271399f638eef3919af1bef5024c9be6d016add Mon Sep 17 00:00:00 2001 From: Paul Romano Date: Mon, 26 Oct 2020 16:54:40 -0500 Subject: [PATCH 08/10] Allow read_source_bank to be distributed or not --- include/openmc/state_point.h | 2 +- src/source.cpp | 4 ++-- src/state_point.cpp | 37 ++++++++++++++++++++---------------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/include/openmc/state_point.h b/include/openmc/state_point.h index 31dffc9ee9..c50ca355b8 100644 --- a/include/openmc/state_point.h +++ b/include/openmc/state_point.h @@ -14,7 +14,7 @@ namespace openmc { void load_state_point(); void write_source_point(const char* filename); void write_source_bank(hid_t group_id); -void read_source_bank(hid_t group_id, std::vector& sites); +void read_source_bank(hid_t group_id, std::vector& sites, bool distribute); void write_tally_results_nr(hid_t file_id); void restart_set_keff(); diff --git a/src/source.cpp b/src/source.cpp index fd2d6cd8f5..a7bb807478 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -254,8 +254,8 @@ FileSource::FileSource(std::string path) fatal_error("Specified starting source file not a source file type."); } - // Read in the source bank - read_source_bank(file_id, sites_); + // Read in the source particles + read_source_bank(file_id, sites_, false); // Close file file_close(file_id); diff --git a/src/state_point.cpp b/src/state_point.cpp index 6fcc900a64..66706a676f 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -489,7 +489,7 @@ void load_state_point() } // Read source - read_source_bank(file_id, simulation::source_bank); + read_source_bank(file_id, simulation::source_bank, true); } @@ -653,7 +653,7 @@ write_source_bank(hid_t group_id) } -void read_source_bank(hid_t group_id, std::vector& sites) +void read_source_bank(hid_t group_id, std::vector& sites, bool distribute) { hid_t banktype = h5banktype(); @@ -663,22 +663,27 @@ void read_source_bank(hid_t group_id, std::vector& sites) hsize_t n_sites; H5Sget_simple_extent_dims(dspace, &n_sites, nullptr); - // Make sure source bank is big enough - sites.resize(n_sites); + // Make sure vector is big enough in case where we're reading entire source on + // each process + if (!distribute) sites.resize(n_sites); - // Determine number of source particles to read from each rank and offset - hsize_t min_sites = n_sites / mpi::n_procs; - hsize_t remainder = n_sites % mpi::n_procs; - hsize_t n_sites_local = (mpi::rank < remainder) ? min_sites + 1 : min_sites; - hsize_t offset = (mpi::rank <= remainder) ? - mpi::rank*(min_sites + 1) : - remainder*(min_sites + 1) + (mpi::rank - remainder)*min_sites; + hid_t memspace; + if (distribute) { + if (simulation::work_index[mpi::n_procs] > n_sites) { + fatal_error("Number of source sites in source file is less " + "than number of source particles per generation."); + } - // Create another data space but for each proc individually - hid_t memspace = H5Screate_simple(1, &n_sites_local, nullptr); + // Create another data space but for each proc individually + hsize_t n_sites_local = simulation::work_per_rank; + memspace = H5Screate_simple(1, &n_sites_local, nullptr); - // Select hyperslab for each process - H5Sselect_hyperslab(dspace, H5S_SELECT_SET, &offset, nullptr, &n_sites_local, nullptr); + // Select hyperslab for each process + hsize_t offset = simulation::work_index[mpi::rank]; + H5Sselect_hyperslab(dspace, H5S_SELECT_SET, &offset, nullptr, &n_sites_local, nullptr); + } else { + memspace = H5S_ALL; + } #ifdef PHDF5 // Read data in parallel @@ -692,7 +697,7 @@ void read_source_bank(hid_t group_id, std::vector& sites) // Close all ids H5Sclose(dspace); - H5Sclose(memspace); + if (distribute) H5Sclose(memspace); H5Dclose(dset); H5Tclose(banktype); } From 9af5541e88764e639b2f6bc5d02b8d3f168bd1cd Mon Sep 17 00:00:00 2001 From: Paul Romano Date: Mon, 26 Oct 2020 17:00:26 -0500 Subject: [PATCH 09/10] Make Source::sample() method const --- docs/source/usersguide/settings.rst | 4 ++-- examples/custom_source/source_ring.cpp | 2 +- .../parameterized_source_ring.cpp | 2 +- include/openmc/source.h | 8 ++++---- src/source.cpp | 4 ++-- tests/regression_tests/source_dlopen/source_sampling.cpp | 2 +- .../parameterized_source_sampling.cpp | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/source/usersguide/settings.rst b/docs/source/usersguide/settings.rst index 1d14038749..df22328e3a 100644 --- a/docs/source/usersguide/settings.rst +++ b/docs/source/usersguide/settings.rst @@ -196,7 +196,7 @@ below. class CustomSource : public openmc::Source { - openmc::Particle::Bank sample(uint64_t* seed) + openmc::Particle::Bank sample(uint64_t* seed) const { openmc::Particle::Bank particle; // weight @@ -271,7 +271,7 @@ the source class when it is created: CustomSource(double energy) : energy_{energy} { } // Samples from an instance of this class. - openmc::Particle::Bank sample(uint64_t* seed) + openmc::Particle::Bank sample(uint64_t* seed) const { openmc::Particle::Bank particle; // weight diff --git a/examples/custom_source/source_ring.cpp b/examples/custom_source/source_ring.cpp index 7ff3b03b46..a752216dce 100644 --- a/examples/custom_source/source_ring.cpp +++ b/examples/custom_source/source_ring.cpp @@ -7,7 +7,7 @@ class RingSource : public openmc::Source { - openmc::Particle::Bank sample(uint64_t* seed) + openmc::Particle::Bank sample(uint64_t* seed) const { openmc::Particle::Bank particle; // wgt diff --git a/examples/parameterized_custom_source/parameterized_source_ring.cpp b/examples/parameterized_custom_source/parameterized_source_ring.cpp index 2176a073ae..9f913dfee3 100644 --- a/examples/parameterized_custom_source/parameterized_source_ring.cpp +++ b/examples/parameterized_custom_source/parameterized_source_ring.cpp @@ -31,7 +31,7 @@ class RingSource : public openmc::Source { } // Samples from an instance of this class. - openmc::Particle::Bank sample(uint64_t* seed) + openmc::Particle::Bank sample(uint64_t* seed) const { openmc::Particle::Bank particle; // wgt diff --git a/include/openmc/source.h b/include/openmc/source.h index 93e6bcf545..12e8156992 100644 --- a/include/openmc/source.h +++ b/include/openmc/source.h @@ -36,7 +36,7 @@ public: virtual ~Source() = default; // Methods that must be implemented - virtual Particle::Bank sample(uint64_t* seed) = 0; + virtual Particle::Bank sample(uint64_t* seed) const = 0; // Methods that can be overridden virtual double strength() const { return 1.0; } @@ -55,7 +55,7 @@ public: //! Sample from the external source distribution //! \param[inout] seed Pseudorandom seed pointer //! \return Sampled site - Particle::Bank sample(uint64_t* seed) override; + Particle::Bank sample(uint64_t* seed) const override; // Properties Particle::Type particle_type() const { return particle_; } @@ -84,7 +84,7 @@ public: explicit FileSource(std::string path); // Methods - Particle::Bank sample(uint64_t* seed) override; + Particle::Bank sample(uint64_t* seed) const override; private: std::vector sites_; //!< Source sites from a file @@ -101,7 +101,7 @@ public: ~CustomSourceWrapper(); // Defer implementation to custom source library - Particle::Bank sample(uint64_t* seed) override + Particle::Bank sample(uint64_t* seed) const override { return custom_source_->sample(seed); } diff --git a/src/source.cpp b/src/source.cpp index a7bb807478..ff83031472 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -141,7 +141,7 @@ IndependentSource::IndependentSource(pugi::xml_node node) } } -Particle::Bank IndependentSource::sample(uint64_t* seed) +Particle::Bank IndependentSource::sample(uint64_t* seed) const { Particle::Bank site; @@ -261,7 +261,7 @@ FileSource::FileSource(std::string path) file_close(file_id); } -Particle::Bank FileSource::sample(uint64_t* seed) +Particle::Bank FileSource::sample(uint64_t* seed) const { size_t i_site = sites_.size()*prn(seed); return sites_[i_site]; diff --git a/tests/regression_tests/source_dlopen/source_sampling.cpp b/tests/regression_tests/source_dlopen/source_sampling.cpp index 796b504f3c..8eaf6265cc 100644 --- a/tests/regression_tests/source_dlopen/source_sampling.cpp +++ b/tests/regression_tests/source_dlopen/source_sampling.cpp @@ -7,7 +7,7 @@ class CustomSource : public openmc::Source { - openmc::Particle::Bank sample(uint64_t *seed) + openmc::Particle::Bank sample(uint64_t *seed) const { openmc::Particle::Bank particle; // wgt diff --git a/tests/regression_tests/source_parameterized_dlopen/parameterized_source_sampling.cpp b/tests/regression_tests/source_parameterized_dlopen/parameterized_source_sampling.cpp index 91c165dbc4..da7d70e378 100644 --- a/tests/regression_tests/source_parameterized_dlopen/parameterized_source_sampling.cpp +++ b/tests/regression_tests/source_parameterized_dlopen/parameterized_source_sampling.cpp @@ -6,7 +6,7 @@ class CustomSource : public openmc::Source { CustomSource(double energy) : energy_(energy) { } // Samples from an instance of this class. - openmc::Particle::Bank sample(uint64_t* seed) + openmc::Particle::Bank sample(uint64_t* seed) const { openmc::Particle::Bank particle; // wgt From b57ee1f757fe37e80540a5a9537c73f24f32bd63 Mon Sep 17 00:00:00 2001 From: Paul Romano Date: Tue, 27 Oct 2020 16:03:11 -0500 Subject: [PATCH 10/10] Fix class name in comment in source.cpp Co-authored-by: DanShort12 <66615563+DanShort12@users.noreply.github.com> --- src/source.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/source.cpp b/src/source.cpp index ff83031472..87644f054d 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -230,7 +230,7 @@ Particle::Bank IndependentSource::sample(uint64_t* seed) const } //============================================================================== -// SourceFile implementation +// FileSource implementation //============================================================================== FileSource::FileSource(std::string path)