From 50a9a0ac3bd2e68ee43ae28170162044f7fe3325 Mon Sep 17 00:00:00 2001 From: davidjohnlong Date: Wed, 4 Mar 2020 12:14:27 +0000 Subject: [PATCH 01/20] Modification to load_state_point() so source bank existence in statepoint files is only checked for EIGENVALUE restart cases as the new fixed source bank model will never write source particles to file --- src/state_point.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index d077590365..0115817683 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -380,15 +380,14 @@ void load_state_point() // Read batch number to restart at read_dataset(file_id, "current_batch", simulation::restart_batch); - // Check for source in statepoint if needed - bool source_present; - read_attribute(file_id, "source_present", source_present); - if (simulation::restart_batch > settings::n_batches) { fatal_error("The number batches specified in settings.xml is fewer " " than the number of batches in the given statepoint file."); } + // Logical flag for source present in statepoint file + bool source_present; + // Read information specific to eigenvalue run if (settings::run_mode == RunMode::EIGENVALUE) { read_dataset(file_id, "n_inactive", temp); @@ -396,6 +395,13 @@ void load_state_point() // Take maximum of statepoint n_inactive and input n_inactive settings::n_inactive = std::max(settings::n_inactive, temp); + + // Check to make sure source bank is present + read_attribute(file_id, "source_present", source_present); + if (settings::path_sourcepoint == settings::path_statepoint && + !source_present) { + fatal_error("Source bank must be contained in statepoint restart file"); + } } // Read number of realizations for global tallies @@ -406,12 +412,6 @@ void load_state_point() restart_set_keff(); simulation::current_batch = simulation::restart_batch; - // Check to make sure source bank is present - if (settings::path_sourcepoint == settings::path_statepoint && - !source_present) { - fatal_error("Source bank must be contained in statepoint restart file"); - } - // Read tallies to master. If we are using Parallel HDF5, all processes // need to be included in the HDF5 calls. #ifdef PHDF5 From 0ae1077a39a146fa6c4bb65bdd901d203076a0a5 Mon Sep 17 00:00:00 2001 From: davidjohnlong Date: Thu, 5 Mar 2020 10:48:36 +0000 Subject: [PATCH 02/20] Implement the new fixed source bank model. For a fixed source calcultion the primary source bank is not allocated memory and source particles can not be read/written to files. The source particles for each history are now sampled at the start of initialize_history() through call to sample_external_source() with the appropriate seed. The timing stats for sampling has been removed from the output since this processing is now part of the transport timer --- include/openmc/particle.h | 11 ++++++++ src/output.cpp | 2 -- src/particle.cpp | 16 +++++++---- src/simulation.cpp | 58 +++++++++++++++++++++++---------------- src/source.cpp | 17 ------------ src/state_point.cpp | 8 ++++-- 6 files changed, 62 insertions(+), 50 deletions(-) diff --git a/include/openmc/particle.h b/include/openmc/particle.h index 2917b45e4a..f5f43affbc 100644 --- a/include/openmc/particle.h +++ b/include/openmc/particle.h @@ -181,6 +181,15 @@ public: int delayed_group; //!< particle delayed group }; + //! Saved ("banked") state of primary source particle, for particle restart + struct RestartBank { + Position r; + Direction u; + double E; + double wgt; + }; + + //========================================================================== // Constructors @@ -361,6 +370,8 @@ public: #endif int64_t n_progeny_ {0}; // Number of progeny produced by this particle + + RestartBank primary_source_; // primary source particle data }; } // namespace openmc diff --git a/src/output.cpp b/src/output.cpp index 18b94be9be..8da32de55a 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -441,8 +441,6 @@ void print_runtime() show_time("Time synchronizing fission bank", time_bank.elapsed(), 1); show_time("Sampling source sites", time_bank_sample.elapsed(), 2); show_time("SEND/RECV source sites", time_bank_sendrecv.elapsed(), 2); - } else { - show_time("Time sampling source", time_sample_source.elapsed(), 1); } show_time("Time accumulating tallies", time_tallies.elapsed(), 1); show_time("Total time for finalization", time_finalize.elapsed()); diff --git a/src/particle.cpp b/src/particle.cpp index 5f9713941e..376ee35cdb 100644 --- a/src/particle.cpp +++ b/src/particle.cpp @@ -128,6 +128,13 @@ Particle::from_source(const Bank* src) E_ = data::mg.energy_bin_avg_[g_]; } E_last_ = E_; + + // Saved copy of primary source attributes + primary_source_.r = src->r; + primary_source_.u = src->u; + primary_source_.E = E_last_; + primary_source_.wgt = wgt_; + } void @@ -681,11 +688,10 @@ Particle::write_restart() const write_dataset(file_id, "id", id_); write_dataset(file_id, "type", static_cast(type_)); - int64_t i = current_work_; - write_dataset(file_id, "weight", simulation::source_bank[i-1].wgt); - write_dataset(file_id, "energy", simulation::source_bank[i-1].E); - write_dataset(file_id, "xyz", simulation::source_bank[i-1].r); - write_dataset(file_id, "uvw", simulation::source_bank[i-1].u); + write_dataset(file_id, "weight", primary_source_.wgt); + write_dataset(file_id, "energy", primary_source_.E); + write_dataset(file_id, "xyz", primary_source_.r); + write_dataset(file_id, "uvw", primary_source_.u); // Close file file_close(file_id); diff --git a/src/simulation.cpp b/src/simulation.cpp index 086d740593..2176e7d5cc 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -69,9 +69,10 @@ int openmc_simulation_init() // Determine how much work each process should do calculate_work(); - // Allocate source bank, and for eigenvalue simulations also allocate the - // fission bank - allocate_banks(); + // Allocate source and fission banks for eigenvalue simulations + if (settings::run_mode == RunMode::EIGENVALUE) { + allocate_banks(); + } // If doing an event-based simulation, intialize the particle buffer // and event queues @@ -105,7 +106,10 @@ int openmc_simulation_init() load_state_point(); write_message("Resuming simulation...", 6); } else { - initialize_source(); + //Only initialize primary source bank for eigenvalue simulations + if (settings::run_mode == RunMode::EIGENVALUE) { + initialize_source(); + } } // Display header @@ -271,10 +275,8 @@ void allocate_banks() { // Allocate source bank simulation::source_bank.resize(simulation::work_per_rank); - - if (settings::run_mode == RunMode::EIGENVALUE) { - init_fission_bank(3*simulation::work_per_rank); - } + // Allocate fission bank + init_fission_bank(3*simulation::work_per_rank); } void initialize_batch() @@ -350,7 +352,7 @@ void finalize_batch() && !settings::cmfd_run) { if (contains(settings::sourcepoint_batch, simulation::current_batch) && settings::source_write && !settings::source_separate) { - bool b = true; + bool b = (settings::run_mode == RunMode::EIGENVALUE) ? true : false; openmc_statepoint_write(nullptr, &b); } else { bool b = false; @@ -358,16 +360,18 @@ void finalize_batch() } } - // Write out a separate source point if it's been specified for this batch - if (contains(settings::sourcepoint_batch, simulation::current_batch) - && settings::source_write && settings::source_separate) { - write_source_point(nullptr); - } + if (settings::run_mode == RunMode::EIGENVALUE) { + // Write out a separate source point if it's been specified for this batch + if (contains(settings::sourcepoint_batch, simulation::current_batch) + && settings::source_write && settings::source_separate) { + write_source_point(nullptr); + } - // Write a continously-overwritten source point if requested. - if (settings::source_latest) { - auto filename = settings::path_output + "source.h5"; - write_source_point(filename.c_str()); + // Write a continously-overwritten source point if requested. + if (settings::source_latest) { + auto filename = settings::path_output + "source.h5"; + write_source_point(filename.c_str()); + } } } @@ -429,18 +433,24 @@ void finalize_generation() } } - } else if (settings::run_mode == RunMode::FIXED_SOURCE) { - // For fixed-source mode, we need to sample the external source - simulation::time_sample_source.start(); - fill_source_bank_fixedsource(); - simulation::time_sample_source.stop(); } } void initialize_history(Particle* p, int64_t index_source) { // set defaults - p->from_source(&simulation::source_bank[index_source - 1]); + if(settings::run_mode == RunMode::FIXED_SOURCE) { + // initialize random number seed + int64_t id = (simulation::total_gen + overall_generation() - 1)*settings::n_particles + + simulation::work_index[mpi::rank] + index_source; + uint64_t seed = init_seed(id, STREAM_SOURCE); + // sample external source distribution then set + Particle::Bank site = sample_external_source(&seed); + p->from_source(&site); + } else if (settings::run_mode == RunMode::EIGENVALUE) { + // set defaults for eigenvalue simulations from primary bank + p->from_source(&simulation::source_bank[index_source - 1]); + } p->current_work_ = index_source; // set identifier for particle diff --git a/src/source.cpp b/src/source.cpp index 96175804ce..c0ee6a8536 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -396,21 +396,4 @@ void fill_source_bank_custom_source() #endif } -void fill_source_bank_fixedsource() -{ - if (settings::path_source.empty() && settings::path_source_library.empty()) { - for (int64_t i = 0; i < simulation::work_per_rank; ++i) { - // initialize random number seed - int64_t id = (simulation::total_gen + overall_generation()) * - settings::n_particles + simulation::work_index[mpi::rank] + i + 1; - uint64_t seed = init_seed(id, STREAM_SOURCE); - - // sample external source distribution - simulation::source_bank[i] = sample_external_source(&seed); - } - } else if (settings::path_source.empty() && !settings::path_source_library.empty()) { - fill_source_bank_custom_source(); - } -} - } // namespace openmc diff --git a/src/state_point.cpp b/src/state_point.cpp index 0115817683..a3a05197eb 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -387,6 +387,7 @@ void load_state_point() // Logical flag for source present in statepoint file bool source_present; + read_attribute(file_id, "source_present", source_present); // Read information specific to eigenvalue run if (settings::run_mode == RunMode::EIGENVALUE) { @@ -397,7 +398,6 @@ void load_state_point() settings::n_inactive = std::max(settings::n_inactive, temp); // Check to make sure source bank is present - read_attribute(file_id, "source_present", source_present); if (settings::path_sourcepoint == settings::path_statepoint && !source_present) { fatal_error("Source bank must be contained in statepoint restart file"); @@ -409,7 +409,11 @@ void load_state_point() // Set k_sum, keff, and current_batch based on whether restart file is part // of active cycle or inactive cycle - restart_set_keff(); + if (settings::run_mode == RunMode::EIGENVALUE) { + restart_set_keff(); + } + + // Set current batch number simulation::current_batch = simulation::restart_batch; // Read tallies to master. If we are using Parallel HDF5, all processes From 907459d02f6802faa827c3a5e10af124bdd55eda Mon Sep 17 00:00:00 2001 From: davidjohnlong Date: Fri, 6 Mar 2020 14:00:47 +0000 Subject: [PATCH 03/20] Implement new fixed source bank model for custom library source. Additional non-member functions in source.cpp with function pointer saved in model namespace allow custom library to be opened, sampled as needed in initialize_history and closed. --- include/openmc/source.h | 7 +++-- src/simulation.cpp | 21 +++++++++++-- src/source.cpp | 67 +++++++++++++++++++++++++---------------- 3 files changed, 65 insertions(+), 30 deletions(-) diff --git a/include/openmc/source.h b/include/openmc/source.h index 4db17f241a..34183e0b02 100644 --- a/include/openmc/source.h +++ b/include/openmc/source.h @@ -65,8 +65,11 @@ extern "C" void initialize_source(); //! \return Sampled source site Particle::Bank sample_external_source(uint64_t* seed); -//! Fill source bank at end of generation for fixed source simulations -void fill_source_bank_fixedsource(); +//! Sample a site from custom source library +Particle::Bank sample_custom_source_library(uint64_t* seed); + +void load_custom_source_library(); +void close_custom_source_library(); //! Fill source bank at the end of a generation for dlopen based source simulation void fill_source_bank_custom_source(); diff --git a/src/simulation.cpp b/src/simulation.cpp index 2176e7d5cc..4115e8a345 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -112,6 +112,12 @@ int openmc_simulation_init() } } + // If fixed source and using custom source library then need to load + if (settings::run_mode == RunMode::FIXED_SOURCE && + !settings::path_source_library.empty()) { + load_custom_source_library(); + } + // Display header if (mpi::master) { if (settings::run_mode == RunMode::FIXED_SOURCE) { @@ -158,6 +164,12 @@ int openmc_simulation_finalize() t->active_ = false; } + // If fixed source and using custom source library then need to close + if (settings::run_mode == RunMode::FIXED_SOURCE && + !settings::path_source_library.empty()) { + close_custom_source_library(); + } + // Stop timers and show timing statistics simulation::time_finalize.stop(); simulation::time_total.stop(); @@ -444,8 +456,13 @@ void initialize_history(Particle* p, int64_t index_source) int64_t id = (simulation::total_gen + overall_generation() - 1)*settings::n_particles + simulation::work_index[mpi::rank] + index_source; uint64_t seed = init_seed(id, STREAM_SOURCE); - // sample external source distribution then set - Particle::Bank site = sample_external_source(&seed); + // sample from external source distribution or custom library then set + Particle::Bank site; + if (!settings::path_source_library.empty()) { + site = sample_custom_source_library(&seed); + } else { + site = sample_external_source(&seed); + } p->from_source(&site); } else if (settings::run_mode == RunMode::EIGENVALUE) { // set defaults for eigenvalue simulations from primary bank diff --git a/src/source.cpp b/src/source.cpp index c0ee6a8536..9f97c829e1 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -38,6 +38,10 @@ namespace openmc { namespace model { +typedef Particle::Bank (*sample_t)(uint64_t &seed); +sample_t sample_source; +void* source_library; + std::vector external_sources; } @@ -276,25 +280,10 @@ void initialize_source() file_close(file_id); } else if (!settings::path_source_library.empty()) { - #ifdef HAS_DYNAMIC_LINKING - // Get the source from a library object write_message(fmt::format("Sampling from library source {}...", settings::path_source), 6); - // Open the library - auto source_library = dlopen(settings::path_source_library.c_str(), RTLD_LAZY); - if (!source_library) { - fatal_error("Couldn't open source library " + settings::path_source_library); - } - - // reset errors - dlerror(); - fill_source_bank_custom_source(); - #else - fatal_error("Custom source libraries have not yet been implemented for " - "non-POSIX systems"); - #endif } else { // Generation source sites from specified distribution in user input @@ -355,13 +344,14 @@ void free_memory_source() model::external_sources.clear(); } -// fill the source bank from the external source -void fill_source_bank_custom_source() +//Load custom source library +void load_custom_source_library() { #ifdef HAS_DYNAMIC_LINKING + // Open the library - auto source_library = dlopen(settings::path_source_library.c_str(), RTLD_LAZY); - if (!source_library) { + model::source_library = dlopen(settings::path_source_library.c_str(), RTLD_LAZY); + if (!model::source_library) { fatal_error("Couldn't open source library " + settings::path_source_library); } @@ -369,16 +359,42 @@ void fill_source_bank_custom_source() dlerror(); // get the function from the library - using sample_t = Particle::Bank (*)(uint64_t* seed); - auto sample_source = reinterpret_cast(dlsym(source_library, "sample_source")); + //using sample_t = Particle::Bank (*)(uint64_t* seed); + model::sample_source = reinterpret_cast(dlsym(model::source_library, "sample_source")); // check for any dlsym errors auto dlsym_error = dlerror(); if (dlsym_error) { - dlclose(source_library); + dlclose(model::source_library); fatal_error(fmt::format("Couldn't open the sample_source symbol: {}", dlsym_error)); } +#else + fatal_error("Custom source libraries have not yet been implemented for " + "non-POSIX systems"); +#endif + +} + +//Release custom source library +void close_custom_source_library() +{ + dlclose(model::source_library); +} + +//Sample source particle from custom library +Particle::Bank sample_custom_source_library(uint64_t* seed) +{ + Particle::Bank site = model::sample_source(*seed); + return site; +} + +// fill the source bank from the external source +void fill_source_bank_custom_source() +{ + // Load the custom library + load_custom_source_library(); + // Generation source sites from specified distribution in the // library source for (int64_t i = 0; i < simulation::work_per_rank; ++i) { @@ -387,13 +403,12 @@ void fill_source_bank_custom_source() settings::n_particles + simulation::work_index[mpi::rank] + i + 1; uint64_t seed = init_seed(id, STREAM_SOURCE); - // sample external source distribution - simulation::source_bank[i] = sample_source(&seed); + // sample custom library source + simulation::source_bank[i] = sample_custom_source_library(&seed); } // release the library - dlclose(source_library); -#endif + close_custom_source_library(); } } // namespace openmc From 3cc7ae4618bc3795641170677dcbeec2a20423b6 Mon Sep 17 00:00:00 2001 From: davidjohnlong Date: Tue, 10 Mar 2020 09:22:31 +0000 Subject: [PATCH 04/20] Use correct style guide formatting for indentation in modifications --- include/openmc/particle.h | 8 ++++---- src/simulation.cpp | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/include/openmc/particle.h b/include/openmc/particle.h index f5f43affbc..4b72e96c0f 100644 --- a/include/openmc/particle.h +++ b/include/openmc/particle.h @@ -183,10 +183,10 @@ public: //! Saved ("banked") state of primary source particle, for particle restart struct RestartBank { - Position r; - Direction u; - double E; - double wgt; + Position r; + Direction u; + double E; + double wgt; }; diff --git a/src/simulation.cpp b/src/simulation.cpp index 4115e8a345..c56e22681f 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -106,16 +106,16 @@ int openmc_simulation_init() load_state_point(); write_message("Resuming simulation...", 6); } else { - //Only initialize primary source bank for eigenvalue simulations - if (settings::run_mode == RunMode::EIGENVALUE) { + //Only initialize primary source bank for eigenvalue simulations + if (settings::run_mode == RunMode::EIGENVALUE) { initialize_source(); - } + } } // If fixed source and using custom source library then need to load if (settings::run_mode == RunMode::FIXED_SOURCE && - !settings::path_source_library.empty()) { - load_custom_source_library(); + !settings::path_source_library.empty()) { + load_custom_source_library(); } // Display header From 7e5a6945e3c190f5c892dbb2e3c56494755c01b8 Mon Sep 17 00:00:00 2001 From: davidjohnlong <59730407+davidjohnlong@users.noreply.github.com> Date: Tue, 10 Mar 2020 11:56:20 +0000 Subject: [PATCH 05/20] Correct formatting in state_point.cpp Co-Authored-By: Simon Richards --- src/state_point.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index a3a05197eb..c8c3c87bd8 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -399,8 +399,8 @@ void load_state_point() // Check to make sure source bank is present if (settings::path_sourcepoint == settings::path_statepoint && - !source_present) { - fatal_error("Source bank must be contained in statepoint restart file"); + !source_present) { + fatal_error("Source bank must be contained in statepoint restart file"); } } From 315c6dfc9ea05f571a1be9324bc62df4989db252 Mon Sep 17 00:00:00 2001 From: davidjohnlong <59730407+davidjohnlong@users.noreply.github.com> Date: Tue, 10 Mar 2020 11:56:52 +0000 Subject: [PATCH 06/20] Correct formatting in include/openmc/particle.h Co-Authored-By: Simon Richards --- include/openmc/particle.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/openmc/particle.h b/include/openmc/particle.h index 4b72e96c0f..bef2b19840 100644 --- a/include/openmc/particle.h +++ b/include/openmc/particle.h @@ -186,7 +186,7 @@ public: Position r; Direction u; double E; - double wgt; + double wgt; }; From 8556b8b6811d95c0b6fe76b41770b12fb7dc5619 Mon Sep 17 00:00:00 2001 From: davidjohnlong Date: Tue, 10 Mar 2020 12:07:01 +0000 Subject: [PATCH 07/20] Further correction to formatting --- src/simulation.cpp | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/simulation.cpp b/src/simulation.cpp index c56e22681f..6a5ec073da 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -108,7 +108,7 @@ int openmc_simulation_init() } else { //Only initialize primary source bank for eigenvalue simulations if (settings::run_mode == RunMode::EIGENVALUE) { - initialize_source(); + initialize_source(); } } @@ -167,7 +167,7 @@ int openmc_simulation_finalize() // If fixed source and using custom source library then need to close if (settings::run_mode == RunMode::FIXED_SOURCE && !settings::path_source_library.empty()) { - close_custom_source_library(); + close_custom_source_library(); } // Stop timers and show timing statistics @@ -452,20 +452,20 @@ void initialize_history(Particle* p, int64_t index_source) { // set defaults if(settings::run_mode == RunMode::FIXED_SOURCE) { - // initialize random number seed - int64_t id = (simulation::total_gen + overall_generation() - 1)*settings::n_particles + - simulation::work_index[mpi::rank] + index_source; - uint64_t seed = init_seed(id, STREAM_SOURCE); - // sample from external source distribution or custom library then set - Particle::Bank site; - if (!settings::path_source_library.empty()) { - site = sample_custom_source_library(&seed); - } else { - site = sample_external_source(&seed); - } - p->from_source(&site); + // initialize random number seed + int64_t id = (simulation::total_gen + overall_generation() - 1)*settings::n_particles + + simulation::work_index[mpi::rank] + index_source; + uint64_t seed = init_seed(id, STREAM_SOURCE); + // sample from external source distribution or custom library then set + Particle::Bank site; + if (!settings::path_source_library.empty()) { + site = sample_custom_source_library(&seed); + } else { + site = sample_external_source(&seed); + } + p->from_source(&site); } else if (settings::run_mode == RunMode::EIGENVALUE) { - // set defaults for eigenvalue simulations from primary bank + // set defaults for eigenvalue simulations from primary bank p->from_source(&simulation::source_bank[index_source - 1]); } p->current_work_ = index_source; From 13896a315952bc66c6fc1877fea9da1dfbd93fcd Mon Sep 17 00:00:00 2001 From: davidjohnlong <59730407+davidjohnlong@users.noreply.github.com> Date: Thu, 12 Mar 2020 07:41:33 +0000 Subject: [PATCH 08/20] Remove empty line in src/particle.cpp Co-Authored-By: Paul Romano --- src/particle.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/particle.cpp b/src/particle.cpp index 376ee35cdb..e278383cf7 100644 --- a/src/particle.cpp +++ b/src/particle.cpp @@ -134,7 +134,6 @@ Particle::from_source(const Bank* src) primary_source_.u = src->u; primary_source_.E = E_last_; primary_source_.wgt = wgt_; - } void From edb9cbc69feccd0c4c48f5eed935408bf13e8292 Mon Sep 17 00:00:00 2001 From: davidjohnlong <59730407+davidjohnlong@users.noreply.github.com> Date: Thu, 12 Mar 2020 07:42:58 +0000 Subject: [PATCH 09/20] Update formatting in src/simulation.cpp Co-Authored-By: Paul Romano --- src/simulation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation.cpp b/src/simulation.cpp index 6a5ec073da..5b3ad7aef7 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -106,7 +106,7 @@ int openmc_simulation_init() load_state_point(); write_message("Resuming simulation...", 6); } else { - //Only initialize primary source bank for eigenvalue simulations + // Only initialize primary source bank for eigenvalue simulations if (settings::run_mode == RunMode::EIGENVALUE) { initialize_source(); } From f61fad5f75198a2213a1ed3b276bac222ccfb0d9 Mon Sep 17 00:00:00 2001 From: davidjohnlong <59730407+davidjohnlong@users.noreply.github.com> Date: Thu, 12 Mar 2020 08:00:13 +0000 Subject: [PATCH 10/20] Adjust setting of boolean variable "b" Co-Authored-By: Paul Romano --- src/simulation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation.cpp b/src/simulation.cpp index 5b3ad7aef7..89a0758a1f 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -364,7 +364,7 @@ void finalize_batch() && !settings::cmfd_run) { if (contains(settings::sourcepoint_batch, simulation::current_batch) && settings::source_write && !settings::source_separate) { - bool b = (settings::run_mode == RunMode::EIGENVALUE) ? true : false; + bool b = (settings::run_mode == RunMode::EIGENVALUE); openmc_statepoint_write(nullptr, &b); } else { bool b = false; From c5aee746e81bb230e804ebcc25f1046686878416 Mon Sep 17 00:00:00 2001 From: davidjohnlong <59730407+davidjohnlong@users.noreply.github.com> Date: Thu, 12 Mar 2020 08:00:49 +0000 Subject: [PATCH 11/20] Correct formatting in src/simulation.cpp Co-Authored-By: Paul Romano --- src/simulation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation.cpp b/src/simulation.cpp index 89a0758a1f..2eeb7e3ce8 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -451,7 +451,7 @@ void finalize_generation() void initialize_history(Particle* p, int64_t index_source) { // set defaults - if(settings::run_mode == RunMode::FIXED_SOURCE) { + if (settings::run_mode == RunMode::FIXED_SOURCE) { // initialize random number seed int64_t id = (simulation::total_gen + overall_generation() - 1)*settings::n_particles + simulation::work_index[mpi::rank] + index_source; From d355347e8eb4297092043514b712c5801255d8b2 Mon Sep 17 00:00:00 2001 From: davidjohnlong <59730407+davidjohnlong@users.noreply.github.com> Date: Thu, 12 Mar 2020 08:10:49 +0000 Subject: [PATCH 12/20] Remove unnecessary variable in src/source.cpp Co-Authored-By: Paul Romano --- src/source.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/source.cpp b/src/source.cpp index 9f97c829e1..e659f31f32 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -385,8 +385,7 @@ void close_custom_source_library() //Sample source particle from custom library Particle::Bank sample_custom_source_library(uint64_t* seed) { - Particle::Bank site = model::sample_source(*seed); - return site; + return model::sample_source(*seed); } // fill the source bank from the external source From 3f7afb7b4f0c32176eb153632cc1ae725ef7a33b Mon Sep 17 00:00:00 2001 From: davidjohnlong <59730407+davidjohnlong@users.noreply.github.com> Date: Thu, 12 Mar 2020 08:14:31 +0000 Subject: [PATCH 13/20] Adjust naming of custom source function and library for clarity Co-Authored-By: Paul Romano --- src/source.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/source.cpp b/src/source.cpp index e659f31f32..a5bc91cb0b 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -39,8 +39,8 @@ namespace openmc { namespace model { typedef Particle::Bank (*sample_t)(uint64_t &seed); -sample_t sample_source; -void* source_library; +sample_t custom_source_function; +void* custom_source_library; std::vector external_sources; From cc2017786b06798307cd7bdb9472f7d07c9035fa Mon Sep 17 00:00:00 2001 From: davidjohnlong Date: Thu, 12 Mar 2020 09:29:28 +0000 Subject: [PATCH 14/20] Rename custom source library and function and move to anonymous namespace --- src/source.cpp | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/source.cpp b/src/source.cpp index a5bc91cb0b..fee3be507e 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -38,14 +38,19 @@ namespace openmc { namespace model { +std::vector external_sources; + +} + +namespace { + typedef Particle::Bank (*sample_t)(uint64_t &seed); sample_t custom_source_function; void* custom_source_library; -std::vector external_sources; - } + //============================================================================== // SourceDistribution implementation //============================================================================== @@ -350,8 +355,8 @@ void load_custom_source_library() #ifdef HAS_DYNAMIC_LINKING // Open the library - model::source_library = dlopen(settings::path_source_library.c_str(), RTLD_LAZY); - if (!model::source_library) { + custom_source_library = dlopen(settings::path_source_library.c_str(), RTLD_LAZY); + if (!custom_source_library) { fatal_error("Couldn't open source library " + settings::path_source_library); } @@ -360,12 +365,12 @@ void load_custom_source_library() // get the function from the library //using sample_t = Particle::Bank (*)(uint64_t* seed); - model::sample_source = reinterpret_cast(dlsym(model::source_library, "sample_source")); + custom_source_function = reinterpret_cast(dlsym(custom_source_library, "sample_source")); // check for any dlsym errors auto dlsym_error = dlerror(); if (dlsym_error) { - dlclose(model::source_library); + dlclose(custom_source_library); fatal_error(fmt::format("Couldn't open the sample_source symbol: {}", dlsym_error)); } @@ -379,13 +384,13 @@ void load_custom_source_library() //Release custom source library void close_custom_source_library() { - dlclose(model::source_library); + dlclose(custom_source_library); } //Sample source particle from custom library Particle::Bank sample_custom_source_library(uint64_t* seed) { - return model::sample_source(*seed); + return custom_source_function(*seed); } // fill the source bank from the external source From eac442f31c28fa490b1c3ec99b31425a4b510105 Mon Sep 17 00:00:00 2001 From: davidjohnlong Date: Thu, 12 Mar 2020 09:34:38 +0000 Subject: [PATCH 15/20] Move function description to header file --- include/openmc/source.h | 3 +++ src/source.cpp | 5 ----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/include/openmc/source.h b/include/openmc/source.h index 34183e0b02..16d91fc564 100644 --- a/include/openmc/source.h +++ b/include/openmc/source.h @@ -68,7 +68,10 @@ Particle::Bank sample_external_source(uint64_t* seed); //! Sample a site from custom source library Particle::Bank sample_custom_source_library(uint64_t* seed); +//Load custom source library void load_custom_source_library(); + +//Release custom source library void close_custom_source_library(); //! Fill source bank at the end of a generation for dlopen based source simulation diff --git a/src/source.cpp b/src/source.cpp index fee3be507e..3b51b2373f 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -349,7 +349,6 @@ void free_memory_source() model::external_sources.clear(); } -//Load custom source library void load_custom_source_library() { #ifdef HAS_DYNAMIC_LINKING @@ -378,22 +377,18 @@ void load_custom_source_library() fatal_error("Custom source libraries have not yet been implemented for " "non-POSIX systems"); #endif - } -//Release custom source library void close_custom_source_library() { dlclose(custom_source_library); } -//Sample source particle from custom library Particle::Bank sample_custom_source_library(uint64_t* seed) { return custom_source_function(*seed); } -// fill the source bank from the external source void fill_source_bank_custom_source() { // Load the custom library From 086568b5620094aebd4dc10dfd3c3e1cb6691537 Mon Sep 17 00:00:00 2001 From: davidjohnlong Date: Thu, 12 Mar 2020 12:10:55 +0000 Subject: [PATCH 16/20] Remove primary_source_ structure from particle class and sample directly from source distribution in Particle::write_restart() function for fixed source simulations. Eigenvalue simulations reverted back to indexing from primary source bank --- include/openmc/particle.h | 10 ---------- src/particle.cpp | 35 +++++++++++++++++++++++++---------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/include/openmc/particle.h b/include/openmc/particle.h index bef2b19840..69a8eb6ba4 100644 --- a/include/openmc/particle.h +++ b/include/openmc/particle.h @@ -181,15 +181,6 @@ public: int delayed_group; //!< particle delayed group }; - //! Saved ("banked") state of primary source particle, for particle restart - struct RestartBank { - Position r; - Direction u; - double E; - double wgt; - }; - - //========================================================================== // Constructors @@ -371,7 +362,6 @@ public: int64_t n_progeny_ {0}; // Number of progeny produced by this particle - RestartBank primary_source_; // primary source particle data }; } // namespace openmc diff --git a/src/particle.cpp b/src/particle.cpp index e278383cf7..3465d6e6c0 100644 --- a/src/particle.cpp +++ b/src/particle.cpp @@ -21,6 +21,7 @@ #include "openmc/physics_mg.h" #include "openmc/random_lcg.h" #include "openmc/settings.h" +#include "openmc/source.h" #include "openmc/surface.h" #include "openmc/simulation.h" #include "openmc/tallies/derivative.h" @@ -128,12 +129,6 @@ Particle::from_source(const Bank* src) E_ = data::mg.energy_bin_avg_[g_]; } E_last_ = E_; - - // Saved copy of primary source attributes - primary_source_.r = src->r; - primary_source_.u = src->u; - primary_source_.E = E_last_; - primary_source_.wgt = wgt_; } void @@ -687,10 +682,30 @@ Particle::write_restart() const write_dataset(file_id, "id", id_); write_dataset(file_id, "type", static_cast(type_)); - write_dataset(file_id, "weight", primary_source_.wgt); - write_dataset(file_id, "energy", primary_source_.E); - write_dataset(file_id, "xyz", primary_source_.r); - write_dataset(file_id, "uvw", primary_source_.u); + int64_t i = current_work_; + if (settings::run_mode == RunMode::EIGENVALUE) { + //take source data from primary bank for eigenvalue simulation + write_dataset(file_id, "weight", simulation::source_bank[i-1].wgt); + write_dataset(file_id, "energy", simulation::source_bank[i-1].E); + write_dataset(file_id, "xyz", simulation::source_bank[i-1].r); + write_dataset(file_id, "uvw", simulation::source_bank[i-1].u); + } else if (settings::run_mode == RunMode::FIXED_SOURCE) { + // re-sample using rng random number seed used to generate source particle + int64_t id = (simulation::total_gen + overall_generation() - 1)*settings::n_particles + + simulation::work_index[mpi::rank] + i; + uint64_t seed = init_seed(id, STREAM_SOURCE); + // re-sample source site + Particle::Bank site; + if (!settings::path_source_library.empty()) { + site = sample_custom_source_library(&seed); + } else { + site = sample_external_source(&seed); + } + write_dataset(file_id, "weight", site.wgt); + write_dataset(file_id, "energy", site.E); + write_dataset(file_id, "xyz", site.r); + write_dataset(file_id, "uvw", site.u); + } // Close file file_close(file_id); From 29cd47344870bfb4f01465e5cf9fea4836215523 Mon Sep 17 00:00:00 2001 From: davidjohnlong <59730407+davidjohnlong@users.noreply.github.com> Date: Thu, 12 Mar 2020 15:44:02 +0000 Subject: [PATCH 17/20] Remove line in include/openmc/particle.h Co-Authored-By: Paul Romano --- include/openmc/particle.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/openmc/particle.h b/include/openmc/particle.h index 69a8eb6ba4..2917b45e4a 100644 --- a/include/openmc/particle.h +++ b/include/openmc/particle.h @@ -361,7 +361,6 @@ public: #endif int64_t n_progeny_ {0}; // Number of progeny produced by this particle - }; } // namespace openmc From 47ea7bbcf507a86fc42c309047a439014de9943e Mon Sep 17 00:00:00 2001 From: davidjohnlong <59730407+davidjohnlong@users.noreply.github.com> Date: Thu, 12 Mar 2020 15:46:08 +0000 Subject: [PATCH 18/20] Update include/openmc/source.h Co-Authored-By: Paul Romano --- include/openmc/source.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/openmc/source.h b/include/openmc/source.h index 16d91fc564..47761ff8cb 100644 --- a/include/openmc/source.h +++ b/include/openmc/source.h @@ -68,7 +68,7 @@ Particle::Bank sample_external_source(uint64_t* seed); //! Sample a site from custom source library Particle::Bank sample_custom_source_library(uint64_t* seed); -//Load custom source library +//! Load custom source library void load_custom_source_library(); //Release custom source library From 6c188f25788013175f6a768b566c3f520ab91fad Mon Sep 17 00:00:00 2001 From: davidjohnlong <59730407+davidjohnlong@users.noreply.github.com> Date: Thu, 12 Mar 2020 15:46:23 +0000 Subject: [PATCH 19/20] Update include/openmc/source.h Co-Authored-By: Paul Romano --- include/openmc/source.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/openmc/source.h b/include/openmc/source.h index 47761ff8cb..97b18d2286 100644 --- a/include/openmc/source.h +++ b/include/openmc/source.h @@ -71,7 +71,7 @@ Particle::Bank sample_custom_source_library(uint64_t* seed); //! Load custom source library void load_custom_source_library(); -//Release custom source library +//! Release custom source library void close_custom_source_library(); //! Fill source bank at the end of a generation for dlopen based source simulation From ce135cc4a993e3be86be1246729eb4ead951e9bc Mon Sep 17 00:00:00 2001 From: davidjohnlong Date: Thu, 12 Mar 2020 17:02:22 +0000 Subject: [PATCH 20/20] Refactor fixed source sampling so call to custom library is made from call to sample_external_source() --- src/particle.cpp | 7 +------ src/simulation.cpp | 7 +------ src/source.cpp | 11 ++++++++--- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/particle.cpp b/src/particle.cpp index 3465d6e6c0..96cf3fdcf8 100644 --- a/src/particle.cpp +++ b/src/particle.cpp @@ -695,12 +695,7 @@ Particle::write_restart() const simulation::work_index[mpi::rank] + i; uint64_t seed = init_seed(id, STREAM_SOURCE); // re-sample source site - Particle::Bank site; - if (!settings::path_source_library.empty()) { - site = sample_custom_source_library(&seed); - } else { - site = sample_external_source(&seed); - } + auto site = sample_external_source(&seed); write_dataset(file_id, "weight", site.wgt); write_dataset(file_id, "energy", site.E); write_dataset(file_id, "xyz", site.r); diff --git a/src/simulation.cpp b/src/simulation.cpp index 2eeb7e3ce8..64a7d0e806 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -457,12 +457,7 @@ void initialize_history(Particle* p, int64_t index_source) simulation::work_index[mpi::rank] + index_source; uint64_t seed = init_seed(id, STREAM_SOURCE); // sample from external source distribution or custom library then set - Particle::Bank site; - if (!settings::path_source_library.empty()) { - site = sample_custom_source_library(&seed); - } else { - site = sample_external_source(&seed); - } + auto site = sample_external_source(&seed); p->from_source(&site); } else if (settings::run_mode == RunMode::EIGENVALUE) { // set defaults for eigenvalue simulations from primary bank diff --git a/src/source.cpp b/src/source.cpp index 3b51b2373f..21b12ed0dc 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -44,7 +44,7 @@ std::vector external_sources; namespace { -typedef Particle::Bank (*sample_t)(uint64_t &seed); +using sample_t = Particle::Bank (*)(uint64_t* seed); sample_t custom_source_function; void* custom_source_library; @@ -315,6 +315,11 @@ void initialize_source() Particle::Bank sample_external_source(uint64_t* seed) { + // return values from custom source if using + if (!settings::path_source_library.empty()) { + return sample_custom_source_library(seed); + } + // Determine total source strength double total_strength = 0.0; for (auto& s : model::external_sources) @@ -363,7 +368,7 @@ void load_custom_source_library() dlerror(); // get the function from the library - //using sample_t = Particle::Bank (*)(uint64_t* seed); + using sample_t = Particle::Bank (*)(uint64_t* seed); custom_source_function = reinterpret_cast(dlsym(custom_source_library, "sample_source")); // check for any dlsym errors @@ -386,7 +391,7 @@ void close_custom_source_library() Particle::Bank sample_custom_source_library(uint64_t* seed) { - return custom_source_function(*seed); + return custom_source_function(seed); } void fill_source_bank_custom_source()