From aa571c2c33b421e2524e249ca00f9584e891dd0a Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 6 Sep 2022 02:22:21 +0200 Subject: [PATCH 01/55] wip add MCPL-out to openmc code for writing a source_bank/point compiles but is untested --- include/openmc/state_point.h | 9 +++ src/state_point.cpp | 125 +++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/include/openmc/state_point.h b/include/openmc/state_point.h index 5ada2bf88d..2d92c935a6 100644 --- a/include/openmc/state_point.h +++ b/include/openmc/state_point.h @@ -9,6 +9,10 @@ #include "openmc/particle.h" #include "openmc/vector.h" +#ifdef OPENMC_MCPL +#include +#endif + namespace openmc { void load_state_point(); @@ -21,5 +25,10 @@ void write_tally_results_nr(hid_t file_id); void restart_set_keff(); void write_unstructured_mesh_results(); +#ifdef OPENMC_MCPL +void write_mcpl_source_point(const char *filename_, bool surf_source_bank = false); +void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank); +#endif + } // namespace openmc #endif // OPENMC_STATE_POINT_H diff --git a/src/state_point.cpp b/src/state_point.cpp index 9ea0459153..babd670a2d 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -28,6 +28,10 @@ #include "openmc/timer.h" #include "openmc/vector.h" +#ifdef OPENMC_MCPL +#include +#endif + namespace openmc { extern "C" int openmc_statepoint_write(const char* filename, bool* write_source) @@ -598,6 +602,45 @@ void write_source_point(const char* filename, bool surf_source_bank) file_close(file_id); } +void write_mcpl_point(const char *filename, bool surf_source_bank) +{ + std::string filename_; + if (filename) { + filename_ = filename; + } else { + // Determine width for zero padding + int w = std::to_string(settings::n_max_batches).size(); + + filename_ = fmt::format("{0}source.{1:0{2}}.mcpl", settings::path_output, + simulation::current_batch, w); + } + + mcpl_outfile_t file_id; + + std::string line; + if (mpi::master) { + // this must be rewritten: file_open is h5-specific + file_id = mcpl_create_outfile(filename_.c_str()); + //write_attribute(file_id, "filetype", "source"); + //write header stuff (oopy in xml-files as binary blobs for instance)) + if (VERSION_DEV){ + line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}-development"); + } else { + line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}"); + } + mcpl_hdr_set_srcname(file_id,line.c_str()); + } + + write_mcpl_source_bank(file_id, surf_source_bank); + + if (mpi::master) { + //change this - this is h5 specific + mcpl_closeandgzip_outfile(file_id); + } + +} + + void write_source_bank(hid_t group_id, bool surf_source_bank) { hid_t banktype = h5banktype(); @@ -714,6 +757,88 @@ void write_source_bank(hid_t group_id, bool surf_source_bank) H5Tclose(banktype); } +void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) +{ + int64_t dims_size = settings::n_particles; + int64_t count_size = simulation::work_per_rank; + + // Set vectors for source bank and starting bank index of each process + vector* bank_index = &simulation::work_index; + vector* source_bank = &simulation::source_bank; + vector surf_source_index_vector; + vector surf_source_bank_vector; + + if(surf_source_bank) { + surf_source_index_vector = calculate_surf_source_size(); + dims_size = surf_source_index_vector[mpi::n_procs]; + count_size = simulation::surf_source_bank.size(); + + bank_index = &surf_source_index_vector; + + // Copy data in a SharedArray into a vector. + surf_source_bank_vector.resize(count_size); + surf_source_bank_vector.assign(simulation::surf_source_bank.data(), + simulation::surf_source_bank.data() + count_size); + source_bank = &surf_source_bank_vector; + } + + if (mpi::master) { + //write particles from the master node + //loop over the other nodes and receive data - then write those. + for (int i = 0; i < mpi::n_procs; ++i) { +#ifdef OPENMC_MPI + if (i>0) + MPI_Recv(source_bank->data(), count[0], mpi::source_site, i, i, + mpi::intracomm, MPI_STATUS_IGNORE); +#endif + //now write the source_banke data again. + for (vector::iterator _site=source_bank->begin(); _site!=source_bank->end();_site++){ + //particle is now at the iterator + //write it to the mcpl-file + mcpl_particle_t p; + p.position[0]=_site->r.x; + p.position[1]=_site->r.y; + p.position[2]=_site->r.z; + + p.direction[0]=_site->u.x; + p.direction[1]=_site->u.y; + p.direction[2]=_site->u.x; + + //mcpl stores kinetic energy in MeV + p.ekin=_site->E*1e-6; + + p.time=_site->time*1e3; + + p.weight=_site->wgt; + + switch(_site->particle){ + case ParticleType::neutron: + p.pdgcode=2112; + break; + case ParticleType::photon: + p.pdgcode=22; + break; + case ParticleType::electron: + p.pdgcode=11; + break; + case ParticleType::positron: + p.pdgcode=-11; + break; + } + + mcpl_add_particle(file_id,&p); + } + } + } else { +#ifdef OPENMC_MPI + MPI_Send(source_bank->data(), count_size, mpi::source_site, 0, mpi::rank, + mpi::intracomm); +#endif + } + +} + + // Determine member names of a compound HDF5 datatype std::string dtype_member_names(hid_t dtype_id) { From be09b442fbfc41d029ed0eccef87ea25477f40aa Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Tue, 6 Sep 2022 09:11:41 +0200 Subject: [PATCH 02/55] allow electrons and positrons as well --- src/source.cpp | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/source.cpp b/src/source.cpp index 82978f1030..e7c944cb58 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -383,19 +383,30 @@ SourceSite MCPLFileSource::read_single_particle() const { SourceSite omc_particle_; const mcpl_particle_t *mcpl_particle; - //extract particle from mcpl-file + // extract particle from mcpl-file mcpl_particle=mcpl_read(mcpl_file); - // check if it is a neutron or a photon. otherwise skip - while ( mcpl_particle->pdgcode!=2112 && mcpl_particle->pdgcode!=22 ) { + // check if it is a neutron, photon, electron, or positron. Otherwise skip. + int pdg=mcpl_particle->pdgcode; + while ( pdg!=2112 && pdg!=22 && pdg!=11 && pdg!=-11) { mcpl_particle=mcpl_read(mcpl_file); - //should check for file exhaustion This could happen if particles are other than - //neutrons or photons + pdg=mcpl_particle->pdgcode; + //should check for file exhaustion. This could happen if particles are other than + //neutrons, photons, electrons, or positrons. } - if(mcpl_particle->pdgcode==2112) { - omc_particle_.particle=ParticleType::neutron; - } else if (mcpl_particle->pdgcode==22) { - omc_particle_.particle=ParticleType::photon; + switch(pdg){ + case 2112: + omc_particle_.particle=ParticleType::neutron; + break; + case 22: + omc_particle_.particle=ParticleType::photon; + break; + case 11: + omc_particle_.particle=ParticleType::electron; + break; + case -11: + omc_particle_.particle=ParticleType::positron; + break; } //particle is good, convert to openmc-formalism From 4e5ba4b4601209f389c394cd3747891e76b43f1c Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Tue, 6 Sep 2022 09:12:01 +0200 Subject: [PATCH 03/55] expose a python function to check for mcpl --- openmc/lib/__init__.py | 3 +++ src/source.cpp | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/openmc/lib/__init__.py b/openmc/lib/__init__.py index c14b0d9c28..1e337fc073 100644 --- a/openmc/lib/__init__.py +++ b/openmc/lib/__init__.py @@ -48,6 +48,9 @@ def _coord_levels(): def _libmesh_enabled(): return c_bool.in_dll(_dll, "LIBMESH_ENABLED").value +def _mcpl_enabled(): + return c_bool.in_dll(_dll, "MCPL_ENABLED").value + from .error import * from .core import * from .nuclide import * diff --git a/src/source.cpp b/src/source.cpp index e7c944cb58..9b426c3cfa 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -41,6 +41,12 @@ namespace openmc { // Global variables //============================================================================== +#ifdef OPENMC_MCPL +const bool MCPL_ENABLED = true; +#else +const bool MCPL_ENABLED = false; +#endif + namespace model { vector> external_sources; From 32e71395b3ad44ecc7080ba39d4b920adfb88702 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Tue, 6 Sep 2022 23:06:19 +0200 Subject: [PATCH 04/55] add a trigger to simulation control --- src/simulation.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/simulation.cpp b/src/simulation.cpp index b70535c330..05c8c04508 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -406,6 +406,13 @@ void finalize_batch() auto filename = settings::path_output + "surface_source.h5"; write_source_point(filename.c_str(), true); } + + if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ + auto filename = settings::path_output + ".mcpl"; + write_mcplt_source_point(filename.c_str(), true); + } + + } void initialize_generation() From ab665fb4563be28b6cfd365634049c90f133f485 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Fri, 15 Jul 2022 21:35:59 +0200 Subject: [PATCH 05/55] fix typo Co-authored-by: Paul Romano --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6af9643cdb..be56f1c12f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,7 +36,7 @@ option(OPENMC_ENABLE_COVERAGE "Compile with coverage analysis flags" option(OPENMC_USE_DAGMC "Enable support for DAGMC (CAD) geometry" OFF) option(OPENMC_USE_LIBMESH "Enable support for libMesh unstructured mesh tallies" OFF) option(OPENMC_USE_MPI "Enable MPI" OFF) -option(OPENMC_USE_MCPL "Enable MPCPL" OFF) +option(OPENMC_USE_MCPL "Enable MCPL" OFF) #=============================================================================== # Set a default build configuration if not explicitly specified From 71568d4f4bffa21a2a9c26725619cd180a9a59ff Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 9 Aug 2022 08:12:19 +0200 Subject: [PATCH 06/55] refactor MCPL file read part of FileSource A new constructor is added to FileSource that reads from a given MCPL-file. --- include/openmc/source.h | 30 ++--------- src/settings.cpp | 2 +- src/source.cpp | 116 +++++++++++++++++----------------------- 3 files changed, 53 insertions(+), 95 deletions(-) diff --git a/include/openmc/source.h b/include/openmc/source.h index 4d7ec104aa..5c2dae9311 100644 --- a/include/openmc/source.h +++ b/include/openmc/source.h @@ -96,7 +96,9 @@ class FileSource : public Source { public: // Constructors explicit FileSource(std::string path); - +#ifdef OPENMC_MCPL + explicit FileSource(mcpl_file_t mcpl_file); +#endif // Methods SourceSite sample(uint64_t* seed) const override; @@ -104,32 +106,6 @@ private: vector sites_; //!< Source sites from a file }; -#ifdef OPENMC_MCPL -//============================================================================== -// MCPL-file input source -//============================================================================== -class MCPLFileSource : public Source { -public: - // Constructors, destructors - MCPLFileSource(std::string path); - ~MCPLFileSource(); - - // Methods - //! Sample from the external source distribution - //! \param[inout] seed Pseudorandom seed pointer - //! \return Site read from MCPL-file - SourceSite sample(uint64_t* seed) const; - -private: - SourceSite read_single_particle() const; - void read_source_bank(vector &sites_); - - vector sites_; //!(path)); + model::external_sources.push_back(make_unique(mcpl_open_file(path.c_str()))); #endif } else if (check_for_node(node, "library")) { // Get shared library path and parameters diff --git a/src/source.cpp b/src/source.cpp index 9b426c3cfa..9cdf1632bf 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -283,6 +283,54 @@ FileSource::FileSource(std::string path) file_close(file_id); } +#ifdef OPENMC_MCPL +FileSource::FileSource(mcpl_file_t mcpl_file) +{ + //do checks on the mcpl_file to see if particles are many enough. + // should model this on the example source shown in the docs. + size_t n_sites=mcpl_hdr_nparticles(mcpl_file); + + sites_.resize(n_sites); + for (int i=0;ipdgcode!=2112 && mcpl_particle->pdgcode!=22 ) { + mcpl_particle=mcpl_read(mcpl_file); + //should check for file exhaustion This could happen if particles are other than + //neutrons or photons + } + + if(mcpl_particle->pdgcode==2112) { + site_.particle=ParticleType::neutron; + } else if (mcpl_particle->pdgcode==22) { + site_.particle=ParticleType::photon; + } + + //particle is good, convert to openmc-formalism + site_.r.x=mcpl_particle->position[0]; + site_.r.y=mcpl_particle->position[1]; + site_.r.z=mcpl_particle->position[2]; + + site_.u.x=mcpl_particle->direction[0]; + site_.u.y=mcpl_particle->direction[1]; + site_.u.z=mcpl_particle->direction[2]; + + //mcpl stores kinetic energy in MeV + site_.E=mcpl_particle->ekin*1e6; + //mcpl stores time in ms + site_.time=mcpl_particle->time*1e-3; + site_.wgt=mcpl_particle->weight; + site_.delayed_group=0; + sites_[i]=site_; + } + mcpl_close_file(mcpl_file); +} +#endif //OPENMC_MCPL + SourceSite FileSource::sample(uint64_t* seed) const { size_t i_site = sites_.size() * prn(seed); @@ -366,73 +414,7 @@ MCPLFileSource::MCPLFileSource(std::string path) read_source_bank(sites_); } - -MCPLFileSource::~MCPLFileSource(){ - mcpl_close_file(mcpl_file); -} - -SourceSite MCPLFileSource::sample(uint64_t* seed) const -{ - size_t i_site = sites_.size() * prn(seed); - return sites_[i_site]; -} - -void MCPLFileSource::read_source_bank(vector &sites_) -{ - sites_.resize(n_sites); - for (int i=0;ipdgcode; - while ( pdg!=2112 && pdg!=22 && pdg!=11 && pdg!=-11) { - mcpl_particle=mcpl_read(mcpl_file); - pdg=mcpl_particle->pdgcode; - //should check for file exhaustion. This could happen if particles are other than - //neutrons, photons, electrons, or positrons. - } - - switch(pdg){ - case 2112: - omc_particle_.particle=ParticleType::neutron; - break; - case 22: - omc_particle_.particle=ParticleType::photon; - break; - case 11: - omc_particle_.particle=ParticleType::electron; - break; - case -11: - omc_particle_.particle=ParticleType::positron; - break; - } - - //particle is good, convert to openmc-formalism - omc_particle_.r.x=mcpl_particle->position[0]; - omc_particle_.r.y=mcpl_particle->position[1]; - omc_particle_.r.z=mcpl_particle->position[2]; - - omc_particle_.u.x=mcpl_particle->direction[0]; - omc_particle_.u.y=mcpl_particle->direction[1]; - omc_particle_.u.z=mcpl_particle->direction[2]; - - //mcpl stores kinetic energy in MeV - omc_particle_.E=mcpl_particle->ekin*1e6; - //mcpl stores time in ms - omc_particle_.time=mcpl_particle->time*1e-3; - omc_particle_.wgt=mcpl_particle->weight; - omc_particle_.delayed_group=0; - return omc_particle_; -} -#endif //OPENMC_MCPL +#endif //============================================================================== // Non-member functions From 73a391daef86e3a93c0c7180d9ff7f670afff240 Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 6 Sep 2022 02:22:21 +0200 Subject: [PATCH 07/55] wip add MCPL-out to openmc code for writing a source_bank/point compiles but is untested --- include/openmc/state_point.h | 9 +++ src/state_point.cpp | 125 +++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/include/openmc/state_point.h b/include/openmc/state_point.h index 5ada2bf88d..2d92c935a6 100644 --- a/include/openmc/state_point.h +++ b/include/openmc/state_point.h @@ -9,6 +9,10 @@ #include "openmc/particle.h" #include "openmc/vector.h" +#ifdef OPENMC_MCPL +#include +#endif + namespace openmc { void load_state_point(); @@ -21,5 +25,10 @@ void write_tally_results_nr(hid_t file_id); void restart_set_keff(); void write_unstructured_mesh_results(); +#ifdef OPENMC_MCPL +void write_mcpl_source_point(const char *filename_, bool surf_source_bank = false); +void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank); +#endif + } // namespace openmc #endif // OPENMC_STATE_POINT_H diff --git a/src/state_point.cpp b/src/state_point.cpp index 9ea0459153..babd670a2d 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -28,6 +28,10 @@ #include "openmc/timer.h" #include "openmc/vector.h" +#ifdef OPENMC_MCPL +#include +#endif + namespace openmc { extern "C" int openmc_statepoint_write(const char* filename, bool* write_source) @@ -598,6 +602,45 @@ void write_source_point(const char* filename, bool surf_source_bank) file_close(file_id); } +void write_mcpl_point(const char *filename, bool surf_source_bank) +{ + std::string filename_; + if (filename) { + filename_ = filename; + } else { + // Determine width for zero padding + int w = std::to_string(settings::n_max_batches).size(); + + filename_ = fmt::format("{0}source.{1:0{2}}.mcpl", settings::path_output, + simulation::current_batch, w); + } + + mcpl_outfile_t file_id; + + std::string line; + if (mpi::master) { + // this must be rewritten: file_open is h5-specific + file_id = mcpl_create_outfile(filename_.c_str()); + //write_attribute(file_id, "filetype", "source"); + //write header stuff (oopy in xml-files as binary blobs for instance)) + if (VERSION_DEV){ + line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}-development"); + } else { + line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}"); + } + mcpl_hdr_set_srcname(file_id,line.c_str()); + } + + write_mcpl_source_bank(file_id, surf_source_bank); + + if (mpi::master) { + //change this - this is h5 specific + mcpl_closeandgzip_outfile(file_id); + } + +} + + void write_source_bank(hid_t group_id, bool surf_source_bank) { hid_t banktype = h5banktype(); @@ -714,6 +757,88 @@ void write_source_bank(hid_t group_id, bool surf_source_bank) H5Tclose(banktype); } +void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) +{ + int64_t dims_size = settings::n_particles; + int64_t count_size = simulation::work_per_rank; + + // Set vectors for source bank and starting bank index of each process + vector* bank_index = &simulation::work_index; + vector* source_bank = &simulation::source_bank; + vector surf_source_index_vector; + vector surf_source_bank_vector; + + if(surf_source_bank) { + surf_source_index_vector = calculate_surf_source_size(); + dims_size = surf_source_index_vector[mpi::n_procs]; + count_size = simulation::surf_source_bank.size(); + + bank_index = &surf_source_index_vector; + + // Copy data in a SharedArray into a vector. + surf_source_bank_vector.resize(count_size); + surf_source_bank_vector.assign(simulation::surf_source_bank.data(), + simulation::surf_source_bank.data() + count_size); + source_bank = &surf_source_bank_vector; + } + + if (mpi::master) { + //write particles from the master node + //loop over the other nodes and receive data - then write those. + for (int i = 0; i < mpi::n_procs; ++i) { +#ifdef OPENMC_MPI + if (i>0) + MPI_Recv(source_bank->data(), count[0], mpi::source_site, i, i, + mpi::intracomm, MPI_STATUS_IGNORE); +#endif + //now write the source_banke data again. + for (vector::iterator _site=source_bank->begin(); _site!=source_bank->end();_site++){ + //particle is now at the iterator + //write it to the mcpl-file + mcpl_particle_t p; + p.position[0]=_site->r.x; + p.position[1]=_site->r.y; + p.position[2]=_site->r.z; + + p.direction[0]=_site->u.x; + p.direction[1]=_site->u.y; + p.direction[2]=_site->u.x; + + //mcpl stores kinetic energy in MeV + p.ekin=_site->E*1e-6; + + p.time=_site->time*1e3; + + p.weight=_site->wgt; + + switch(_site->particle){ + case ParticleType::neutron: + p.pdgcode=2112; + break; + case ParticleType::photon: + p.pdgcode=22; + break; + case ParticleType::electron: + p.pdgcode=11; + break; + case ParticleType::positron: + p.pdgcode=-11; + break; + } + + mcpl_add_particle(file_id,&p); + } + } + } else { +#ifdef OPENMC_MPI + MPI_Send(source_bank->data(), count_size, mpi::source_site, 0, mpi::rank, + mpi::intracomm); +#endif + } + +} + + // Determine member names of a compound HDF5 datatype std::string dtype_member_names(hid_t dtype_id) { From d07223bf681e5d6e1d7cbde0dac6f97059a12767 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Tue, 6 Sep 2022 23:06:19 +0200 Subject: [PATCH 08/55] add a trigger to simulation control --- src/simulation.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/simulation.cpp b/src/simulation.cpp index b70535c330..05c8c04508 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -406,6 +406,13 @@ void finalize_batch() auto filename = settings::path_output + "surface_source.h5"; write_source_point(filename.c_str(), true); } + + if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ + auto filename = settings::path_output + ".mcpl"; + write_mcplt_source_point(filename.c_str(), true); + } + + } void initialize_generation() From 55f2be19e6c0ef51d7a7946ac15a9f2df51da6da Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Fri, 16 Sep 2022 16:07:22 +0200 Subject: [PATCH 09/55] refactor MCPL file read part of FileSource A new constructor is added to FileSource that reads from a given MCPL-file. --- src/source.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/source.cpp b/src/source.cpp index 626d02f9d7..21522438cf 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -426,6 +426,7 @@ MCPLFileSource::MCPLFileSource(std::string path) read_source_bank(sites_); } +<<<<<<< HEAD MCPLFileSource::~MCPLFileSource(){ mcpl_close_file(mcpl_file); @@ -495,6 +496,9 @@ SourceSite MCPLFileSource::read_single_particle() const #endif //OPENMC_MCPL ======= >>>>>>> 380cfc150b449ef56debb5261caa010a52c42bdd +======= +#endif +>>>>>>> 71568d4f4 (refactor MCPL file read part of FileSource) //============================================================================== // Non-member functions From f0af56b181555f52d05f245240846f4ce8eadcaa Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 6 Sep 2022 02:22:21 +0200 Subject: [PATCH 10/55] wip add MCPL-out to openmc code for writing a source_bank/point compiles but is untested --- include/openmc/state_point.h | 9 +++ src/state_point.cpp | 125 +++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/include/openmc/state_point.h b/include/openmc/state_point.h index 5ada2bf88d..2d92c935a6 100644 --- a/include/openmc/state_point.h +++ b/include/openmc/state_point.h @@ -9,6 +9,10 @@ #include "openmc/particle.h" #include "openmc/vector.h" +#ifdef OPENMC_MCPL +#include +#endif + namespace openmc { void load_state_point(); @@ -21,5 +25,10 @@ void write_tally_results_nr(hid_t file_id); void restart_set_keff(); void write_unstructured_mesh_results(); +#ifdef OPENMC_MCPL +void write_mcpl_source_point(const char *filename_, bool surf_source_bank = false); +void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank); +#endif + } // namespace openmc #endif // OPENMC_STATE_POINT_H diff --git a/src/state_point.cpp b/src/state_point.cpp index 9ea0459153..babd670a2d 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -28,6 +28,10 @@ #include "openmc/timer.h" #include "openmc/vector.h" +#ifdef OPENMC_MCPL +#include +#endif + namespace openmc { extern "C" int openmc_statepoint_write(const char* filename, bool* write_source) @@ -598,6 +602,45 @@ void write_source_point(const char* filename, bool surf_source_bank) file_close(file_id); } +void write_mcpl_point(const char *filename, bool surf_source_bank) +{ + std::string filename_; + if (filename) { + filename_ = filename; + } else { + // Determine width for zero padding + int w = std::to_string(settings::n_max_batches).size(); + + filename_ = fmt::format("{0}source.{1:0{2}}.mcpl", settings::path_output, + simulation::current_batch, w); + } + + mcpl_outfile_t file_id; + + std::string line; + if (mpi::master) { + // this must be rewritten: file_open is h5-specific + file_id = mcpl_create_outfile(filename_.c_str()); + //write_attribute(file_id, "filetype", "source"); + //write header stuff (oopy in xml-files as binary blobs for instance)) + if (VERSION_DEV){ + line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}-development"); + } else { + line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}"); + } + mcpl_hdr_set_srcname(file_id,line.c_str()); + } + + write_mcpl_source_bank(file_id, surf_source_bank); + + if (mpi::master) { + //change this - this is h5 specific + mcpl_closeandgzip_outfile(file_id); + } + +} + + void write_source_bank(hid_t group_id, bool surf_source_bank) { hid_t banktype = h5banktype(); @@ -714,6 +757,88 @@ void write_source_bank(hid_t group_id, bool surf_source_bank) H5Tclose(banktype); } +void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) +{ + int64_t dims_size = settings::n_particles; + int64_t count_size = simulation::work_per_rank; + + // Set vectors for source bank and starting bank index of each process + vector* bank_index = &simulation::work_index; + vector* source_bank = &simulation::source_bank; + vector surf_source_index_vector; + vector surf_source_bank_vector; + + if(surf_source_bank) { + surf_source_index_vector = calculate_surf_source_size(); + dims_size = surf_source_index_vector[mpi::n_procs]; + count_size = simulation::surf_source_bank.size(); + + bank_index = &surf_source_index_vector; + + // Copy data in a SharedArray into a vector. + surf_source_bank_vector.resize(count_size); + surf_source_bank_vector.assign(simulation::surf_source_bank.data(), + simulation::surf_source_bank.data() + count_size); + source_bank = &surf_source_bank_vector; + } + + if (mpi::master) { + //write particles from the master node + //loop over the other nodes and receive data - then write those. + for (int i = 0; i < mpi::n_procs; ++i) { +#ifdef OPENMC_MPI + if (i>0) + MPI_Recv(source_bank->data(), count[0], mpi::source_site, i, i, + mpi::intracomm, MPI_STATUS_IGNORE); +#endif + //now write the source_banke data again. + for (vector::iterator _site=source_bank->begin(); _site!=source_bank->end();_site++){ + //particle is now at the iterator + //write it to the mcpl-file + mcpl_particle_t p; + p.position[0]=_site->r.x; + p.position[1]=_site->r.y; + p.position[2]=_site->r.z; + + p.direction[0]=_site->u.x; + p.direction[1]=_site->u.y; + p.direction[2]=_site->u.x; + + //mcpl stores kinetic energy in MeV + p.ekin=_site->E*1e-6; + + p.time=_site->time*1e3; + + p.weight=_site->wgt; + + switch(_site->particle){ + case ParticleType::neutron: + p.pdgcode=2112; + break; + case ParticleType::photon: + p.pdgcode=22; + break; + case ParticleType::electron: + p.pdgcode=11; + break; + case ParticleType::positron: + p.pdgcode=-11; + break; + } + + mcpl_add_particle(file_id,&p); + } + } + } else { +#ifdef OPENMC_MPI + MPI_Send(source_bank->data(), count_size, mpi::source_site, 0, mpi::rank, + mpi::intracomm); +#endif + } + +} + + // Determine member names of a compound HDF5 datatype std::string dtype_member_names(hid_t dtype_id) { From 04c8ffc81c7ed7ced0c3525b5ec13f15f8fc079c Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Tue, 6 Sep 2022 23:06:19 +0200 Subject: [PATCH 11/55] add a trigger to simulation control --- src/simulation.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/simulation.cpp b/src/simulation.cpp index b70535c330..05c8c04508 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -406,6 +406,13 @@ void finalize_batch() auto filename = settings::path_output + "surface_source.h5"; write_source_point(filename.c_str(), true); } + + if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ + auto filename = settings::path_output + ".mcpl"; + write_mcplt_source_point(filename.c_str(), true); + } + + } void initialize_generation() From 314f3dbbab75104f217d1e20e2146ced28108707 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Tue, 4 Oct 2022 10:11:06 +0200 Subject: [PATCH 12/55] do not define these functions if no MCPL present --- src/state_point.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index babd670a2d..5b00f302dc 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -602,6 +602,7 @@ void write_source_point(const char* filename, bool surf_source_bank) file_close(file_id); } +#ifdef OPENMC_MCPL void write_mcpl_point(const char *filename, bool surf_source_bank) { std::string filename_; @@ -639,7 +640,7 @@ void write_mcpl_point(const char *filename, bool surf_source_bank) } } - +#endif void write_source_bank(hid_t group_id, bool surf_source_bank) { @@ -757,6 +758,7 @@ void write_source_bank(hid_t group_id, bool surf_source_bank) H5Tclose(banktype); } +#ifdef OPENMC_MCPL void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) { int64_t dims_size = settings::n_particles; @@ -837,7 +839,7 @@ void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) } } - +#endif // Determine member names of a compound HDF5 datatype std::string dtype_member_names(hid_t dtype_id) From 2de20231ad584fa7a77a5b135c82f6e995671bcc Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 10:56:36 +0200 Subject: [PATCH 13/55] use find_package instead - MCPL now supports this --- CMakeLists.txt | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index be56f1c12f..98dc4fcaf0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,6 +128,13 @@ if(${HDF5_VERSION} VERSION_GREATER_EQUAL 1.12.0) list(APPEND cxxflags -DH5Oget_info_by_idx_vers=1 -DH5O_info_t_vers=1) endif() +#=============================================================================== +# MCPL +#=============================================================================== +if (OPENMC_USE_MCPL) + find_package(MCPL REQUIRED) +endif() + #=============================================================================== # Set compile/link flags based on which compiler is being used #=============================================================================== @@ -155,10 +162,6 @@ if(OPENMC_ENABLE_COVERAGE) list(APPEND ldflags --coverage) endif() -if(OPENMC_USE_MCPL) - list(APPEND ldflags -lmcpl) -endif() - # Show flags being used message(STATUS "OpenMC C++ flags: ${cxxflags}") message(STATUS "OpenMC Linker flags: ${ldflags}") @@ -413,10 +416,6 @@ endif() if (OPENMC_USE_MPI) target_compile_definitions(libopenmc PUBLIC -DOPENMC_MPI) endif() -if(OPENMC_USE_MCPL) - target_compile_definitions(libopenmc PUBLIC -DOPENMC_MCPL) -endif() - # Set git SHA1 hash as a compile definition if(GIT_FOUND) @@ -460,6 +459,11 @@ if (OPENMC_USE_MPI) target_link_libraries(libopenmc MPI::MPI_CXX) endif() +if (OPENMC_USE_MCPL) + target_compile_definitions(libopenmc PUBLIC OPENMC_MCPL) + target_link_libraries(libopenmc MCPL::mcpl) +endif() + #=============================================================================== # openmc executable #=============================================================================== From 7418cb99dd7c22e80532dcc46a9d2c4c04ceb63d Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 10:59:02 +0200 Subject: [PATCH 14/55] add mcpl-output settings --- include/openmc/settings.h | 1 + src/settings.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/include/openmc/settings.h b/include/openmc/settings.h index 1e061b235b..66825867d8 100644 --- a/include/openmc/settings.h +++ b/include/openmc/settings.h @@ -48,6 +48,7 @@ extern bool source_latest; //!< write latest source at each batch? extern bool source_separate; //!< write source to separate file? extern bool source_write; //!< write source in HDF5 files? extern bool surf_source_write; //!< write surface source file? +extern bool surf_mcpl_write; //!< write surface mcpl file? extern bool surf_source_read; //!< read surface source file? extern bool survival_biasing; //!< use survival biasing? extern bool temperature_multipole; //!< use multipole data? diff --git a/src/settings.cpp b/src/settings.cpp index 11dd96e025..d1762596c6 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -61,6 +61,7 @@ bool source_latest {false}; bool source_separate {false}; bool source_write {true}; bool surf_source_write {false}; +bool surf_mcpl_write {false}; bool surf_source_read {false}; bool survival_biasing {false}; bool temperature_multipole {false}; @@ -682,9 +683,14 @@ void read_settings_xml() max_surface_particles = std::stoll(get_node_value(node_ssw, "max_particles")); } +#ifdef OPENMC_MCPL + if(check_for_node(node_ssw, "mcpl")){ + surf_mcpl_write=true; + } +#endif } - // If source is not seperate and is to be written out in the statepoint file, + // If source is not separate and is to be written out in the statepoint file, // make sure that the sourcepoint batch numbers are contained in the // statepoint list if (!source_separate) { From 172c518f0e161d0176385c67d3717ad9fb229038 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 10:59:28 +0200 Subject: [PATCH 15/55] fix typo --- src/simulation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation.cpp b/src/simulation.cpp index 05c8c04508..353ef99265 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -409,7 +409,7 @@ void finalize_batch() if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ auto filename = settings::path_output + ".mcpl"; - write_mcplt_source_point(filename.c_str(), true); + write_mcpl_source_point(filename.c_str(), true); } From 83e2921034f5c448d500b93634999cdb3f350e0b Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 11:00:39 +0200 Subject: [PATCH 16/55] add cond. compilation guards for MCPL functions --- src/state_point.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index babd670a2d..7c1a454476 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -602,7 +602,8 @@ void write_source_point(const char* filename, bool surf_source_bank) file_close(file_id); } -void write_mcpl_point(const char *filename, bool surf_source_bank) +#ifdef OPENMC_MCPL +void write_mcpl_source_point(const char *filename, bool surf_source_bank) { std::string filename_; if (filename) { @@ -639,7 +640,7 @@ void write_mcpl_point(const char *filename, bool surf_source_bank) } } - +#endif void write_source_bank(hid_t group_id, bool surf_source_bank) { @@ -757,6 +758,7 @@ void write_source_bank(hid_t group_id, bool surf_source_bank) H5Tclose(banktype); } +#ifdef OPENMC_MCPL void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) { int64_t dims_size = settings::n_particles; @@ -837,7 +839,7 @@ void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) } } - +#endif // Determine member names of a compound HDF5 datatype std::string dtype_member_names(hid_t dtype_id) From 5e82bb2e1f1b2857375566f98669be6c93b22638 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Tue, 6 Sep 2022 09:12:01 +0200 Subject: [PATCH 17/55] expose a python function to check for mcpl --- openmc/lib/__init__.py | 3 +++ src/source.cpp | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/openmc/lib/__init__.py b/openmc/lib/__init__.py index c14b0d9c28..1e337fc073 100644 --- a/openmc/lib/__init__.py +++ b/openmc/lib/__init__.py @@ -48,6 +48,9 @@ def _coord_levels(): def _libmesh_enabled(): return c_bool.in_dll(_dll, "LIBMESH_ENABLED").value +def _mcpl_enabled(): + return c_bool.in_dll(_dll, "MCPL_ENABLED").value + from .error import * from .core import * from .nuclide import * diff --git a/src/source.cpp b/src/source.cpp index 49e51c4b16..eace931d63 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -41,6 +41,12 @@ namespace openmc { // Global variables //============================================================================== +#ifdef OPENMC_MCPL +const bool MCPL_ENABLED = true; +#else +const bool MCPL_ENABLED = false; +#endif + namespace model { vector> external_sources; From 6038555def28547a0f392f0eeded8d8d49269d51 Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 6 Sep 2022 02:22:21 +0200 Subject: [PATCH 18/55] wip add MCPL-out to openmc code for writing a source_bank/point compiles but is untested --- include/openmc/state_point.h | 9 +++ src/state_point.cpp | 125 +++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/include/openmc/state_point.h b/include/openmc/state_point.h index 5ada2bf88d..2d92c935a6 100644 --- a/include/openmc/state_point.h +++ b/include/openmc/state_point.h @@ -9,6 +9,10 @@ #include "openmc/particle.h" #include "openmc/vector.h" +#ifdef OPENMC_MCPL +#include +#endif + namespace openmc { void load_state_point(); @@ -21,5 +25,10 @@ void write_tally_results_nr(hid_t file_id); void restart_set_keff(); void write_unstructured_mesh_results(); +#ifdef OPENMC_MCPL +void write_mcpl_source_point(const char *filename_, bool surf_source_bank = false); +void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank); +#endif + } // namespace openmc #endif // OPENMC_STATE_POINT_H diff --git a/src/state_point.cpp b/src/state_point.cpp index 9ea0459153..babd670a2d 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -28,6 +28,10 @@ #include "openmc/timer.h" #include "openmc/vector.h" +#ifdef OPENMC_MCPL +#include +#endif + namespace openmc { extern "C" int openmc_statepoint_write(const char* filename, bool* write_source) @@ -598,6 +602,45 @@ void write_source_point(const char* filename, bool surf_source_bank) file_close(file_id); } +void write_mcpl_point(const char *filename, bool surf_source_bank) +{ + std::string filename_; + if (filename) { + filename_ = filename; + } else { + // Determine width for zero padding + int w = std::to_string(settings::n_max_batches).size(); + + filename_ = fmt::format("{0}source.{1:0{2}}.mcpl", settings::path_output, + simulation::current_batch, w); + } + + mcpl_outfile_t file_id; + + std::string line; + if (mpi::master) { + // this must be rewritten: file_open is h5-specific + file_id = mcpl_create_outfile(filename_.c_str()); + //write_attribute(file_id, "filetype", "source"); + //write header stuff (oopy in xml-files as binary blobs for instance)) + if (VERSION_DEV){ + line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}-development"); + } else { + line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}"); + } + mcpl_hdr_set_srcname(file_id,line.c_str()); + } + + write_mcpl_source_bank(file_id, surf_source_bank); + + if (mpi::master) { + //change this - this is h5 specific + mcpl_closeandgzip_outfile(file_id); + } + +} + + void write_source_bank(hid_t group_id, bool surf_source_bank) { hid_t banktype = h5banktype(); @@ -714,6 +757,88 @@ void write_source_bank(hid_t group_id, bool surf_source_bank) H5Tclose(banktype); } +void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) +{ + int64_t dims_size = settings::n_particles; + int64_t count_size = simulation::work_per_rank; + + // Set vectors for source bank and starting bank index of each process + vector* bank_index = &simulation::work_index; + vector* source_bank = &simulation::source_bank; + vector surf_source_index_vector; + vector surf_source_bank_vector; + + if(surf_source_bank) { + surf_source_index_vector = calculate_surf_source_size(); + dims_size = surf_source_index_vector[mpi::n_procs]; + count_size = simulation::surf_source_bank.size(); + + bank_index = &surf_source_index_vector; + + // Copy data in a SharedArray into a vector. + surf_source_bank_vector.resize(count_size); + surf_source_bank_vector.assign(simulation::surf_source_bank.data(), + simulation::surf_source_bank.data() + count_size); + source_bank = &surf_source_bank_vector; + } + + if (mpi::master) { + //write particles from the master node + //loop over the other nodes and receive data - then write those. + for (int i = 0; i < mpi::n_procs; ++i) { +#ifdef OPENMC_MPI + if (i>0) + MPI_Recv(source_bank->data(), count[0], mpi::source_site, i, i, + mpi::intracomm, MPI_STATUS_IGNORE); +#endif + //now write the source_banke data again. + for (vector::iterator _site=source_bank->begin(); _site!=source_bank->end();_site++){ + //particle is now at the iterator + //write it to the mcpl-file + mcpl_particle_t p; + p.position[0]=_site->r.x; + p.position[1]=_site->r.y; + p.position[2]=_site->r.z; + + p.direction[0]=_site->u.x; + p.direction[1]=_site->u.y; + p.direction[2]=_site->u.x; + + //mcpl stores kinetic energy in MeV + p.ekin=_site->E*1e-6; + + p.time=_site->time*1e3; + + p.weight=_site->wgt; + + switch(_site->particle){ + case ParticleType::neutron: + p.pdgcode=2112; + break; + case ParticleType::photon: + p.pdgcode=22; + break; + case ParticleType::electron: + p.pdgcode=11; + break; + case ParticleType::positron: + p.pdgcode=-11; + break; + } + + mcpl_add_particle(file_id,&p); + } + } + } else { +#ifdef OPENMC_MPI + MPI_Send(source_bank->data(), count_size, mpi::source_site, 0, mpi::rank, + mpi::intracomm); +#endif + } + +} + + // Determine member names of a compound HDF5 datatype std::string dtype_member_names(hid_t dtype_id) { From 2ba7f8d1882cf5e2562b473e233b4ca84ab81080 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Tue, 6 Sep 2022 23:06:19 +0200 Subject: [PATCH 19/55] add a trigger to simulation control --- src/simulation.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/simulation.cpp b/src/simulation.cpp index b70535c330..05c8c04508 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -406,6 +406,13 @@ void finalize_batch() auto filename = settings::path_output + "surface_source.h5"; write_source_point(filename.c_str(), true); } + + if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ + auto filename = settings::path_output + ".mcpl"; + write_mcplt_source_point(filename.c_str(), true); + } + + } void initialize_generation() From 91cd304f0e4fd32eb6df0bdb75dd3f7d169013ce Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 10:56:36 +0200 Subject: [PATCH 20/55] use find_package instead - MCPL now supports this --- CMakeLists.txt | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index be56f1c12f..98dc4fcaf0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,6 +128,13 @@ if(${HDF5_VERSION} VERSION_GREATER_EQUAL 1.12.0) list(APPEND cxxflags -DH5Oget_info_by_idx_vers=1 -DH5O_info_t_vers=1) endif() +#=============================================================================== +# MCPL +#=============================================================================== +if (OPENMC_USE_MCPL) + find_package(MCPL REQUIRED) +endif() + #=============================================================================== # Set compile/link flags based on which compiler is being used #=============================================================================== @@ -155,10 +162,6 @@ if(OPENMC_ENABLE_COVERAGE) list(APPEND ldflags --coverage) endif() -if(OPENMC_USE_MCPL) - list(APPEND ldflags -lmcpl) -endif() - # Show flags being used message(STATUS "OpenMC C++ flags: ${cxxflags}") message(STATUS "OpenMC Linker flags: ${ldflags}") @@ -413,10 +416,6 @@ endif() if (OPENMC_USE_MPI) target_compile_definitions(libopenmc PUBLIC -DOPENMC_MPI) endif() -if(OPENMC_USE_MCPL) - target_compile_definitions(libopenmc PUBLIC -DOPENMC_MCPL) -endif() - # Set git SHA1 hash as a compile definition if(GIT_FOUND) @@ -460,6 +459,11 @@ if (OPENMC_USE_MPI) target_link_libraries(libopenmc MPI::MPI_CXX) endif() +if (OPENMC_USE_MCPL) + target_compile_definitions(libopenmc PUBLIC OPENMC_MCPL) + target_link_libraries(libopenmc MCPL::mcpl) +endif() + #=============================================================================== # openmc executable #=============================================================================== From 9b58c2f6dc93c4c51ce6ee7a26e7ffc645434020 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 10:59:02 +0200 Subject: [PATCH 21/55] add mcpl-output settings --- include/openmc/settings.h | 1 + src/settings.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/include/openmc/settings.h b/include/openmc/settings.h index 1e061b235b..66825867d8 100644 --- a/include/openmc/settings.h +++ b/include/openmc/settings.h @@ -48,6 +48,7 @@ extern bool source_latest; //!< write latest source at each batch? extern bool source_separate; //!< write source to separate file? extern bool source_write; //!< write source in HDF5 files? extern bool surf_source_write; //!< write surface source file? +extern bool surf_mcpl_write; //!< write surface mcpl file? extern bool surf_source_read; //!< read surface source file? extern bool survival_biasing; //!< use survival biasing? extern bool temperature_multipole; //!< use multipole data? diff --git a/src/settings.cpp b/src/settings.cpp index 11dd96e025..d1762596c6 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -61,6 +61,7 @@ bool source_latest {false}; bool source_separate {false}; bool source_write {true}; bool surf_source_write {false}; +bool surf_mcpl_write {false}; bool surf_source_read {false}; bool survival_biasing {false}; bool temperature_multipole {false}; @@ -682,9 +683,14 @@ void read_settings_xml() max_surface_particles = std::stoll(get_node_value(node_ssw, "max_particles")); } +#ifdef OPENMC_MCPL + if(check_for_node(node_ssw, "mcpl")){ + surf_mcpl_write=true; + } +#endif } - // If source is not seperate and is to be written out in the statepoint file, + // If source is not separate and is to be written out in the statepoint file, // make sure that the sourcepoint batch numbers are contained in the // statepoint list if (!source_separate) { From 43538ac2021f36caf0d72e002a119a806aa14b71 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 10:59:28 +0200 Subject: [PATCH 22/55] fix typo --- src/simulation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation.cpp b/src/simulation.cpp index 05c8c04508..353ef99265 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -409,7 +409,7 @@ void finalize_batch() if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ auto filename = settings::path_output + ".mcpl"; - write_mcplt_source_point(filename.c_str(), true); + write_mcpl_source_point(filename.c_str(), true); } From cba745b28b6e384311393c177456b518b60b4f55 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 11:00:39 +0200 Subject: [PATCH 23/55] add cond. compilation guards for MCPL functions --- src/state_point.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index babd670a2d..7c1a454476 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -602,7 +602,8 @@ void write_source_point(const char* filename, bool surf_source_bank) file_close(file_id); } -void write_mcpl_point(const char *filename, bool surf_source_bank) +#ifdef OPENMC_MCPL +void write_mcpl_source_point(const char *filename, bool surf_source_bank) { std::string filename_; if (filename) { @@ -639,7 +640,7 @@ void write_mcpl_point(const char *filename, bool surf_source_bank) } } - +#endif void write_source_bank(hid_t group_id, bool surf_source_bank) { @@ -757,6 +758,7 @@ void write_source_bank(hid_t group_id, bool surf_source_bank) H5Tclose(banktype); } +#ifdef OPENMC_MCPL void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) { int64_t dims_size = settings::n_particles; @@ -837,7 +839,7 @@ void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) } } - +#endif // Determine member names of a compound HDF5 datatype std::string dtype_member_names(hid_t dtype_id) From 179df586853b65600518a0ea62526cc2193e0206 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Fri, 16 Sep 2022 16:07:22 +0200 Subject: [PATCH 24/55] refactor MCPL file read part of FileSource A new constructor is added to FileSource that reads from a given MCPL-file. --- src/source.cpp | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/src/source.cpp b/src/source.cpp index eace931d63..bd76969d17 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -391,6 +391,79 @@ CustomSourceWrapper::~CustomSourceWrapper() } +MCPLFileSource::~MCPLFileSource(){ + mcpl_close_file(mcpl_file); +} + +SourceSite MCPLFileSource::sample(uint64_t* seed) const +{ + size_t i_site = sites_.size() * prn(seed); + return sites_[i_site]; +} + +void MCPLFileSource::read_source_bank(vector &sites_) +{ + sites_.resize(n_sites); + for (int i=0;ipdgcode; + while ( pdg!=2112 && pdg!=22 && pdg!=11 && pdg!=-11) { + mcpl_particle=mcpl_read(mcpl_file); + pdg=mcpl_particle->pdgcode; + //should check for file exhaustion. This could happen if particles are other than + //neutrons, photons, electrons, or positrons. + } + + switch(pdg){ + case 2112: + omc_particle_.particle=ParticleType::neutron; + break; + case 22: + omc_particle_.particle=ParticleType::photon; + break; + case 11: + omc_particle_.particle=ParticleType::electron; + break; + case -11: + omc_particle_.particle=ParticleType::positron; + break; + } + + //particle is good, convert to openmc-formalism + omc_particle_.r.x=mcpl_particle->position[0]; + omc_particle_.r.y=mcpl_particle->position[1]; + omc_particle_.r.z=mcpl_particle->position[2]; + + omc_particle_.u.x=mcpl_particle->direction[0]; + omc_particle_.u.y=mcpl_particle->direction[1]; + omc_particle_.u.z=mcpl_particle->direction[2]; + + //mcpl stores kinetic energy in MeV + omc_particle_.E=mcpl_particle->ekin*1e6; + //mcpl stores time in ms + omc_particle_.time=mcpl_particle->time*1e-3; + omc_particle_.wgt=mcpl_particle->weight; + omc_particle_.delayed_group=0; + return omc_particle_; +} +#endif //OPENMC_MCPL +======= +>>>>>>> 380cfc150b449ef56debb5261caa010a52c42bdd +======= +#endif +>>>>>>> 71568d4f4 (refactor MCPL file read part of FileSource) +>>>>>>> 55f2be19e (refactor MCPL file read part of FileSource) + //============================================================================== // Non-member functions //============================================================================== From e3fbeac636c2e9a47599c60e04b2a1ba4d9ba46d Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 6 Sep 2022 02:22:21 +0200 Subject: [PATCH 25/55] wip add MCPL-out to openmc code for writing a source_bank/point compiles but is untested --- src/source.cpp | 6 ------ src/state_point.cpp | 1 + 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/source.cpp b/src/source.cpp index bd76969d17..f205a0a256 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -457,12 +457,6 @@ SourceSite MCPLFileSource::read_single_particle() const return omc_particle_; } #endif //OPENMC_MCPL -======= ->>>>>>> 380cfc150b449ef56debb5261caa010a52c42bdd -======= -#endif ->>>>>>> 71568d4f4 (refactor MCPL file read part of FileSource) ->>>>>>> 55f2be19e (refactor MCPL file read part of FileSource) //============================================================================== // Non-member functions diff --git a/src/state_point.cpp b/src/state_point.cpp index 7c1a454476..31979c0c56 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -642,6 +642,7 @@ void write_mcpl_source_point(const char *filename, bool surf_source_bank) } #endif + void write_source_bank(hid_t group_id, bool surf_source_bank) { hid_t banktype = h5banktype(); From 3b7c4f6d2f141311cba5ad9e5f773b3bc3381852 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Tue, 4 Oct 2022 10:11:06 +0200 Subject: [PATCH 26/55] do not define these functions if no MCPL present --- src/state_point.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index 31979c0c56..7c1a454476 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -642,7 +642,6 @@ void write_mcpl_source_point(const char *filename, bool surf_source_bank) } #endif - void write_source_bank(hid_t group_id, bool surf_source_bank) { hid_t banktype = h5banktype(); From 982377b0cdfc573469aad799852a0f2dd50b2e68 Mon Sep 17 00:00:00 2001 From: erkn Date: Wed, 5 Oct 2022 14:31:05 +0200 Subject: [PATCH 27/55] fix misnamed variable and remove accientally committed code --- src/source.cpp | 169 ++----------------------------------------------- 1 file changed, 4 insertions(+), 165 deletions(-) diff --git a/src/source.cpp b/src/source.cpp index e1891c5e94..e35734747f 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -308,16 +308,16 @@ FileSource::FileSource(mcpl_file_t mcpl_file) switch(pdg){ case 2112: - omc_particle_.particle=ParticleType::neutron; + site_.particle=ParticleType::neutron; break; case 22: - omc_particle_.particle=ParticleType::photon; + site_.particle=ParticleType::photon; break; case 11: - omc_particle_.particle=ParticleType::electron; + site_.particle=ParticleType::electron; break; case -11: - omc_particle_.particle=ParticleType::positron; + site_.particle=ParticleType::positron; break; } @@ -401,167 +401,6 @@ CustomSourceWrapper::~CustomSourceWrapper() #endif } -<<<<<<< HEAD -<<<<<<< HEAD -#ifdef OPENMC_MCPL -//=========================================================================== -// Read particles from an MCPL-file -//=========================================================================== -MCPLFileSource::MCPLFileSource(std::string path) -{ - // Check if source file exists - if (!file_exists(path)) { - fatal_error(fmt::format("Source file '{}' does not exist.", path)); - } - - // Read the source from a binary file instead of sampling from some - // assumed source distribution - write_message(6, "Reading mcpl source file from {}",path); - - // Open the mcpl file - mcpl_file = mcpl_open_file(path.c_str()); - - //do checks on the mcpl_file to see if particles are many enough. - // should model this on the example source shown in the docs. - n_sites=mcpl_hdr_nparticles(mcpl_file); - - read_source_bank(sites_); -} -<<<<<<< HEAD - -MCPLFileSource::~MCPLFileSource(){ - mcpl_close_file(mcpl_file); -} - -SourceSite MCPLFileSource::sample(uint64_t* seed) const -{ - size_t i_site = sites_.size() * prn(seed); - return sites_[i_site]; -} - -void MCPLFileSource::read_source_bank(vector &sites_) -{ - sites_.resize(n_sites); - for (int i=0;ipdgcode; - while ( pdg!=2112 && pdg!=22 && pdg!=11 && pdg!=-11) { - mcpl_particle=mcpl_read(mcpl_file); - pdg=mcpl_particle->pdgcode; - //should check for file exhaustion. This could happen if particles are other than - //neutrons, photons, electrons, or positrons. - } - - switch(pdg){ - case 2112: - omc_particle_.particle=ParticleType::neutron; - break; - case 22: - omc_particle_.particle=ParticleType::photon; - break; - case 11: - omc_particle_.particle=ParticleType::electron; - break; - case -11: - omc_particle_.particle=ParticleType::positron; - break; - } - - //particle is good, convert to openmc-formalism - omc_particle_.r.x=mcpl_particle->position[0]; - omc_particle_.r.y=mcpl_particle->position[1]; - omc_particle_.r.z=mcpl_particle->position[2]; - - omc_particle_.u.x=mcpl_particle->direction[0]; - omc_particle_.u.y=mcpl_particle->direction[1]; - omc_particle_.u.z=mcpl_particle->direction[2]; - - //mcpl stores kinetic energy in MeV - omc_particle_.E=mcpl_particle->ekin*1e6; - //mcpl stores time in ms - omc_particle_.time=mcpl_particle->time*1e-3; - omc_particle_.wgt=mcpl_particle->weight; - omc_particle_.delayed_group=0; - return omc_particle_; -} -#endif //OPENMC_MCPL - -MCPLFileSource::~MCPLFileSource(){ - mcpl_close_file(mcpl_file); -} - -SourceSite MCPLFileSource::sample(uint64_t* seed) const -{ - size_t i_site = sites_.size() * prn(seed); - return sites_[i_site]; -} - -void MCPLFileSource::read_source_bank(vector &sites_) -{ - sites_.resize(n_sites); - for (int i=0;ipdgcode; - while ( pdg!=2112 && pdg!=22 && pdg!=11 && pdg!=-11) { - mcpl_particle=mcpl_read(mcpl_file); - pdg=mcpl_particle->pdgcode; - //should check for file exhaustion. This could happen if particles are other than - //neutrons, photons, electrons, or positrons. - } - - switch(pdg){ - case 2112: - omc_particle_.particle=ParticleType::neutron; - break; - case 22: - omc_particle_.particle=ParticleType::photon; - break; - case 11: - omc_particle_.particle=ParticleType::electron; - break; - case -11: - omc_particle_.particle=ParticleType::positron; - break; - } - - //particle is good, convert to openmc-formalism - omc_particle_.r.x=mcpl_particle->position[0]; - omc_particle_.r.y=mcpl_particle->position[1]; - omc_particle_.r.z=mcpl_particle->position[2]; - - omc_particle_.u.x=mcpl_particle->direction[0]; - omc_particle_.u.y=mcpl_particle->direction[1]; - omc_particle_.u.z=mcpl_particle->direction[2]; - - //mcpl stores kinetic energy in MeV - omc_particle_.E=mcpl_particle->ekin*1e6; - //mcpl stores time in ms - omc_particle_.time=mcpl_particle->time*1e-3; - omc_particle_.wgt=mcpl_particle->weight; - omc_particle_.delayed_group=0; - return omc_particle_; -} -#endif //OPENMC_MCPL //============================================================================== // Non-member functions From 12f088a5ce68c639c9772574eb69c4be5de452fe Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Thu, 6 Oct 2022 01:33:21 +0200 Subject: [PATCH 28/55] must have a full filename --- src/simulation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation.cpp b/src/simulation.cpp index 353ef99265..2c9e7e8566 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -408,7 +408,7 @@ void finalize_batch() } if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ - auto filename = settings::path_output + ".mcpl"; + auto filename = settings::path_output + "surface_source.mcpl"; write_mcpl_source_point(filename.c_str(), true); } From 0706a0eca305c8fc4a0a5cbfb0c5b6e46099753e Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 11 Oct 2022 00:46:32 +0200 Subject: [PATCH 29/55] fix malformed format string --- src/state_point.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index 7c1a454476..e197ee162a 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -625,9 +625,9 @@ void write_mcpl_source_point(const char *filename, bool surf_source_bank) //write_attribute(file_id, "filetype", "source"); //write header stuff (oopy in xml-files as binary blobs for instance)) if (VERSION_DEV){ - line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}-development"); + line=fmt::format("OpenMC {0}.{1}.{2}-development",VERSION_MAJOR,VERSION_MINOR,VERSION_RELEASE); } else { - line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}"); + line=fmt::format("OpenMC {0}.{1}.{2}",VERSION_MAJOR,VERSION_MINOR,VERSION_RELEASE); } mcpl_hdr_set_srcname(file_id,line.c_str()); } From aeadaf0b2f1a59fcd2c3f8cb3ad8fa7dab36e342 Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 11 Oct 2022 00:48:44 +0200 Subject: [PATCH 30/55] fix typo causing unit length errors --- src/state_point.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index e197ee162a..a8a4be034e 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -802,9 +802,11 @@ void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) p.position[1]=_site->r.y; p.position[2]=_site->r.z; + //mcpl requires that the direction vector is unit length + //which is also the case in openmc p.direction[0]=_site->u.x; p.direction[1]=_site->u.y; - p.direction[2]=_site->u.x; + p.direction[2]=_site->u.z; //mcpl stores kinetic energy in MeV p.ekin=_site->E*1e-6; From 72251eb30a996e6aaa9672292a1483c69d53a652 Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 11 Oct 2022 02:00:33 +0200 Subject: [PATCH 31/55] enable mcpl surf_source_write in the python layer --- openmc/settings.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/openmc/settings.py b/openmc/settings.py index 2d81216599..45ff0edc68 100644 --- a/openmc/settings.py +++ b/openmc/settings.py @@ -165,6 +165,7 @@ class Settings: banked (int) :max_particles: Maximum number of particles to be banked on surfaces per process (int) + :mcpl: Output in the form of an MCPL-file (bool) survival_biasing : bool Indicate whether survival biasing is to be used tabular_legendre : dict @@ -643,7 +644,7 @@ class Settings: cv.check_type('surface source writing options', surf_source_write, Mapping) for key, value in surf_source_write.items(): cv.check_value('surface source writing key', key, - ('surface_ids', 'max_particles')) + ('surface_ids', 'max_particles', 'mcpl')) if key == 'surface_ids': cv.check_type('surface ids for source banking', value, Iterable, Integral) @@ -655,6 +656,9 @@ class Settings: value, Integral) cv.check_greater_than('maximum particle banks on surfaces per process', value, 0) + elif key == 'mcpl': + cv.check_type('write to an MCPL-format file', value, bool) + self._surf_source_write = surf_source_write @confidence_intervals.setter @@ -1023,6 +1027,9 @@ class Settings: if 'max_particles' in self._surf_source_write: subelement = ET.SubElement(element, "max_particles") subelement.text = str(self._surf_source_write['max_particles']) + if 'mcpl' in self._surf_source_write: + subelement = ET.SubElement(element, "mcpl") + subelement.text = str(self._surf_source_write['mcpl']).lower() def _create_confidence_intervals(self, root): if self._confidence_intervals is not None: From 5462216deba46d017db7905a312f0ed458cde6a7 Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 6 Sep 2022 02:22:21 +0200 Subject: [PATCH 32/55] wip add MCPL-out to openmc code for writing a source_bank/point compiles but is untested --- include/openmc/state_point.h | 9 +++ src/state_point.cpp | 125 +++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/include/openmc/state_point.h b/include/openmc/state_point.h index 5ada2bf88d..2d92c935a6 100644 --- a/include/openmc/state_point.h +++ b/include/openmc/state_point.h @@ -9,6 +9,10 @@ #include "openmc/particle.h" #include "openmc/vector.h" +#ifdef OPENMC_MCPL +#include +#endif + namespace openmc { void load_state_point(); @@ -21,5 +25,10 @@ void write_tally_results_nr(hid_t file_id); void restart_set_keff(); void write_unstructured_mesh_results(); +#ifdef OPENMC_MCPL +void write_mcpl_source_point(const char *filename_, bool surf_source_bank = false); +void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank); +#endif + } // namespace openmc #endif // OPENMC_STATE_POINT_H diff --git a/src/state_point.cpp b/src/state_point.cpp index 470dd77182..dfbd7f65c5 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -28,6 +28,10 @@ #include "openmc/timer.h" #include "openmc/vector.h" +#ifdef OPENMC_MCPL +#include +#endif + namespace openmc { extern "C" int openmc_statepoint_write(const char* filename, bool* write_source) @@ -599,6 +603,45 @@ void write_source_point(const char* filename, bool surf_source_bank) file_close(file_id); } +void write_mcpl_point(const char *filename, bool surf_source_bank) +{ + std::string filename_; + if (filename) { + filename_ = filename; + } else { + // Determine width for zero padding + int w = std::to_string(settings::n_max_batches).size(); + + filename_ = fmt::format("{0}source.{1:0{2}}.mcpl", settings::path_output, + simulation::current_batch, w); + } + + mcpl_outfile_t file_id; + + std::string line; + if (mpi::master) { + // this must be rewritten: file_open is h5-specific + file_id = mcpl_create_outfile(filename_.c_str()); + //write_attribute(file_id, "filetype", "source"); + //write header stuff (oopy in xml-files as binary blobs for instance)) + if (VERSION_DEV){ + line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}-development"); + } else { + line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}"); + } + mcpl_hdr_set_srcname(file_id,line.c_str()); + } + + write_mcpl_source_bank(file_id, surf_source_bank); + + if (mpi::master) { + //change this - this is h5 specific + mcpl_closeandgzip_outfile(file_id); + } + +} + + void write_source_bank(hid_t group_id, bool surf_source_bank) { hid_t banktype = h5banktype(); @@ -715,6 +758,88 @@ void write_source_bank(hid_t group_id, bool surf_source_bank) H5Tclose(banktype); } +void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) +{ + int64_t dims_size = settings::n_particles; + int64_t count_size = simulation::work_per_rank; + + // Set vectors for source bank and starting bank index of each process + vector* bank_index = &simulation::work_index; + vector* source_bank = &simulation::source_bank; + vector surf_source_index_vector; + vector surf_source_bank_vector; + + if(surf_source_bank) { + surf_source_index_vector = calculate_surf_source_size(); + dims_size = surf_source_index_vector[mpi::n_procs]; + count_size = simulation::surf_source_bank.size(); + + bank_index = &surf_source_index_vector; + + // Copy data in a SharedArray into a vector. + surf_source_bank_vector.resize(count_size); + surf_source_bank_vector.assign(simulation::surf_source_bank.data(), + simulation::surf_source_bank.data() + count_size); + source_bank = &surf_source_bank_vector; + } + + if (mpi::master) { + //write particles from the master node + //loop over the other nodes and receive data - then write those. + for (int i = 0; i < mpi::n_procs; ++i) { +#ifdef OPENMC_MPI + if (i>0) + MPI_Recv(source_bank->data(), count[0], mpi::source_site, i, i, + mpi::intracomm, MPI_STATUS_IGNORE); +#endif + //now write the source_banke data again. + for (vector::iterator _site=source_bank->begin(); _site!=source_bank->end();_site++){ + //particle is now at the iterator + //write it to the mcpl-file + mcpl_particle_t p; + p.position[0]=_site->r.x; + p.position[1]=_site->r.y; + p.position[2]=_site->r.z; + + p.direction[0]=_site->u.x; + p.direction[1]=_site->u.y; + p.direction[2]=_site->u.x; + + //mcpl stores kinetic energy in MeV + p.ekin=_site->E*1e-6; + + p.time=_site->time*1e3; + + p.weight=_site->wgt; + + switch(_site->particle){ + case ParticleType::neutron: + p.pdgcode=2112; + break; + case ParticleType::photon: + p.pdgcode=22; + break; + case ParticleType::electron: + p.pdgcode=11; + break; + case ParticleType::positron: + p.pdgcode=-11; + break; + } + + mcpl_add_particle(file_id,&p); + } + } + } else { +#ifdef OPENMC_MPI + MPI_Send(source_bank->data(), count_size, mpi::source_site, 0, mpi::rank, + mpi::intracomm); +#endif + } + +} + + // Determine member names of a compound HDF5 datatype std::string dtype_member_names(hid_t dtype_id) { From 1519f4f3e6acc5d91dbbb7c952589a2fe39e5537 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Tue, 6 Sep 2022 23:06:19 +0200 Subject: [PATCH 33/55] add a trigger to simulation control --- src/simulation.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/simulation.cpp b/src/simulation.cpp index d7292cb9d9..9c19c2106d 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -407,6 +407,13 @@ void finalize_batch() auto filename = settings::path_output + "surface_source.h5"; write_source_point(filename.c_str(), true); } + + if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ + auto filename = settings::path_output + ".mcpl"; + write_mcplt_source_point(filename.c_str(), true); + } + + } void initialize_generation() From f76e416ee64524ef06705db4f2fa306e4be472f1 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 10:56:36 +0200 Subject: [PATCH 34/55] use find_package instead - MCPL now supports this --- CMakeLists.txt | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 32beb41ecd..24018eb0f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,6 +159,13 @@ if(${HDF5_VERSION} VERSION_GREATER_EQUAL 1.12.0) list(APPEND cxxflags -DH5Oget_info_by_idx_vers=1 -DH5O_info_t_vers=1) endif() +#=============================================================================== +# MCPL +#=============================================================================== +if (OPENMC_USE_MCPL) + find_package(MCPL REQUIRED) +endif() + #=============================================================================== # Set compile/link flags based on which compiler is being used #=============================================================================== @@ -186,10 +193,6 @@ if(OPENMC_ENABLE_COVERAGE) list(APPEND ldflags --coverage) endif() -if(OPENMC_USE_MCPL) - list(APPEND ldflags -lmcpl) -endif() - # Show flags being used message(STATUS "OpenMC C++ flags: ${cxxflags}") message(STATUS "OpenMC Linker flags: ${ldflags}") @@ -444,10 +447,6 @@ endif() if (OPENMC_USE_MPI) target_compile_definitions(libopenmc PUBLIC -DOPENMC_MPI) endif() -if(OPENMC_USE_MCPL) - target_compile_definitions(libopenmc PUBLIC -DOPENMC_MCPL) -endif() - # Set git SHA1 hash as a compile definition if(GIT_FOUND) @@ -491,6 +490,11 @@ if (OPENMC_USE_MPI) target_link_libraries(libopenmc MPI::MPI_CXX) endif() +if (OPENMC_USE_MCPL) + target_compile_definitions(libopenmc PUBLIC OPENMC_MCPL) + target_link_libraries(libopenmc MCPL::mcpl) +endif() + #=============================================================================== # openmc executable #=============================================================================== From b6aa2dc429bf7545826991c426bf0843624c7377 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 10:59:02 +0200 Subject: [PATCH 35/55] add mcpl-output settings --- include/openmc/settings.h | 1 + src/settings.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/include/openmc/settings.h b/include/openmc/settings.h index 1e061b235b..66825867d8 100644 --- a/include/openmc/settings.h +++ b/include/openmc/settings.h @@ -48,6 +48,7 @@ extern bool source_latest; //!< write latest source at each batch? extern bool source_separate; //!< write source to separate file? extern bool source_write; //!< write source in HDF5 files? extern bool surf_source_write; //!< write surface source file? +extern bool surf_mcpl_write; //!< write surface mcpl file? extern bool surf_source_read; //!< read surface source file? extern bool survival_biasing; //!< use survival biasing? extern bool temperature_multipole; //!< use multipole data? diff --git a/src/settings.cpp b/src/settings.cpp index 11dd96e025..d1762596c6 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -61,6 +61,7 @@ bool source_latest {false}; bool source_separate {false}; bool source_write {true}; bool surf_source_write {false}; +bool surf_mcpl_write {false}; bool surf_source_read {false}; bool survival_biasing {false}; bool temperature_multipole {false}; @@ -682,9 +683,14 @@ void read_settings_xml() max_surface_particles = std::stoll(get_node_value(node_ssw, "max_particles")); } +#ifdef OPENMC_MCPL + if(check_for_node(node_ssw, "mcpl")){ + surf_mcpl_write=true; + } +#endif } - // If source is not seperate and is to be written out in the statepoint file, + // If source is not separate and is to be written out in the statepoint file, // make sure that the sourcepoint batch numbers are contained in the // statepoint list if (!source_separate) { From 22ce618601c8dbefd4759c51d50b7a17ba09dbba Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 10:59:28 +0200 Subject: [PATCH 36/55] fix typo --- src/simulation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation.cpp b/src/simulation.cpp index 9c19c2106d..fcb115849d 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -410,7 +410,7 @@ void finalize_batch() if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ auto filename = settings::path_output + ".mcpl"; - write_mcplt_source_point(filename.c_str(), true); + write_mcpl_source_point(filename.c_str(), true); } From 76e726b527c1b738b8b5842999e69ec4d6c4c57d Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 11:00:39 +0200 Subject: [PATCH 37/55] add cond. compilation guards for MCPL functions --- src/state_point.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index dfbd7f65c5..14a5411283 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -603,7 +603,8 @@ void write_source_point(const char* filename, bool surf_source_bank) file_close(file_id); } -void write_mcpl_point(const char *filename, bool surf_source_bank) +#ifdef OPENMC_MCPL +void write_mcpl_source_point(const char *filename, bool surf_source_bank) { std::string filename_; if (filename) { @@ -640,7 +641,7 @@ void write_mcpl_point(const char *filename, bool surf_source_bank) } } - +#endif void write_source_bank(hid_t group_id, bool surf_source_bank) { @@ -758,6 +759,7 @@ void write_source_bank(hid_t group_id, bool surf_source_bank) H5Tclose(banktype); } +#ifdef OPENMC_MCPL void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) { int64_t dims_size = settings::n_particles; @@ -838,7 +840,7 @@ void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) } } - +#endif // Determine member names of a compound HDF5 datatype std::string dtype_member_names(hid_t dtype_id) From bdc76d95fce06f3e94dea3afa9b8305309958179 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Fri, 16 Sep 2022 16:07:22 +0200 Subject: [PATCH 38/55] refactor MCPL file read part of FileSource A new constructor is added to FileSource that reads from a given MCPL-file. --- src/source.cpp | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/src/source.cpp b/src/source.cpp index e43f03929b..cf584e12ea 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -437,6 +437,79 @@ CustomSourceWrapper::~CustomSourceWrapper() } +MCPLFileSource::~MCPLFileSource(){ + mcpl_close_file(mcpl_file); +} + +SourceSite MCPLFileSource::sample(uint64_t* seed) const +{ + size_t i_site = sites_.size() * prn(seed); + return sites_[i_site]; +} + +void MCPLFileSource::read_source_bank(vector &sites_) +{ + sites_.resize(n_sites); + for (int i=0;ipdgcode; + while ( pdg!=2112 && pdg!=22 && pdg!=11 && pdg!=-11) { + mcpl_particle=mcpl_read(mcpl_file); + pdg=mcpl_particle->pdgcode; + //should check for file exhaustion. This could happen if particles are other than + //neutrons, photons, electrons, or positrons. + } + + switch(pdg){ + case 2112: + omc_particle_.particle=ParticleType::neutron; + break; + case 22: + omc_particle_.particle=ParticleType::photon; + break; + case 11: + omc_particle_.particle=ParticleType::electron; + break; + case -11: + omc_particle_.particle=ParticleType::positron; + break; + } + + //particle is good, convert to openmc-formalism + omc_particle_.r.x=mcpl_particle->position[0]; + omc_particle_.r.y=mcpl_particle->position[1]; + omc_particle_.r.z=mcpl_particle->position[2]; + + omc_particle_.u.x=mcpl_particle->direction[0]; + omc_particle_.u.y=mcpl_particle->direction[1]; + omc_particle_.u.z=mcpl_particle->direction[2]; + + //mcpl stores kinetic energy in MeV + omc_particle_.E=mcpl_particle->ekin*1e6; + //mcpl stores time in ms + omc_particle_.time=mcpl_particle->time*1e-3; + omc_particle_.wgt=mcpl_particle->weight; + omc_particle_.delayed_group=0; + return omc_particle_; +} +#endif //OPENMC_MCPL +======= +>>>>>>> 380cfc150b449ef56debb5261caa010a52c42bdd +======= +#endif +>>>>>>> 71568d4f4 (refactor MCPL file read part of FileSource) +>>>>>>> 55f2be19e (refactor MCPL file read part of FileSource) + //============================================================================== // Non-member functions //============================================================================== From 6bceb7c30c05509823a7b0a6cb2e518d3a7ff5e4 Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 6 Sep 2022 02:22:21 +0200 Subject: [PATCH 39/55] wip add MCPL-out to openmc code for writing a source_bank/point compiles but is untested --- src/source.cpp | 6 ------ src/state_point.cpp | 1 + 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/source.cpp b/src/source.cpp index cf584e12ea..99ec3a74d1 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -503,12 +503,6 @@ SourceSite MCPLFileSource::read_single_particle() const return omc_particle_; } #endif //OPENMC_MCPL -======= ->>>>>>> 380cfc150b449ef56debb5261caa010a52c42bdd -======= -#endif ->>>>>>> 71568d4f4 (refactor MCPL file read part of FileSource) ->>>>>>> 55f2be19e (refactor MCPL file read part of FileSource) //============================================================================== // Non-member functions diff --git a/src/state_point.cpp b/src/state_point.cpp index 14a5411283..1b133a1215 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -643,6 +643,7 @@ void write_mcpl_source_point(const char *filename, bool surf_source_bank) } #endif + void write_source_bank(hid_t group_id, bool surf_source_bank) { hid_t banktype = h5banktype(); From 56cddd25e2d8540da5356fbbe8f3bbde13ce11b5 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Tue, 4 Oct 2022 10:11:06 +0200 Subject: [PATCH 40/55] do not define these functions if no MCPL present --- src/state_point.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index 1b133a1215..14a5411283 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -643,7 +643,6 @@ void write_mcpl_source_point(const char *filename, bool surf_source_bank) } #endif - void write_source_bank(hid_t group_id, bool surf_source_bank) { hid_t banktype = h5banktype(); From 3c33ec3b9fc513794e7e683620fbce99aedb9bcb Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 9 Aug 2022 08:12:19 +0200 Subject: [PATCH 41/55] refactor MCPL file read part of FileSource A new constructor is added to FileSource that reads from a given MCPL-file. --- src/source.cpp | 67 -------------------------------------------------- 1 file changed, 67 deletions(-) diff --git a/src/source.cpp b/src/source.cpp index 99ec3a74d1..e43f03929b 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -437,73 +437,6 @@ CustomSourceWrapper::~CustomSourceWrapper() } -MCPLFileSource::~MCPLFileSource(){ - mcpl_close_file(mcpl_file); -} - -SourceSite MCPLFileSource::sample(uint64_t* seed) const -{ - size_t i_site = sites_.size() * prn(seed); - return sites_[i_site]; -} - -void MCPLFileSource::read_source_bank(vector &sites_) -{ - sites_.resize(n_sites); - for (int i=0;ipdgcode; - while ( pdg!=2112 && pdg!=22 && pdg!=11 && pdg!=-11) { - mcpl_particle=mcpl_read(mcpl_file); - pdg=mcpl_particle->pdgcode; - //should check for file exhaustion. This could happen if particles are other than - //neutrons, photons, electrons, or positrons. - } - - switch(pdg){ - case 2112: - omc_particle_.particle=ParticleType::neutron; - break; - case 22: - omc_particle_.particle=ParticleType::photon; - break; - case 11: - omc_particle_.particle=ParticleType::electron; - break; - case -11: - omc_particle_.particle=ParticleType::positron; - break; - } - - //particle is good, convert to openmc-formalism - omc_particle_.r.x=mcpl_particle->position[0]; - omc_particle_.r.y=mcpl_particle->position[1]; - omc_particle_.r.z=mcpl_particle->position[2]; - - omc_particle_.u.x=mcpl_particle->direction[0]; - omc_particle_.u.y=mcpl_particle->direction[1]; - omc_particle_.u.z=mcpl_particle->direction[2]; - - //mcpl stores kinetic energy in MeV - omc_particle_.E=mcpl_particle->ekin*1e6; - //mcpl stores time in ms - omc_particle_.time=mcpl_particle->time*1e-3; - omc_particle_.wgt=mcpl_particle->weight; - omc_particle_.delayed_group=0; - return omc_particle_; -} -#endif //OPENMC_MCPL - //============================================================================== // Non-member functions //============================================================================== From 52cf6823e6c1df7dbcc30b0c860412dc124a18a3 Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 6 Sep 2022 02:22:21 +0200 Subject: [PATCH 42/55] wip add MCPL-out to openmc code for writing a source_bank/point compiles but is untested --- src/state_point.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/state_point.cpp b/src/state_point.cpp index 14a5411283..1b133a1215 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -643,6 +643,7 @@ void write_mcpl_source_point(const char *filename, bool surf_source_bank) } #endif + void write_source_bank(hid_t group_id, bool surf_source_bank) { hid_t banktype = h5banktype(); From 9d7105ebc7ff43d973f55d59a815d62be0a8ba5b Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 6 Sep 2022 02:22:21 +0200 Subject: [PATCH 43/55] wip add MCPL-out to openmc code for writing a source_bank/point compiles but is untested --- src/state_point.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index 1b133a1215..c41c3dd53d 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -603,6 +603,7 @@ void write_source_point(const char* filename, bool surf_source_bank) file_close(file_id); } + #ifdef OPENMC_MCPL void write_mcpl_source_point(const char *filename, bool surf_source_bank) { @@ -641,8 +642,8 @@ void write_mcpl_source_point(const char *filename, bool surf_source_bank) } } -#endif +#endif void write_source_bank(hid_t group_id, bool surf_source_bank) { From cd9eb5b1b237bbd420b7c2eef93023d3e1f1e20b Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Tue, 6 Sep 2022 23:06:19 +0200 Subject: [PATCH 44/55] add a trigger to simulation control --- src/simulation.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/simulation.cpp b/src/simulation.cpp index fcb115849d..7fda9d5239 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -408,10 +408,10 @@ void finalize_batch() write_source_point(filename.c_str(), true); } - if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ - auto filename = settings::path_output + ".mcpl"; - write_mcpl_source_point(filename.c_str(), true); - } +if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ + auto filename = settings::path_output + ".mcpl"; + write_mcpl_source_point(filename.c_str(), true); +} } From 8b783749341798cc9c8bc71d0ab374297824f1fc Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 10:59:28 +0200 Subject: [PATCH 45/55] fix typo --- src/simulation.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/simulation.cpp b/src/simulation.cpp index 7fda9d5239..fcb115849d 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -408,10 +408,10 @@ void finalize_batch() write_source_point(filename.c_str(), true); } -if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ - auto filename = settings::path_output + ".mcpl"; - write_mcpl_source_point(filename.c_str(), true); -} + if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ + auto filename = settings::path_output + ".mcpl"; + write_mcpl_source_point(filename.c_str(), true); + } } From 172fec23568ae6793c2b23c1b54e02c6b01483d7 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 5 Oct 2022 11:00:39 +0200 Subject: [PATCH 46/55] add cond. compilation guards for MCPL functions --- src/state_point.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index c41c3dd53d..14a5411283 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -603,7 +603,6 @@ void write_source_point(const char* filename, bool surf_source_bank) file_close(file_id); } - #ifdef OPENMC_MCPL void write_mcpl_source_point(const char *filename, bool surf_source_bank) { @@ -642,7 +641,6 @@ void write_mcpl_source_point(const char *filename, bool surf_source_bank) } } - #endif void write_source_bank(hid_t group_id, bool surf_source_bank) From 9065bdac3aeafe0d411c2d4a26ef873e39cb4c40 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Fri, 16 Sep 2022 16:07:22 +0200 Subject: [PATCH 47/55] refactor MCPL file read part of FileSource A new constructor is added to FileSource that reads from a given MCPL-file. --- src/source.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/source.cpp b/src/source.cpp index e43f03929b..3e1ddb3f38 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -436,7 +436,6 @@ CustomSourceWrapper::~CustomSourceWrapper() #endif } - //============================================================================== // Non-member functions //============================================================================== From 28cd0c5d3ae44a1ab3fe11f3ced08d6589530aa6 Mon Sep 17 00:00:00 2001 From: erkn Date: Wed, 5 Oct 2022 14:31:05 +0200 Subject: [PATCH 48/55] fix misnamed variable and remove accientally committed code --- src/source.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/source.cpp b/src/source.cpp index 3e1ddb3f38..d3037386b2 100644 --- a/src/source.cpp +++ b/src/source.cpp @@ -350,10 +350,19 @@ FileSource::FileSource(mcpl_file_t mcpl_file) //neutrons or photons } - if(mcpl_particle->pdgcode==2112) { - site_.particle=ParticleType::neutron; - } else if (mcpl_particle->pdgcode==22) { - site_.particle=ParticleType::photon; + switch(pdg){ + case 2112: + site_.particle=ParticleType::neutron; + break; + case 22: + site_.particle=ParticleType::photon; + break; + case 11: + site_.particle=ParticleType::electron; + break; + case -11: + site_.particle=ParticleType::positron; + break; } //particle is good, convert to openmc-formalism From 6e523fa1f8ca0b6eef6a5e110fc7ceb86446d8f6 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Thu, 6 Oct 2022 01:33:21 +0200 Subject: [PATCH 49/55] must have a full filename --- src/simulation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation.cpp b/src/simulation.cpp index fcb115849d..82fce7c221 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -409,7 +409,7 @@ void finalize_batch() } if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ - auto filename = settings::path_output + ".mcpl"; + auto filename = settings::path_output + "surface_source.mcpl"; write_mcpl_source_point(filename.c_str(), true); } From 93a9270efcf0697db410c21d184b4e0d933da875 Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 11 Oct 2022 00:46:32 +0200 Subject: [PATCH 50/55] fix malformed format string --- src/state_point.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index 14a5411283..acae2e8d01 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -626,9 +626,9 @@ void write_mcpl_source_point(const char *filename, bool surf_source_bank) //write_attribute(file_id, "filetype", "source"); //write header stuff (oopy in xml-files as binary blobs for instance)) if (VERSION_DEV){ - line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}-development"); + line=fmt::format("OpenMC {0}.{1}.{2}-development",VERSION_MAJOR,VERSION_MINOR,VERSION_RELEASE); } else { - line=fmt::format("OpenMC {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_RELEASE}"); + line=fmt::format("OpenMC {0}.{1}.{2}",VERSION_MAJOR,VERSION_MINOR,VERSION_RELEASE); } mcpl_hdr_set_srcname(file_id,line.c_str()); } From 59a949de362f5133c43971636bcb71a557ae03e0 Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 11 Oct 2022 00:48:44 +0200 Subject: [PATCH 51/55] fix typo causing unit length errors --- src/state_point.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index acae2e8d01..fb915484b3 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -803,9 +803,11 @@ void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) p.position[1]=_site->r.y; p.position[2]=_site->r.z; + //mcpl requires that the direction vector is unit length + //which is also the case in openmc p.direction[0]=_site->u.x; p.direction[1]=_site->u.y; - p.direction[2]=_site->u.x; + p.direction[2]=_site->u.z; //mcpl stores kinetic energy in MeV p.ekin=_site->E*1e-6; From 7759028d803d052f220abbc00a876a1c33b00949 Mon Sep 17 00:00:00 2001 From: erkn Date: Tue, 11 Oct 2022 02:00:33 +0200 Subject: [PATCH 52/55] enable mcpl surf_source_write in the python layer --- openmc/settings.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/openmc/settings.py b/openmc/settings.py index ad5a9b28ae..0937a3e480 100644 --- a/openmc/settings.py +++ b/openmc/settings.py @@ -171,6 +171,7 @@ class Settings: banked (int) :max_particles: Maximum number of particles to be banked on surfaces per process (int) + :mcpl: Output in the form of an MCPL-file (bool) survival_biasing : bool Indicate whether survival biasing is to be used tabular_legendre : dict @@ -652,7 +653,7 @@ class Settings: cv.check_type('surface source writing options', surf_source_write, Mapping) for key, value in surf_source_write.items(): cv.check_value('surface source writing key', key, - ('surface_ids', 'max_particles')) + ('surface_ids', 'max_particles', 'mcpl')) if key == 'surface_ids': cv.check_type('surface ids for source banking', value, Iterable, Integral) @@ -664,6 +665,9 @@ class Settings: value, Integral) cv.check_greater_than('maximum particle banks on surfaces per process', value, 0) + elif key == 'mcpl': + cv.check_type('write to an MCPL-format file', value, bool) + self._surf_source_write = surf_source_write @confidence_intervals.setter @@ -1032,6 +1036,9 @@ class Settings: if 'max_particles' in self._surf_source_write: subelement = ET.SubElement(element, "max_particles") subelement.text = str(self._surf_source_write['max_particles']) + if 'mcpl' in self._surf_source_write: + subelement = ET.SubElement(element, "mcpl") + subelement.text = str(self._surf_source_write['mcpl']).lower() def _create_confidence_intervals(self, root): if self._confidence_intervals is not None: From e3f1bdaca1f5144563c4ceac45f10e4b0b5b7963 Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 12 Oct 2022 19:30:38 +0200 Subject: [PATCH 53/55] enable mcpl from the python layer if the source filename ends with mcpl use that --- openmc/source.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openmc/source.py b/openmc/source.py index 48e14f6f71..fdb2d5bc25 100644 --- a/openmc/source.py +++ b/openmc/source.py @@ -223,7 +223,10 @@ class Source: if self.particle != 'neutron': element.set("particle", self.particle) if self.file is not None: - element.set("file", self.file) + if (self.file.endswith('.mcpl')): + element.set("mcpl", self.file) + else: + element.set("file", self.file) if self.library is not None: element.set("library", self.library) if self.parameters is not None: From ca396268b586719df5638057b9161dbb2cdb048a Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 12 Oct 2022 19:32:09 +0200 Subject: [PATCH 54/55] protect against compilation wo mcpl --- src/simulation.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/simulation.cpp b/src/simulation.cpp index 82fce7c221..d90c33ffd6 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -407,12 +407,12 @@ void finalize_batch() auto filename = settings::path_output + "surface_source.h5"; write_source_point(filename.c_str(), true); } - +#ifdef OPENMC_MCPL if(settings::surf_mcpl_write && simulation::current_batch == settings::n_batches){ auto filename = settings::path_output + "surface_source.mcpl"; write_mcpl_source_point(filename.c_str(), true); } - +#endif } From 94fb5f8fa6818f88c9fcf9007535a80846cb6c2d Mon Sep 17 00:00:00 2001 From: Erik B Knudsen Date: Wed, 12 Oct 2022 19:32:40 +0200 Subject: [PATCH 55/55] enable MPI mpi-nodes are gathered sequentially (as for h5) --- src/state_point.cpp | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/state_point.cpp b/src/state_point.cpp index fb915484b3..983147c62e 100644 --- a/src/state_point.cpp +++ b/src/state_point.cpp @@ -786,30 +786,40 @@ void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) } if (mpi::master) { - //write particles from the master node + // Particles are writeen to disk from the master node only + + // Save source bank sites since the array is overwritten below +#ifdef OPENMC_MPI + vector temp_source {source_bank->begin(), source_bank->end()}; +#endif + //loop over the other nodes and receive data - then write those. for (int i = 0; i < mpi::n_procs; ++i) { + // number of particles for node node i + size_t count[] { + static_cast((*bank_index)[i + 1] - (*bank_index)[i])}; + #ifdef OPENMC_MPI if (i>0) MPI_Recv(source_bank->data(), count[0], mpi::source_site, i, i, mpi::intracomm, MPI_STATUS_IGNORE); #endif - //now write the source_banke data again. + // now write the source_bank data again. for (vector::iterator _site=source_bank->begin(); _site!=source_bank->end();_site++){ - //particle is now at the iterator - //write it to the mcpl-file + // particle is now at the iterator + // write it to the mcpl-file mcpl_particle_t p; p.position[0]=_site->r.x; p.position[1]=_site->r.y; p.position[2]=_site->r.z; - //mcpl requires that the direction vector is unit length - //which is also the case in openmc + // mcpl requires that the direction vector is unit length + // which is also the case in openmc p.direction[0]=_site->u.x; p.direction[1]=_site->u.y; p.direction[2]=_site->u.z; - //mcpl stores kinetic energy in MeV + // mcpl stores kinetic energy in MeV p.ekin=_site->E*1e-6; p.time=_site->time*1e3; @@ -834,6 +844,10 @@ void write_mcpl_source_bank(mcpl_outfile_t file_id, bool surf_source_bank) mcpl_add_particle(file_id,&p); } } +#ifdef OPENMC_MPI + // Restore state of source bank + std::copy(temp_source.begin(), temp_source.end(), source_bank->begin()); +#endif } else { #ifdef OPENMC_MPI MPI_Send(source_bank->data(), count_size, mpi::source_site, 0, mpi::rank,