Updates to get the timing of variable addition and score setting right for libmesh.

CMakeLists.txt

@@ -415,8 +415,12 @@ if(dagmc)
 endif()
 
 if(libmesh)
+if (debug)
+  target_link_libraries(libopenmc PkgConfig::LIBMESH_DEVEL)
+else()
   target_link_libraries(libopenmc PkgConfig::LIBMESH)
 endif()
+endif()
 
 #===============================================================================
 # openmc executable

cmake/Modules/FindLIBMESH.cmake

@@ -1,11 +1,5 @@
-# Try to find LIBMESH
-#
-# Once done this will define
-#
-#  LIBMESH_FOUND - system has LIBMESH
-#  LIBMESH_INCLUDE_DIRS - the LIBMESH include directory
-#  LIBMESH_LIBRARIES - Link these to use LIBMESH
-#  LIBMESH_DEFINITIONS - Compiler switches required for using LIBMESH
+# Finds the libMesh installation using CMake's PkgConfig
+# module and creates a libmesh imported target
 
 find_path(LIBMESH_PC NAMES libmesh-opt.pc
           HINTS ${LIBMESH_ROOT} $ENV{LIBMESH_ROOT}
@@ -16,5 +10,6 @@ find_path(LIBMESH_PC NAMES libmesh-opt.pc
 include(FindPkgConfig)
 set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${LIBMESH_PC}")
 pkg_check_modules(LIBMESH REQUIRED libmesh IMPORTED_TARGET)
+pkg_check_modules(LIBMESH_DEVEL REQUIRED libmesh-devel IMPORTED_TARGET)
 
 message(STATUS "Found LIBMESH in ${LIBMESH_PC}")

src/finalize.cpp

@@ -68,10 +68,6 @@ int openmc_finalize()
   // Reset timers
   reset_timers();
 
-#ifdef LIBMESH
-  settings::LMI.reset();
-#endif
-
   // Reset global variables
   settings::assume_separate = false;
   settings::check_overlaps = false;
@@ -141,6 +137,10 @@ int openmc_finalize()
   // Deallocate arrays
   free_memory();
 
+#ifdef LIBMESH
+  settings::LMI.reset();
+#endif
+
   // Free all MPI types
 #ifdef OPENMC_MPI
   if (mpi::bank != MPI_DATATYPE_NULL) MPI_Type_free(&mpi::bank);

src/initialize.cpp

@@ -67,12 +67,8 @@ int openmc_init(int argc, char* argv[], const void* intracomm)
   int n_threads = 1;
 #endif
 
-  // initialize libmesh
-#ifdef OPENMC_MPI
-  if (!settings::LMI) settings::LMI = std::make_unique<libMesh::LibMeshInit>(argc, argv, mpi::intracomm, n_threads);
-#else
-  if (!settings::LMI) settings::LMI = std::make_unique<libMesh::LibMeshInit>(argc, argv, n_threads);
-#endif
+// initialize libmesh
+if (!settings::LMI) settings::LMI = std::make_unique<libMesh::LibMeshInit>(argc, argv, n_threads);
 
 #endif
 

src/mesh.cpp

@@ -2238,12 +2238,11 @@ LibMesh::surface_bins_crossed(const Particle* p,
 void
 LibMesh::add_score(const std::string& var_name) {
   // check if this is a new varaible
-  std::string value_name = var_name + "_value";
+  std::string value_name = var_name + "_mean";
   if (!variable_map_.count(value_name)) {
     auto& eqn_sys = equation_systems_->get_system(eq_system_name_);
     auto var_num = eqn_sys.add_variable(value_name, libMesh::CONSTANT, libMesh::MONOMIAL);
     variable_map_[value_name] = var_num;
-    equation_systems_->init();
   }
 
   std::string std_dev_name = var_name + "_std_dev";
@@ -2252,7 +2251,6 @@ LibMesh::add_score(const std::string& var_name) {
     auto& eqn_sys = equation_systems_->get_system(eq_system_name_);
     auto var_num = eqn_sys.add_variable(std_dev_name, libMesh::CONSTANT, libMesh::MONOMIAL);
     variable_map_[std_dev_name] = var_num;
-    equation_systems_->init();
   }
 }
 
@@ -2261,15 +2259,18 @@ LibMesh::set_score_data(const std::string& var_name,
                       std::vector<double> values,
                       std::vector<double> std_dev)
 {
-    auto& eqn_sys = equation_systems_->get_system(eq_system_name_);
-    const libMesh::DofMap& dof_map = eqn_sys.get_dof_map();
+  auto& eqn_sys = equation_systems_->get_system(eq_system_name_);
 
-    // look up the value variable
-    std::string value_name = var_name + "_value";
-    unsigned int value_num = variable_map_.at(value_name);
-    // look up the std dev variable
-    std::string std_dev_name = var_name + "_std_dev";
-    unsigned int std_dev_num = variable_map_.at(std_dev_name);
+  if (!eqn_sys.is_initialized()) { equation_systems_->init(); }
+
+  const libMesh::DofMap& dof_map = eqn_sys.get_dof_map();
+
+  // look up the value variable
+  std::string value_name = var_name + "_mean";
+  unsigned int value_num = variable_map_.at(value_name);
+  // look up the std dev variable
+  std::string std_dev_name = var_name + "_std_dev";
+  unsigned int std_dev_num = variable_map_.at(std_dev_name);
 
   for (int bin = 0; bin < this->n_bins(); bin++) {
     auto e = m_->elem_ptr(bin);
@@ -2290,6 +2291,7 @@ LibMesh::set_score_data(const std::string& var_name,
 }
 
 void LibMesh::write(std::string filename) const {
+  std::cout << "Writing file : " << filename + ".e" << std::endl;
   libMesh::ExodusII_IO exo(*m_);
   std::set<std::string> systems_out = {eq_system_name_};
   exo.write_discontinuous_exodusII(filename + ".e", *equation_systems_, &systems_out);

src/state_point.cpp

@@ -799,6 +799,20 @@ void write_unstructured_mesh_results() {
 
       int n_realizations = tally->n_realizations_;
 
+      for (int i_score = 0; i_score < tally->scores_.size(); i_score++) {
+        for (int i_nuc = 0; i_nuc < tally->nuclides_.size(); i_nuc++) {
+          // generate a name for the value
+          std::string nuclide_name = "total"; // start with total by default
+          if (tally->nuclides_[i_nuc] > -1) {
+            nuclide_name = data::nuclides[tally->nuclides_[i_nuc]]->name_;
+          }
+
+          std::string score_name = tally->score_name(i_score);
+          auto score_str = fmt::format("{}_{}", score_name, nuclide_name);
+          umesh->add_score(score_str);
+        }
+      }
+
       // write each score/nuclide combination for this tally
       for (int i_score = 0; i_score < tally->scores_.size(); i_score++) {
         for (int i_nuc = 0; i_nuc < tally->nuclides_.size(); i_nuc++) {
@@ -828,10 +842,10 @@ void write_unstructured_mesh_results() {
           if (tally->nuclides_[i_nuc] > -1) {
             nuclide_name = data::nuclides[tally->nuclides_[i_nuc]]->name_;
           }
-
           std::string score_name = tally->score_name(i_score);
           auto score_str = fmt::format("{}_{}", score_name, nuclide_name);
           tally_scores.push_back(score_str);
+          // set the data for this score
           umesh->set_score_data(score_str, mean_vec, std_dev_vec);
         }
       }