diff --git a/include/openmc/capi.h b/include/openmc/capi.h index 8e9edc4a03..5aba984532 100644 --- a/include/openmc/capi.h +++ b/include/openmc/capi.h @@ -48,6 +48,7 @@ extern "C" { int64_t openmc_get_seed(); int openmc_get_tally_index(int32_t id, int32_t* index); void openmc_get_tally_next_id(int32_t* id); + int openmc_global_tallies(double** ptr); int openmc_hard_reset(); int openmc_init(int argc, char* argv[], const void* intracomm); int openmc_init_f(const int* intracomm); diff --git a/include/openmc/simulation.h b/include/openmc/simulation.h index 564516e602..ceb5eb92a5 100644 --- a/include/openmc/simulation.h +++ b/include/openmc/simulation.h @@ -50,14 +50,19 @@ extern "C" int thread_id; //!< ID of a given thread // Functions //============================================================================== +//! Determine number of particles to transport per process +void calculate_work(); + //! Initialize simulation extern "C" void openmc_simulation_init_c(); //! Determine overall generation number extern "C" int overall_generation(); -//! Determine number of particles to transport per process -void calculate_work(); +#ifdef OPENMC_MPI +extern "C" void broadcast_results(); +extern "C" void broadcast_triggers(); +#endif } // namespace openmc diff --git a/src/simulation.F90 b/src/simulation.F90 index b1087e21d2..c4158062eb 100644 --- a/src/simulation.F90 +++ b/src/simulation.F90 @@ -330,6 +330,11 @@ contains #endif character(MAX_FILE_LEN) :: filename + interface + subroutine broadcast_triggers() bind(C) + end subroutine broadcast_triggers + end interface + ! Reduce tallies onto master process and accumulate call time_tallies % start() call accumulate_tallies() @@ -351,8 +356,7 @@ contains ! Check_triggers if (master) call check_triggers() #ifdef OPENMC_MPI - call MPI_BCAST(satisfy_triggers, 1, MPI_LOGICAL, 0, & - mpi_intracomm, mpi_err) + call broadcast_triggers() #endif if (satisfy_triggers .or. & (trigger_on .and. current_batch == n_max_batches)) then @@ -488,6 +492,9 @@ contains interface subroutine print_overlap_check() bind(C) end subroutine print_overlap_check + + subroutine broadcast_results() bind(C) + end subroutine broadcast_results end interface err = 0 @@ -515,37 +522,7 @@ contains total_gen = total_gen + current_batch*gen_per_batch #ifdef OPENMC_MPI - ! Broadcast tally results so that each process has access to results - if (allocated(tallies)) then - do i = 1, size(tallies) - associate (results => tallies(i) % obj % results) - ! Create a new datatype that consists of all values for a given filter - ! bin and then use that to broadcast. This is done to minimize the - ! chance of the 'count' argument of MPI_BCAST exceeding 2**31 - n = size(results, 3) - count_per_filter = size(results, 1) * size(results, 2) - call MPI_TYPE_CONTIGUOUS(count_per_filter, MPI_DOUBLE, & - result_block, mpi_err) - call MPI_TYPE_COMMIT(result_block, mpi_err) - call MPI_BCAST(results, n, result_block, 0, mpi_intracomm, mpi_err) - call MPI_TYPE_FREE(result_block, mpi_err) - end associate - end do - end if - - ! Also broadcast global tally results - n = size(global_tallies) - call MPI_BCAST(global_tallies, n, MPI_DOUBLE, 0, mpi_intracomm, mpi_err) - - ! These guys are needed so that non-master processes can calculate the - ! combined estimate of k-effective - tempr(1) = k_col_abs - tempr(2) = k_col_tra - tempr(3) = k_abs_tra - call MPI_BCAST(tempr, 3, MPI_REAL8, 0, mpi_intracomm, mpi_err) - k_col_abs = tempr(1) - k_col_tra = tempr(2) - k_abs_tra = tempr(3) + call broadcast_results() #endif ! Write tally results to tallies.out diff --git a/src/simulation.cpp b/src/simulation.cpp index 98c3d3677c..249dd090a1 100644 --- a/src/simulation.cpp +++ b/src/simulation.cpp @@ -4,6 +4,12 @@ #include "openmc/message_passing.h" #include "openmc/settings.h" +#ifdef OPENMC_MPI +#include +#endif + +#include + // OPENMC_RUN encompasses all the main logic where iterations are performed // over the batches, generations, and histories in a fixed source or k-eigenvalue // calculation. @@ -98,4 +104,46 @@ void calculate_work() } } +#ifdef OPENMC_MPI +void broadcast_results() { + // Broadcast tally results so that each process has access to results + double* buffer; + if (n_tallies > 0) { + for (int i=1; i <= n_tallies; ++i) { + // Create a new datatype that consists of all values for a given filter + // bin and then use that to broadcast. This is done to minimize the + // chance of the 'count' argument of MPI_BCAST exceeding 2**31 + int shape[3]; + openmc_tally_results(i, &buffer, shape); + + int n = shape[0]; + int count_per_filter = shape[1] * shape[2]; + MPI_Datatype result_block; + MPI_Type_contiguous(count_per_filter, MPI_DOUBLE, &result_block); + MPI_Type_commit(&result_block); + MPI_Bcast(buffer, n, result_block, 0, mpi::intracomm); + MPI_Type_free(&result_block); + } + } + + // Also broadcast global tally results + openmc_global_tallies(&buffer); + MPI_Bcast(buffer, 4, MPI_DOUBLE, 0, mpi::intracomm); + + // These guys are needed so that non-master processes can calculate the + // combined estimate of k-effective + double temp[] {simulation::k_col_abs, simulation::k_col_tra, + simulation::k_abs_tra}; + MPI_Bcast(temp, 3, MPI_DOUBLE, 0, mpi::intracomm); + simulation::k_col_abs = temp[0]; + simulation::k_col_tra = temp[1]; + simulation::k_abs_tra = temp[2]; +} + +void broadcast_triggers() +{ + MPI_Bcast(&simulation::satisfy_triggers, 1, MPI_C_BOOL, 0, mpi::intracomm); +} +#endif + } // namespace openmc