diff --git a/src/api.F90 b/src/api.F90 index 2c1fc968b..b767d1ab4 100644 --- a/src/api.F90 +++ b/src/api.F90 @@ -92,11 +92,11 @@ contains integer(C_INT32_T), optional, intent(in) :: instance integer(C_INT) :: err - integer :: i, n + integer :: n err = E_UNASSIGNED if (index >= 1 .and. index <= size(cells)) then - associate (c => cells(i)) + associate (c => cells(index)) if (allocated(c % sqrtkT)) then n = size(c % sqrtkT) if (present(instance) .and. n > 1) then @@ -122,7 +122,7 @@ contains subroutine openmc_finalize() bind(C) - integer :: hdf5_err + integer :: err ! Clear results call openmc_reset() @@ -190,14 +190,14 @@ contains call free_memory() ! Release compound datatypes - call h5tclose_f(hdf5_bank_t, hdf5_err) + call h5tclose_f(hdf5_bank_t, err) ! Close FORTRAN interface. - call h5close_f(hdf5_err) + call h5close_f(err) #ifdef MPI ! Free all MPI types - call MPI_TYPE_FREE(MPI_BANK, mpi_err) + call MPI_TYPE_FREE(MPI_BANK, err) #endif end subroutine openmc_finalize diff --git a/src/cmfd_execute.F90 b/src/cmfd_execute.F90 index 0d4a9f4d3..e87bff930 100644 --- a/src/cmfd_execute.F90 +++ b/src/cmfd_execute.F90 @@ -108,6 +108,9 @@ contains real(8) :: hxyz(3) ! cell dimensions of current ijk cell real(8) :: vol ! volume of cell real(8),allocatable :: source(:,:,:,:) ! tmp source array for entropy +#ifdef MPI + integer :: mpi_err ! MPI error code +#endif ! Get maximum of spatial and group indices nx = cmfd % indices(1) @@ -232,6 +235,9 @@ contains logical :: outside ! any source sites outside mesh logical :: in_mesh ! source site is inside mesh type(RegularMesh), pointer :: m ! point to mesh +#ifdef MPI + integer :: mpi_err +#endif ! Associate pointer m => meshes(n_user_meshes + 1) diff --git a/src/eigenvalue.F90 b/src/eigenvalue.F90 index 9f4da7c82..accf80b31 100644 --- a/src/eigenvalue.F90 +++ b/src/eigenvalue.F90 @@ -40,6 +40,7 @@ contains & temp_sites(:) ! local array of extra sites on each node #ifdef MPI + integer :: mpi_err ! MPI error code integer(8) :: n ! number of sites to send/recv integer :: neighbor ! processor to send/recv data from #ifdef MPIF08 @@ -372,6 +373,9 @@ contains subroutine calculate_generation_keff() integer :: i ! overall generation +#ifdef MPI + integer :: mpi_err ! MPI error code +#endif ! Get keff for this generation by subtracting off the starting value keff_generation = global_tallies(RESULT_VALUE, K_TRACKLENGTH) - keff_generation @@ -614,6 +618,7 @@ contains logical :: sites_outside ! were there sites outside the ufs mesh? #ifdef MPI integer :: n ! total number of ufs mesh cells + integer :: mpi_err ! MPI error code #endif if (current_batch == 1 .and. current_gen == 1) then diff --git a/src/initialize.F90 b/src/initialize.F90 index 69479907e..1e29c638c 100644 --- a/src/initialize.F90 +++ b/src/initialize.F90 @@ -172,6 +172,7 @@ contains integer, intent(in) :: intracomm ! MPI intracommunicator #endif + integer :: mpi_err ! MPI error code integer :: bank_blocks(5) ! Count for each datatype #ifdef MPIF08 type(MPI_Datatype) :: bank_types(5) diff --git a/src/main.F90 b/src/main.F90 index b8c96927e..fadf9f3bb 100644 --- a/src/main.F90 +++ b/src/main.F90 @@ -9,6 +9,10 @@ program main implicit none +#ifdef MPI + integer :: mpi_err ! MPI error code +#endif + ! Initialize run -- when run with MPI, pass communicator #ifdef MPI #ifdef MPIF08 diff --git a/src/mesh.F90 b/src/mesh.F90 index 427710790..c25e3f7a6 100644 --- a/src/mesh.F90 +++ b/src/mesh.F90 @@ -144,6 +144,9 @@ contains integer :: ijk(3) ! indices on mesh integer :: n ! number of energy groups / size integer :: e_bin ! energy_bin +#ifdef MPI + integer :: mpi_err ! MPI error code +#endif logical :: in_mesh ! was single site outside mesh? logical :: outside ! was any site outside mesh? diff --git a/src/message_passing.F90 b/src/message_passing.F90 index 6c13b00e8..0f2b94d1a 100644 --- a/src/message_passing.F90 +++ b/src/message_passing.F90 @@ -16,7 +16,6 @@ module message_passing integer :: rank = 0 ! rank of process logical :: master = .true. ! master process? logical :: mpi_enabled = .false. ! is MPI in use and initialized? - integer :: mpi_err ! MPI error code #ifdef MPIF08 type(MPI_Datatype) :: MPI_BANK ! MPI datatype for fission bank type(MPI_Comm) :: mpi_intracomm ! MPI intra-communicator diff --git a/src/simulation.F90 b/src/simulation.F90 index 8a32493ca..b406679ce 100644 --- a/src/simulation.F90 +++ b/src/simulation.F90 @@ -285,6 +285,10 @@ contains subroutine finalize_batch() +#ifdef MPI + integer :: mpi_err ! MPI error code +#endif + ! Reduce tallies onto master process and accumulate call time_tallies % start() call accumulate_tallies() @@ -392,7 +396,11 @@ contains integer :: i ! loop index for tallies integer :: n ! size of arrays - real(8) :: temp(3) ! temporary array for communication +#ifdef MPI + integer :: mpi_err ! MPI error code + integer(8) :: temp + real(8) :: tempr(3) ! temporary array for communication +#endif !$omp parallel deallocate(micro_xs) @@ -420,18 +428,23 @@ contains ! These guys are needed so that non-master processes can calculate the ! combined estimate of k-effective - temp(1) = k_col_abs - temp(2) = k_col_tra - temp(3) = k_abs_tra - call MPI_BCAST(temp, 3, MPI_REAL8, 0, mpi_intracomm, mpi_err) - k_col_abs = temp(1) - k_col_tra = temp(2) - k_abs_tra = temp(3) + tempr(1) = k_col_abs + tempr(2) = k_col_tra + tempr(3) = k_abs_tra + call MPI_BCAST(tempr, 3, MPI_REAL8, 0, mpi_intracomm, mpi_err) + k_col_abs = tempr(1) + k_col_tra = tempr(2) + k_abs_tra = tempr(3) + + if (check_overlaps) then + call MPI_REDUCE(overlap_check_cnt, temp, n_cells, MPI_INTEGER8, & + MPI_SUM, 0, mpi_intracomm, mpi_err) + overlap_check_cnt = temp + end if #endif ! Write tally results to tallies.out if (output_tallies .and. master) call write_tallies() - if (check_overlaps) call reduce_overlap_count() ! Stop timers and show timing statistics call time_finalize%stop() @@ -444,20 +457,4 @@ contains end subroutine finalize_simulation -!=============================================================================== -! REDUCE_OVERLAP_COUNT accumulates cell overlap check counts to master -!=============================================================================== - - subroutine reduce_overlap_count() - - integer(8) :: temp - -#ifdef MPI - call MPI_REDUCE(overlap_check_cnt, temp, n_cells, MPI_INTEGER8, & - MPI_SUM, 0, mpi_intracomm, mpi_err) - overlap_check_cnt = temp -#endif - - end subroutine reduce_overlap_count - end module simulation diff --git a/src/state_point.F90 b/src/state_point.F90 index 301a4efdd..e2a476e7e 100644 --- a/src/state_point.F90 +++ b/src/state_point.F90 @@ -522,7 +522,8 @@ contains real(8), allocatable :: tally_temp(:,:,:) ! contiguous array of results real(8), target :: global_temp(3,N_GLOBAL_TALLIES) #ifdef MPI - real(8) :: dummy ! temporary receive buffer for non-root reduces + integer :: mpi_err ! MPI error code + real(8) :: dummy ! temporary receive buffer for non-root reduces #endif type(TallyObject) :: dummy_tally @@ -841,6 +842,7 @@ contains #else integer :: i #ifdef MPI + integer :: mpi_err ! MPI error code type(Bank), allocatable, target :: temp_source(:) #endif #endif diff --git a/src/tally.F90 b/src/tally.F90 index 8b52ad032..cf62412f0 100644 --- a/src/tally.F90 +++ b/src/tally.F90 @@ -4246,9 +4246,10 @@ contains subroutine reduce_tally_results() integer :: i - integer :: n ! number of filter bins - integer :: m ! number of score bins - integer :: n_bins ! total number of bins + integer :: n ! number of filter bins + integer :: m ! number of score bins + integer :: n_bins ! total number of bins + integer :: mpi_err ! MPI error code real(C_DOUBLE), allocatable :: tally_temp(:,:) ! contiguous array of results real(C_DOUBLE), allocatable :: tally_temp2(:,:) ! reduced contiguous results real(C_DOUBLE) :: temp(N_GLOBAL_TALLIES), temp2(N_GLOBAL_TALLIES) diff --git a/src/volume_calc.F90 b/src/volume_calc.F90 index 3659293b3..f8cf4085a 100644 --- a/src/volume_calc.F90 +++ b/src/volume_calc.F90 @@ -132,6 +132,7 @@ contains integer :: min_samples ! minimum number of samples per process integer :: remainder ! leftover samples from uneven divide #ifdef MPI + integer :: mpi_err ! MPI error code integer :: m ! index over materials integer :: n ! number of materials integer, allocatable :: data(:) ! array used to send number of hits