From 85adf9014ba055b291de8c0be5dcda8d2a592e20 Mon Sep 17 00:00:00 2001 From: Paul Romano Date: Mon, 3 Oct 2011 12:36:14 -0400 Subject: [PATCH] Tallies now work with MPI. Moved copy_from_bank and initialize_particle subroutines out of the source module to prevent circular dependencies. --- src/DEPENDENCIES | 6 ++- src/global.f90 | 9 ++-- src/mpi_routines.f90 | 92 ++++++++++++++++++++++++++++++++++++++--- src/particle_header.f90 | 33 +++++++++++++++ src/source.f90 | 63 +--------------------------- src/tally.f90 | 6 +++ 6 files changed, 136 insertions(+), 73 deletions(-) diff --git a/src/DEPENDENCIES b/src/DEPENDENCIES index 2cee0f12c9..f50be249cc 100644 --- a/src/DEPENDENCIES +++ b/src/DEPENDENCIES @@ -116,7 +116,8 @@ mpi_routines.o: error.o mpi_routines.o: global.o mpi_routines.o: mcnp_random.o mpi_routines.o: output.o -mpi_routines.o: source.o +mpi_routines.o: particle_header.o +mpi_routines.o: tally_header.o output.o: constants.o output.o: datatypes.o @@ -127,6 +128,8 @@ output.o: mesh_header.o output.o: string.o output.o: tally_header.o +particle_header.o: constants.o + physics.o: constants.o physics.o: cross_section_header.o physics.o: endf.o @@ -163,6 +166,7 @@ tally.o: cross_section.o tally.o: error.o tally.o: global.o tally.o: mesh.o +tally.o: mpi_routines.o tally.o: output.o tally.o: search.o tally.o: string.o diff --git a/src/global.f90 b/src/global.f90 index f3c74cdf13..7e58b475f5 100644 --- a/src/global.f90 +++ b/src/global.f90 @@ -82,10 +82,11 @@ module global ! Source and fission bank type(Particle), allocatable, target :: source_bank(:) type(Bank), allocatable, target :: fission_bank(:) - integer(8) :: n_bank ! # of sites in fission bank - integer(8) :: bank_first ! index of first particle in bank - integer(8) :: bank_last ! index of last particle in bank - integer(8) :: work ! number of particles per processor + integer(8) :: n_bank ! # of sites in fission bank + integer(8) :: bank_first ! index of first particle in bank + integer(8) :: bank_last ! index of last particle in bank + integer(8) :: work ! number of particles per processor + integer(8) :: source_index ! index for source particles ! cycle keff real(8) :: keff diff --git a/src/mpi_routines.f90 b/src/mpi_routines.f90 index 6eb0b519d9..de7b51b7ff 100644 --- a/src/mpi_routines.f90 +++ b/src/mpi_routines.f90 @@ -1,11 +1,12 @@ module mpi_routines - use constants, only: MAX_LINE_LEN - use error, only: fatal_error + use constants, only: MAX_LINE_LEN + use error, only: fatal_error use global - use mcnp_random, only: rang, RN_init_particle, RN_skip - use output, only: message - use source, only: copy_from_bank, source_index + use mcnp_random, only: rang, RN_init_particle, RN_skip + use output, only: message + use particle_header, only: Particle, initialize_particle + use tally_header, only: TallyObject #ifdef MPI use mpi @@ -332,5 +333,84 @@ contains deallocate(temp_sites) end subroutine synchronize_bank - + +!=============================================================================== +! COPY_FROM_BANK +!=============================================================================== + + subroutine copy_from_bank(temp_bank, index, n_sites) + + integer, intent(in) :: n_sites ! # of bank sites to copy + type(Bank), intent(in) :: temp_bank(n_sites) + integer, intent(in) :: index ! starting index in source_bank + + integer :: i ! index in temp_bank + integer :: index_source ! index in source_bank + type(Particle), pointer :: p + + do i = 1, n_sites + index_source = index + i - 1 + p => source_bank(index_source) + + p % xyz = temp_bank(i) % xyz + p % xyz_local = temp_bank(i) % xyz + p % uvw = temp_bank(i) % uvw + p % E = temp_bank(i) % E + + ! set defaults + call initialize_particle(p) + + end do + + end subroutine copy_from_bank + +!=============================================================================== +! REDUCE_TALLIES collects all the results from tallies onto one processor +!=============================================================================== + +#ifdef MPI + subroutine reduce_tallies() + + integer :: i + integer :: n + integer :: m + integer :: count + integer :: ierr + real(8), allocatable :: tally_temp(:,:) + type(TallyObject), pointer :: t + + do i = 1, n_tallies + t => tallies(i) + + n = t % n_total_bins + m = t % n_macro_bins + count = n*m + + allocate(tally_temp(n,m)) + + tally_temp = t % scores(:,:) % val_history + + if (master) then + ! Description of MPI_IN_PLANE + call MPI_REDUCE(MPI_IN_PLACE, tally_temp, count, MPI_REAL8, MPI_SUM, & + 0, MPI_COMM_WORLD, ierr) + + ! Transfer values to val_history on master + t % scores(:,:) % val_history = tally_temp + else + ! Receive buffer not significant at other processors + call MPI_REDUCE(tally_temp, tally_temp, count, MPI_REAL8, MPI_SUM, & + 0, MPI_COMM_WORLD, ierr) + + ! Reset val_history on other processors + t % scores(:,:) % val_history = 0 + end if + + deallocate(tally_temp) + + end do + + end subroutine reduce_tallies +#endif + end module mpi_routines diff --git a/src/particle_header.f90 b/src/particle_header.f90 index ba1da0d403..4bf53c470e 100644 --- a/src/particle_header.f90 +++ b/src/particle_header.f90 @@ -1,5 +1,7 @@ module particle_header + use constants, only: NEUTRON, ONE + implicit none !=============================================================================== @@ -44,4 +46,35 @@ module particle_header end type Particle +contains + +!=============================================================================== +! INITIALIZE_PARTICLE sets default attributes for a particle from the source +! bank +!=============================================================================== + + subroutine initialize_particle(p) + + type(Particle), pointer :: p + + ! TODO: if information on the cell, lattice, universe, and material is + ! passed through the fission bank to the source bank, no lookup would be + ! needed at the beginning of a cycle + + p % type = NEUTRON + p % wgt = ONE + p % alive = .true. + p % cell = 0 + p % cell_born = 0 + p % universe = 0 + p % lattice = 0 + p % surface = 0 + p % material = 0 + p % last_material = 0 + p % index_x = 0 + p % index_y = 0 + p % n_collision = 0 + + end subroutine initialize_particle + end module particle_header diff --git a/src/source.f90 b/src/source.f90 index 1744de30f8..19daea98d1 100644 --- a/src/source.f90 +++ b/src/source.f90 @@ -6,13 +6,11 @@ module source use global use mcnp_random, only: rang, RN_init_particle use output, only: message - use particle_header, only: Particle + use particle_header, only: Particle, initialize_particle use physics, only: watt_spectrum implicit none - integer(8) :: source_index - contains !=============================================================================== @@ -125,63 +123,4 @@ contains end function get_source_particle -!=============================================================================== -! COPY_FROM_BANK -!=============================================================================== - - subroutine copy_from_bank(temp_bank, index, n_sites) - - integer, intent(in) :: n_sites ! # of bank sites to copy - type(Bank), intent(in) :: temp_bank(n_sites) - integer, intent(in) :: index ! starting index in source_bank - - integer :: i ! index in temp_bank - integer :: index_source ! index in source_bank - type(Particle), pointer :: p - - do i = 1, n_sites - index_source = index + i - 1 - p => source_bank(index_source) - - p % xyz = temp_bank(i) % xyz - p % xyz_local = temp_bank(i) % xyz - p % uvw = temp_bank(i) % uvw - p % E = temp_bank(i) % E - - ! set defaults - call initialize_particle(p) - - end do - - end subroutine copy_from_bank - -!=============================================================================== -! INITIALIZE_PARTICLE sets default attributes for a particle from the source -! bank -!=============================================================================== - - subroutine initialize_particle(p) - - type(Particle), pointer :: p - - ! TODO: if information on the cell, lattice, universe, and material is - ! passed through the fission bank to the source bank, no lookup would be - ! needed at the beginning of a cycle - - p % type = NEUTRON - p % wgt = ONE - p % alive = .true. - p % cell = 0 - p % cell_born = 0 - p % universe = 0 - p % lattice = 0 - p % surface = 0 - p % material = 0 - p % last_material = 0 - p % index_x = 0 - p % index_y = 0 - p % n_collision = 0 - - end subroutine initialize_particle - end module source diff --git a/src/tally.f90 b/src/tally.f90 index db1f381b40..fa8a2bbe3c 100644 --- a/src/tally.f90 +++ b/src/tally.f90 @@ -12,6 +12,7 @@ module tally #ifdef MPI use mpi + use mpi_routines, only: reduce_tallies #endif implicit none @@ -511,6 +512,11 @@ contains real(8) :: val ! value of accumulated tally type(TallyObject), pointer :: t +#ifdef MPI + call reduce_tallies() + if (.not. master) return +#endif + do i = 1, n_tallies t => tallies(i)