From 5e18b4968c835a549948f3b1cefae8f8764f093a Mon Sep 17 00:00:00 2001
From: Paul Romano <romano7@mit.edu>
Date: Wed, 13 Apr 2011 20:06:05 +0000
Subject: [PATCH] Simple timing of fission bank synchronization.

---
 ChangeLog            |  7 +++++++
 src/Makefile         | 14 +++++++++-----
 src/main.f90         | 15 ++++++++++++++-
 src/mpi_routines.f90 | 10 ++++++++++
 4 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 426914065f..11501105a1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2011-04-13 Paul Romano <romano7@mit.edu>
+
+	* main.f90: Simple timing for total time elapsed and bank
+	synchronization.
+	* Makefile: Added USE_MPI flag and ifeq block.
+	* mpi_routines.f90: Timing of fission bank synchronization.
+
 2011-04-12 Paul Romano <romano7@mit.edu>
 
 	* global.f90: Added ifdef MPI sections
diff --git a/src/Makefile b/src/Makefile
index 2782560cd7..c6d7c828ab 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -28,11 +28,15 @@ test_objects = $(modules:.f90=.o) $(test:.f90=.o)
 F90 = ifort
 F90FLAGS = -g -fpp -traceback
 
-MPI = /opt/mpich2/1.3.3-gcc
-MPI_COMPILE_FLAGS = $(shell $(MPI)/bin/mpif90 -f90='' -compile_info)
-MPI_LINK_FLAGS = $(shell $(MPI)/bin/mpif90 -f90='' -link_info)
-F90 = $(MPI)/bin/mpif90
-F90FLAGS += -DMPI -f90=ifort
+USE_MPI = yes
+
+ifeq ($(USE_MPI),yes)
+  MPI = /opt/mpich2/1.3.3-gcc
+  MPI_COMPILE_FLAGS = $(shell $(MPI)/bin/mpif90 -f90='' -compile_info)
+  MPI_LINK_FLAGS = $(shell $(MPI)/bin/mpif90 -f90='' -link_info)
+  F90 = $(MPI)/bin/mpif90
+  F90FLAGS += -DMPI -f90=ifort
+endif
 
 #--------------------------------------------------------------------
 # Targets
diff --git a/src/main.f90 b/src/main.f90
index c2b82dbede..025e50ee0c 100644
--- a/src/main.f90
+++ b/src/main.f90
@@ -12,7 +12,7 @@ program main
   use cross_section, only: read_xsdata, material_total_xs
   use ace,           only: read_xs
   use energy_grid,   only: unionized_grid, original_indices
-  use mpi_routines,  only: setup_mpi, synchronize_bank
+  use mpi_routines,  only: setup_mpi, synchronize_bank, t_assemble
   use tallies,       only: calculate_keff
 
 #ifdef MPI
@@ -103,12 +103,18 @@ contains
     integer :: i_cycle    ! cycle index
     integer :: i_particle ! history index
     integer :: total_bank ! total number of particles banked
+    real(8) :: t0
+    real(8) :: t1
     type(Particle), pointer :: p => null()
     character(250) :: msg
 
     msg = "Running problem..."
     call message(msg, 6)
 
+#ifdef MPI
+    t0 = MPI_WTIME()
+#endif
+
     CYCLE_LOOP: do i_cycle = 1, n_cycles
        
        ! Set all tallies to zero
@@ -144,7 +150,14 @@ contains
 
     ! Collect all tallies and print
 
+#ifdef MPI
     ! print run time
+    t1 = MPI_WTIME()
+    if (master) then
+       write(6,"(A,F8.2,A)") "Time elapsed = ", t1 - t0, " s"
+       write(6,"(A,G10.4)") "Assemble time = ", t_assemble
+    end if
+#endif
 
   end subroutine run_problem
 
diff --git a/src/mpi_routines.f90 b/src/mpi_routines.f90
index b241c21a71..caaa2dc67f 100644
--- a/src/mpi_routines.f90
+++ b/src/mpi_routines.f90
@@ -14,6 +14,8 @@ module mpi_routines
   integer    :: MPI_BANK   ! MPI datatype for fission bank
   integer(8) :: bank_index ! Fission bank site unique identifier
 
+  real(8) :: t_assemble
+
 contains
 
 !=====================================================================
@@ -115,6 +117,8 @@ contains
     integer :: send_to_right     ! # of bank sites to send/recv to or from right
     integer :: sites_needed      ! # of sites to be sampled
     integer :: sites_remaining   ! # of sites left in fission bank
+    real(8) :: t0
+    real(8) :: t1
     type(Bank), allocatable :: &
          & temp_sites(:),      & ! local array of extra sites on each node
          & left_bank(:),       & ! bank sites to send/recv to or from left node
@@ -125,6 +129,9 @@ contains
     call message(msg, 8)
 
 #ifdef MPI
+    call MPI_BARRIER(MPI_COMM_WORLD, ierr)
+    t0 = MPI_WTIME()
+
     ! Determine starting index for fission bank and total sites in
     ! fission bank
     start = 0
@@ -247,6 +254,9 @@ contains
     source_index = 0
     
 #ifdef MPI
+    t1 = MPI_WTIME()
+    t_assemble = t_assemble + (t1 - t0)
+
     deallocate(left_bank )
     deallocate(right_bank)
 #endif