From 34a84688ed5492651efdddfc6e0a2d087ecc137e Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 8 Jul 2013 08:24:15 -0400
Subject: [PATCH 01/69] added matrix header file

---
 src/OBJECTS           |   1 +
 src/matrix_header.F90 | 121 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 122 insertions(+)
 create mode 100644 src/matrix_header.F90

diff --git a/src/OBJECTS b/src/OBJECTS
index b079906751..4fbaf80a76 100644
--- a/src/OBJECTS
+++ b/src/OBJECTS
@@ -37,6 +37,7 @@ list_header.o \
 main.o \
 material_header.o \
 math.o \
+matrix_header.o \
 mesh_header.o \
 mesh.o \
 mpiio_interface.o \
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
new file mode 100644
index 0000000000..a2ca46f7e4
--- /dev/null
+++ b/src/matrix_header.F90
@@ -0,0 +1,121 @@
+module matrix_header
+
+  implicit none
+  private
+
+# ifdef PETSC
+#  include <finclude/petsc.h90>
+# endif
+
+  type, public :: matrix
+    integer :: n        ! number of rows/cols in matrix
+    integer :: nz       ! number of nonzeros in matrix
+    integer :: n_kount  ! counter for length of matrix
+    integer :: nz_kount ! counter for number of non zeros
+    integer, allocatable :: row(:) ! csr row vector
+    integer, allocatable :: col(:) ! column vector
+    real(8), allocatable :: val(:) ! matrix value vector
+#  ifdef PETSC
+    Mat :: petsc_mat
+#  endif
+   contains
+     procedure :: create       => matrix_create
+     procedure :: destroy      => matrix_destroy
+     procedure :: add_value    => matrix_add_value
+     procedure :: new_row      => matrix_new_row
+#   ifdef PETSC
+     procedure :: setup_petsc  => matrix_setup_petsc
+#   endif
+  end type matrix
+
+contains
+
+!===============================================================================
+! MATRIX_CREATE allocates CSR vectors
+!===============================================================================
+
+  subroutine matrix_create(self, n, nz)
+
+    integer       :: n
+    integer       :: nz
+    class(matrix) :: self
+
+    ! preallocate vectors
+    if (.not.allocated(self % row)) allocate(self % row(n+1))
+    if (.not.allocated(self % col)) allocate(self % col(nz))
+    if (.not.allocated(self % val)) allocate(self % val(nz))
+
+    ! set counters to 1
+    self % n_kount  = 1
+    self % nz_kount = 1
+
+  end subroutine matrix_create
+
+!===============================================================================
+! MATRIX_DESTROY deallocates all space associated with a matrix
+!===============================================================================
+
+  subroutine matrix_destroy(self)
+
+    class(matrix) :: self
+
+    if (allocated(self % row)) deallocate(self % row)
+    if (allocated(self % col)) deallocate(self % col)
+    if (allocated(self % val)) deallocate(self % val)
+
+  end subroutine matrix_destroy
+
+!===============================================================================
+! MATRIX_ADD_VALUE adds a value to the matrix
+!===============================================================================
+
+  subroutine matrix_add_value(self, col, val)
+
+    integer       :: col
+    real(8)       :: val
+    class(matrix) :: self
+
+    self % col(self % nz_kount) = col
+    self % val(self % nz_kount) = val
+    self % nz_kount = self % nz_kount + 1
+
+  end subroutine matrix_add_value
+
+!===============================================================================
+! MATRIX_NEW_ROW adds a new row by saving the column position of the new row
+!===============================================================================
+
+  subroutine matrix_new_row(self)
+
+    class(matrix) :: self
+
+    self % row(self % n_kount) = self % nz_kount
+    self % n_kount = self % n_kount + 1
+
+  end subroutine matrix_new_row
+
+# ifdef PETSC
+
+!===============================================================================
+! MATRIX_USE_PETSC
+!===============================================================================
+
+  subroutine matrix_use_petsc(self)
+
+    class(matrix) :: self
+
+    integer :: petsc_err
+
+    ! change indices to c notation
+    self % row = self % row - 1
+    self % col = self % col - 1
+
+    ! link to petsc
+    call MatCreateSeqAIJWithArrays(PETSC_COMM_WORLD, self % n, self % n, &
+            self % row, self % col, self % val, loss % petsc_mat, petsc_err)
+
+  end subroutine matrix_use_petsc
+
+# endif
+
+end module matrix_header

From 8d23b2355841dd26eb6f92cb261dbac2cf0a39ec Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 8 Jul 2013 11:04:28 -0400
Subject: [PATCH 02/69] fixed incorrect naming in matrix header and added petsc
 solver module

---
 src/OBJECTS           |  1 +
 src/matrix_header.F90 |  8 ++--
 src/petsc_solver.F90  | 97 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 102 insertions(+), 4 deletions(-)
 create mode 100644 src/petsc_solver.F90

diff --git a/src/OBJECTS b/src/OBJECTS
index 4fbaf80a76..030679cca6 100644
--- a/src/OBJECTS
+++ b/src/OBJECTS
@@ -46,6 +46,7 @@ output_interface.o \
 particle_header.o \
 particle_restart.o \
 particle_restart_write.o \
+petsc_solver.o \
 physics.o \
 plot.o \
 plot_header.o \
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index a2ca46f7e4..2b53facd96 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -97,10 +97,10 @@ contains
 # ifdef PETSC
 
 !===============================================================================
-! MATRIX_USE_PETSC
+! MATRIX_SETUP_PETSC
 !===============================================================================
 
-  subroutine matrix_use_petsc(self)
+  subroutine matrix_setup_petsc(self)
 
     class(matrix) :: self
 
@@ -112,9 +112,9 @@ contains
 
     ! link to petsc
     call MatCreateSeqAIJWithArrays(PETSC_COMM_WORLD, self % n, self % n, &
-            self % row, self % col, self % val, loss % petsc_mat, petsc_err)
+            self % row, self % col, self % val, self % petsc_mat, petsc_err)
 
-  end subroutine matrix_use_petsc
+  end subroutine matrix_setup_petsc
 
 # endif
 
diff --git a/src/petsc_solver.F90 b/src/petsc_solver.F90
new file mode 100644
index 0000000000..c9136e4c70
--- /dev/null
+++ b/src/petsc_solver.F90
@@ -0,0 +1,97 @@
+module petsc_solver
+
+  implicit none
+  private
+
+# ifdef PETSC
+#  include <finclude/petsc.h90>
+# endif
+
+  type, public :: gmres
+#  ifdef PETSC
+    KSP :: ksp
+    PC  :: pc
+#  endif
+   contains
+     procedure :: create       => gmres_create
+     procedure :: precondition => gmres_precondition
+     procedure :: set_oper     => gmres_set_oper
+     procedure :: destroy      => gmres_destroy
+  end type gmres
+
+  integer :: petsc_err
+
+contains
+
+!===============================================================================
+! GMRES_CREATE
+!===============================================================================
+
+  subroutine gmres_create(self)
+
+    class(gmres) :: self
+
+    integer :: ilu_levels = 5
+    real(8) :: rtol = 1.0e-10_8
+    real(8) :: atol = 1.0e-10_8
+
+#  ifdef PETSC
+    call KSPCreate(PETSC_COMM_WORLD, self % ksp, petsc_err)
+    call KSPSetTolerances(self % ksp, rtol, atol, &
+         PETSC_DEFAULT_DOUBLE_PRECISION, PETSC_DEFAULT_INTEGER, petsc_err)
+    call KSPSetType(self % ksp, KSPGMRES, petsc_err)
+    call KSPSetInitialGuessNonzero(self % ksp, PETSC_TRUE, petsc_err)
+    call KSPGetPC(self % ksp, self % pc, petsc_err)
+    call PCFactorSetLevels(self % pc, ilu_levels, petsc_err)
+    call KSPSetUp(self % ksp, petsc_err) 
+#  endif
+
+  end subroutine gmres_create
+
+!===============================================================================
+! GMRES_PRECONDITION
+!===============================================================================
+
+  subroutine gmres_precondition(self, matrix)
+
+    class(gmres) :: self
+    Mat          :: matrix
+
+#  ifdef PETSC
+    call PCFactorGetMatrix(self % pc, matrix, petsc_err) 
+#  endif
+
+  end subroutine gmres_precondition
+
+!===============================================================================
+! GMRES_SET_OPER
+!===============================================================================
+
+  subroutine gmres_set_oper(self, prec_matrix, matrix)
+
+    class(gmres) :: self
+    Mat          :: prec_matrix
+    Mat          :: matrix
+
+#  ifdef PETSC
+    call KSPSetOperators(self % ksp, matrix, prec_matrix, &
+         SAME_NONZERO_PATTERN, petsc_err)
+#  endif
+
+  end subroutine gmres_set_oper
+
+!===============================================================================
+! GMRES_DESTROY
+!===============================================================================
+
+  subroutine gmres_destroy(self)
+
+   class(gmres) :: self
+
+#  ifdef PETSC
+    call KSPDestroy(self % ksp, petsc_err)
+#  endif 
+
+  end subroutine gmres_destroy
+
+end module petsc_solver 

From 5710cf6fa9e9279ecf3fd9d5b6a110b05245cbf5 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 8 Jul 2013 11:42:04 -0400
Subject: [PATCH 03/69] changed over loss operator to new notation and fixed up
 matrix header

---
 src/DEPENDENCIES           | 556 +++++++++++++++++++------------------
 src/cmfd_loss_operator.F90 | 320 ++++++++-------------
 src/matrix_header.F90      |  26 +-
 3 files changed, 406 insertions(+), 496 deletions(-)

diff --git a/src/DEPENDENCIES b/src/DEPENDENCIES
index e5d20644b0..973386c406 100644
--- a/src/DEPENDENCIES
+++ b/src/DEPENDENCIES
@@ -1,17 +1,49 @@
-ace.o: ace_header.o
-ace.o: constants.o
-ace.o: endf.o
-ace.o: error.o
-ace.o: fission.o
-ace.o: global.o
-ace.o: material_header.o
-ace.o: output.o
-ace.o: set_header.o
-ace.o: string.o
+set_header.o: constants.o
+set_header.o: list_header.o
+
+energy_grid.o: constants.o
+energy_grid.o: global.o
+energy_grid.o: list_header.o
+energy_grid.o: output.o
+
+cmfd_output.o: cmfd_data.o
+cmfd_output.o: cmfd_header.o
+cmfd_output.o: constants.o
+cmfd_output.o: global.o
+
+list_header.o: constants.o
 
 ace_header.o: constants.o
 ace_header.o: endf_header.o
 
+cmfd_loss_operator.o: constants.o
+cmfd_loss_operator.o: global.o
+cmfd_loss_operator.o: matrix_header.o
+
+particle_restart.o: bank_header.o
+particle_restart.o: constants.o
+particle_restart.o: geometry_header.o
+particle_restart.o: global.o
+particle_restart.o: hdf5_interface.o
+particle_restart.o: output.o
+particle_restart.o: particle_header.o
+particle_restart.o: physics.o
+particle_restart.o: random_lcg.o
+particle_restart.o: source.o
+
+doppler.o: constants.o
+
+tally_header.o: constants.o
+
+cmfd_power_solver.o: cmfd_loss_operator.o
+cmfd_power_solver.o: cmfd_prod_operator.o
+cmfd_power_solver.o: constants.o
+cmfd_power_solver.o: global.o
+cmfd_power_solver.o: string.o
+
+cmfd_message_passing.o: cmfd_header.o
+cmfd_message_passing.o: global.o
+
 cmfd_data.o: cmfd_header.o
 cmfd_data.o: constants.o
 cmfd_data.o: error.o
@@ -20,119 +52,17 @@ cmfd_data.o: mesh.o
 cmfd_data.o: mesh_header.o
 cmfd_data.o: tally_header.o
 
-cmfd_execute.o: cmfd_data.o
-cmfd_execute.o: cmfd_message_passing.o
-cmfd_execute.o: cmfd_power_solver.o
-cmfd_execute.o: cmfd_snes_solver.o
-cmfd_execute.o: constants.o
-cmfd_execute.o: error.o
-cmfd_execute.o: global.o
-cmfd_execute.o: mesh.o
-cmfd_execute.o: mesh_header.o
-cmfd_execute.o: output.o
-cmfd_execute.o: search.o
-cmfd_execute.o: tally.o
-
-cmfd_header.o: constants.o
-
-cmfd_input.o: cmfd_message_passing.o
-cmfd_input.o: error.o
-cmfd_input.o: global.o
-cmfd_input.o: mesh_header.o
-cmfd_input.o: output.o
-cmfd_input.o: string.o
-cmfd_input.o: tally.o
-cmfd_input.o: tally_header.o
-cmfd_input.o: tally_initialize.o
-cmfd_input.o: templates/cmfd_t.o
-
-cmfd_jacobian_operator.o: cmfd_loss_operator.o
-cmfd_jacobian_operator.o: cmfd_prod_operator.o
-cmfd_jacobian_operator.o: constants.o
-cmfd_jacobian_operator.o: global.o
-
-cmfd_loss_operator.o: constants.o
-cmfd_loss_operator.o: global.o
-
-cmfd_message_passing.o: cmfd_header.o
-cmfd_message_passing.o: global.o
-
-cmfd_output.o: cmfd_data.o
-cmfd_output.o: cmfd_header.o
-cmfd_output.o: constants.o
-cmfd_output.o: global.o
-
-cmfd_power_solver.o: cmfd_loss_operator.o
-cmfd_power_solver.o: cmfd_prod_operator.o
-cmfd_power_solver.o: constants.o
-cmfd_power_solver.o: global.o
-cmfd_power_solver.o: string.o
-
-cmfd_prod_operator.o: constants.o
-cmfd_prod_operator.o: global.o
-
-cmfd_slepc_solver.o: cmfd_loss_operator.o
-cmfd_slepc_solver.o: cmfd_prod_operator.o
-cmfd_slepc_solver.o: constants.o
-cmfd_slepc_solver.o: global.o
-
-cmfd_snes_solver.o: cmfd_jacobian_operator.o
-cmfd_snes_solver.o: cmfd_loss_operator.o
-cmfd_snes_solver.o: cmfd_power_solver.o
-cmfd_snes_solver.o: cmfd_prod_operator.o
-cmfd_snes_solver.o: constants.o
-cmfd_snes_solver.o: global.o
-cmfd_snes_solver.o: string.o
-
-cross_section.o: ace_header.o
-cross_section.o: constants.o
-cross_section.o: error.o
-cross_section.o: fission.o
-cross_section.o: global.o
-cross_section.o: material_header.o
-cross_section.o: random_lcg.o
-cross_section.o: search.o
-
-doppler.o: constants.o
-
-eigenvalue.o: cmfd_execute.o
-eigenvalue.o: constants.o
-eigenvalue.o: error.o
-eigenvalue.o: global.o
-eigenvalue.o: math.o
-eigenvalue.o: mesh.o
-eigenvalue.o: mesh_header.o
-eigenvalue.o: output.o
-eigenvalue.o: physics.o
-eigenvalue.o: random_lcg.o
-eigenvalue.o: search.o
-eigenvalue.o: source.o
-eigenvalue.o: state_point.o
-eigenvalue.o: string.o
-eigenvalue.o: tally.o
-
-endf.o: constants.o
-endf.o: string.o
-
-energy_grid.o: constants.o
-energy_grid.o: global.o
-energy_grid.o: list_header.o
-energy_grid.o: output.o
-
-error.o: global.o
-
-finalize.o: cmfd_output.o
-finalize.o: global.o
-finalize.o: hdf5_interface.o
-finalize.o: output.o
-finalize.o: tally.o
-
-fission.o: ace_header.o
-fission.o: constants.o
-fission.o: error.o
-fission.o: global.o
-fission.o: interpolation.o
-fission.o: search.o
+plot.o: constants.o
+plot.o: error.o
+plot.o: geometry.o
+plot.o: geometry_header.o
+plot.o: global.o
+plot.o: output.o
+plot.o: particle_header.o
+plot.o: plot_header.o
+plot.o: ppmlib.o
+plot.o: source.o
+plot.o: string.o
 
 fixed_source.o: constants.o
 fixed_source.o: global.o
@@ -144,64 +74,55 @@ fixed_source.o: state_point.o
 fixed_source.o: string.o
 fixed_source.o: tally.o
 
-geometry.o: constants.o
-geometry.o: error.o
-geometry.o: geometry_header.o
-geometry.o: global.o
-geometry.o: output.o
-geometry.o: particle_header.o
-geometry.o: particle_restart_write.o
-geometry.o: string.o
-geometry.o: tally.o
+source.o: bank_header.o
+source.o: constants.o
+source.o: error.o
+source.o: geometry_header.o
+source.o: global.o
+source.o: output.o
+source.o: particle_header.o
+source.o: physics.o
+source.o: random_lcg.o
+source.o: string.o
 
-global.o: ace_header.o
-global.o: bank_header.o
-global.o: cmfd_header.o
-global.o: constants.o
-global.o: dict_header.o
-global.o: geometry_header.o
-global.o: hdf5_interface.o
-global.o: material_header.o
-global.o: mesh_header.o
-global.o: particle_header.o
-global.o: plot_header.o
-global.o: set_header.o
-global.o: source_header.o
-global.o: tally_header.o
-global.o: timer_header.o
+cmfd_prod_operator.o: constants.o
+cmfd_prod_operator.o: global.o
 
-hdf5_summary.o: ace_header.o
-hdf5_summary.o: constants.o
-hdf5_summary.o: endf.o
-hdf5_summary.o: geometry_header.o
-hdf5_summary.o: global.o
-hdf5_summary.o: material_header.o
-hdf5_summary.o: mesh_header.o
-hdf5_summary.o: output_interface.o
-hdf5_summary.o: output.o
-hdf5_summary.o: string.o
-hdf5_summary.o: tally_header.o
+cmfd_slepc_solver.o: cmfd_loss_operator.o
+cmfd_slepc_solver.o: cmfd_prod_operator.o
+cmfd_slepc_solver.o: constants.o
+cmfd_slepc_solver.o: global.o
 
-initialize.o: ace.o
-initialize.o: bank_header.o
-initialize.o: constants.o
-initialize.o: dict_header.o
-initialize.o: energy_grid.o
-initialize.o: error.o
-initialize.o: geometry.o
-initialize.o: geometry_header.o
-initialize.o: global.o
-initialize.o: hdf5_interface.o
-initialize.o: hdf5_summary.o
-initialize.o: input_xml.o
-initialize.o: output_interface.o
-initialize.o: output.o
-initialize.o: random_lcg.o
-initialize.o: source.o
-initialize.o: state_point.o
-initialize.o: string.o
-initialize.o: tally_header.o
-initialize.o: tally_initialize.o
+fission.o: ace_header.o
+fission.o: constants.o
+fission.o: error.o
+fission.o: global.o
+fission.o: interpolation.o
+fission.o: search.o
+
+cmfd_jacobian_operator.o: cmfd_loss_operator.o
+cmfd_jacobian_operator.o: cmfd_prod_operator.o
+cmfd_jacobian_operator.o: constants.o
+cmfd_jacobian_operator.o: global.o
+
+cmfd_snes_solver.o: cmfd_jacobian_operator.o
+cmfd_snes_solver.o: cmfd_loss_operator.o
+cmfd_snes_solver.o: cmfd_power_solver.o
+cmfd_snes_solver.o: cmfd_prod_operator.o
+cmfd_snes_solver.o: constants.o
+cmfd_snes_solver.o: global.o
+cmfd_snes_solver.o: string.o
+
+cmfd_input.o: cmfd_message_passing.o
+cmfd_input.o: error.o
+cmfd_input.o: global.o
+cmfd_input.o: mesh_header.o
+cmfd_input.o: output.o
+cmfd_input.o: string.o
+cmfd_input.o: tally.o
+cmfd_input.o: tally_header.o
+cmfd_input.o: tally_initialize.o
+cmfd_input.o: templates/cmfd_t.o
 
 input_xml.o: cmfd_input.o
 input_xml.o: constants.o
@@ -224,15 +145,6 @@ input_xml.o: templates/plots_t.o
 input_xml.o: templates/settings_t.o
 input_xml.o: templates/tallies_t.o
 
-interpolation.o: constants.o
-interpolation.o: endf_header.o
-interpolation.o: error.o
-interpolation.o: global.o
-interpolation.o: search.o
-interpolation.o: string.o
-
-list_header.o: constants.o
-
 main.o: constants.o
 main.o: eigenvalue.o
 main.o: finalize.o
@@ -242,14 +154,194 @@ main.o: initialize.o
 main.o: particle_restart.o
 main.o: plot.o
 
+particle_restart_write.o: bank_header.o
+particle_restart_write.o: global.o
+particle_restart_write.o: hdf5_interface.o
+particle_restart_write.o: string.o
+
+timer_header.o: constants.o
+
 math.o: constants.o
 
+interpolation.o: constants.o
+interpolation.o: endf_header.o
+interpolation.o: error.o
+interpolation.o: global.o
+interpolation.o: search.o
+interpolation.o: string.o
+
+random_lcg.o: global.o
+
+global.o: ace_header.o
+global.o: bank_header.o
+global.o: cmfd_header.o
+global.o: constants.o
+global.o: dict_header.o
+global.o: geometry_header.o
+global.o: hdf5_interface.o
+global.o: material_header.o
+global.o: mesh_header.o
+global.o: particle_header.o
+global.o: plot_header.o
+global.o: set_header.o
+global.o: source_header.o
+global.o: tally_header.o
+global.o: timer_header.o
+
+string.o: constants.o
+string.o: error.o
+string.o: global.o
+
+finalize.o: cmfd_output.o
+finalize.o: global.o
+finalize.o: hdf5_interface.o
+finalize.o: output.o
+finalize.o: tally.o
+
+tally_initialize.o: constants.o
+tally_initialize.o: global.o
+tally_initialize.o: tally_header.o
+
+tally.o: ace_header.o
+tally.o: constants.o
+tally.o: error.o
+tally.o: global.o
+tally.o: math.o
+tally.o: mesh.o
+tally.o: mesh_header.o
+tally.o: output.o
+tally.o: particle_header.o
+tally.o: search.o
+tally.o: string.o
+tally.o: tally_header.o
+
+particle_header.o: constants.o
+
+output_interface.o: constants.o
+output_interface.o: error.o
+output_interface.o: global.o
+output_interface.o: hdf5_interface.o
+output_interface.o: mpiio_interface.o
+output_interface.o: tally_header.o
+
 mesh.o: constants.o
 mesh.o: global.o
 mesh.o: mesh_header.o
 mesh.o: particle_header.o
 mesh.o: search.o
 
+endf.o: constants.o
+endf.o: string.o
+
+initialize.o: ace.o
+initialize.o: bank_header.o
+initialize.o: constants.o
+initialize.o: dict_header.o
+initialize.o: energy_grid.o
+initialize.o: error.o
+initialize.o: geometry.o
+initialize.o: geometry_header.o
+initialize.o: global.o
+initialize.o: hdf5_interface.o
+initialize.o: hdf5_summary.o
+initialize.o: input_xml.o
+initialize.o: output.o
+initialize.o: output_interface.o
+initialize.o: random_lcg.o
+initialize.o: source.o
+initialize.o: state_point.o
+initialize.o: string.o
+initialize.o: tally_header.o
+initialize.o: tally_initialize.o
+
+cross_section.o: ace_header.o
+cross_section.o: constants.o
+cross_section.o: error.o
+cross_section.o: fission.o
+cross_section.o: global.o
+cross_section.o: material_header.o
+cross_section.o: random_lcg.o
+cross_section.o: search.o
+
+state_point.o: constants.o
+state_point.o: error.o
+state_point.o: global.o
+state_point.o: output.o
+state_point.o: output_interface.o
+state_point.o: string.o
+state_point.o: tally_header.o
+
+eigenvalue.o: cmfd_execute.o
+eigenvalue.o: constants.o
+eigenvalue.o: error.o
+eigenvalue.o: global.o
+eigenvalue.o: math.o
+eigenvalue.o: mesh.o
+eigenvalue.o: mesh_header.o
+eigenvalue.o: output.o
+eigenvalue.o: physics.o
+eigenvalue.o: random_lcg.o
+eigenvalue.o: search.o
+eigenvalue.o: source.o
+eigenvalue.o: state_point.o
+eigenvalue.o: string.o
+eigenvalue.o: tally.o
+
+search.o: error.o
+search.o: global.o
+
+ace.o: ace_header.o
+ace.o: constants.o
+ace.o: endf.o
+ace.o: error.o
+ace.o: fission.o
+ace.o: global.o
+ace.o: material_header.o
+ace.o: output.o
+ace.o: set_header.o
+ace.o: string.o
+
+geometry.o: constants.o
+geometry.o: error.o
+geometry.o: geometry_header.o
+geometry.o: global.o
+geometry.o: output.o
+geometry.o: particle_header.o
+geometry.o: particle_restart_write.o
+geometry.o: string.o
+geometry.o: tally.o
+
+plot_header.o: constants.o
+
+cmfd_header.o: constants.o
+
+cmfd_execute.o: cmfd_data.o
+cmfd_execute.o: cmfd_message_passing.o
+cmfd_execute.o: cmfd_power_solver.o
+cmfd_execute.o: cmfd_snes_solver.o
+cmfd_execute.o: constants.o
+cmfd_execute.o: error.o
+cmfd_execute.o: global.o
+cmfd_execute.o: mesh.o
+cmfd_execute.o: mesh_header.o
+cmfd_execute.o: output.o
+cmfd_execute.o: search.o
+cmfd_execute.o: tally.o
+
+hdf5_summary.o: ace_header.o
+hdf5_summary.o: constants.o
+hdf5_summary.o: endf.o
+hdf5_summary.o: geometry_header.o
+hdf5_summary.o: global.o
+hdf5_summary.o: material_header.o
+hdf5_summary.o: mesh_header.o
+hdf5_summary.o: output.o
+hdf5_summary.o: output_interface.o
+hdf5_summary.o: string.o
+hdf5_summary.o: tally_header.o
+
+error.o: global.o
+
 output.o: ace_header.o
 output.o: constants.o
 output.o: endf.o
@@ -264,31 +356,6 @@ output.o: plot_header.o
 output.o: string.o
 output.o: tally_header.o
 
-output_interface.o: constants.o
-output_interface.o: error.o
-output_interface.o: global.o
-output_interface.o: hdf5_interface.o
-output_interface.o: mpiio_interface.o
-output_interface.o: tally_header.o
-
-particle_header.o: constants.o
-
-particle_restart.o: bank_header.o
-particle_restart.o: constants.o
-particle_restart.o: geometry_header.o
-particle_restart.o: global.o
-particle_restart.o: hdf5_interface.o
-particle_restart.o: output.o
-particle_restart.o: particle_header.o
-particle_restart.o: physics.o
-particle_restart.o: random_lcg.o
-particle_restart.o: source.o
-
-particle_restart_write.o: bank_header.o
-particle_restart_write.o: global.o
-particle_restart_write.o: hdf5_interface.o
-particle_restart_write.o: string.o
-
 physics.o: ace_header.o
 physics.o: constants.o
 physics.o: cross_section.o
@@ -309,68 +376,3 @@ physics.o: search.o
 physics.o: string.o
 physics.o: tally.o
 
-plot.o: constants.o
-plot.o: error.o
-plot.o: geometry.o
-plot.o: geometry_header.o
-plot.o: global.o
-plot.o: output.o
-plot.o: particle_header.o
-plot.o: plot_header.o
-plot.o: ppmlib.o
-plot.o: source.o
-plot.o: string.o
-
-plot_header.o: constants.o
-
-random_lcg.o: global.o
-
-search.o: error.o
-search.o: global.o
-
-set_header.o: constants.o
-set_header.o: list_header.o
-
-source.o: bank_header.o
-source.o: constants.o
-source.o: error.o
-source.o: geometry_header.o
-source.o: global.o
-source.o: output.o
-source.o: particle_header.o
-source.o: physics.o
-source.o: random_lcg.o
-source.o: string.o
-
-state_point.o: constants.o
-state_point.o: error.o
-state_point.o: global.o
-state_point.o: output.o
-state_point.o: output_interface.o
-state_point.o: string.o
-state_point.o: tally_header.o
-
-string.o: constants.o
-string.o: error.o
-string.o: global.o
-
-tally.o: ace_header.o
-tally.o: constants.o
-tally.o: error.o
-tally.o: global.o
-tally.o: math.o
-tally.o: mesh.o
-tally.o: mesh_header.o
-tally.o: output.o
-tally.o: particle_header.o
-tally.o: search.o
-tally.o: string.o
-tally.o: tally_header.o
-
-tally_header.o: constants.o
-
-tally_initialize.o: constants.o
-tally_initialize.o: global.o
-tally_initialize.o: tally_header.o
-
-timer_header.o: constants.o
diff --git a/src/cmfd_loss_operator.F90 b/src/cmfd_loss_operator.F90
index 4b5fb1196d..258fa650c4 100644
--- a/src/cmfd_loss_operator.F90
+++ b/src/cmfd_loss_operator.F90
@@ -1,64 +1,35 @@
 module cmfd_loss_operator
 
-# ifdef PETSC
+  use constants,      only: CMFD_NOACCEL, ZERO
+  use global,         only: cmfd, cmfd_coremap
+  use matrix_header,  only: Matrix
 
   implicit none
   private
-  public :: init_M_operator, build_loss_matrix, destroy_M_operator
-
-# include <finclude/petsc.h90>
-
-  integer  :: nx   ! maximum number of x cells
-  integer  :: ny   ! maximum number of y cells
-  integer  :: nz   ! maximum number of z cells
-  integer  :: ng   ! maximum number of groups
-  integer  :: ierr ! petsc error code
-
-  type, public :: loss_operator
-    Mat      :: M      ! petsc matrix for neutronic loss operator
-    integer  :: n      ! dimensions of matrix
-    integer  :: nnz    ! max number of nonzeros
-    integer  :: localn ! local size on proc
-    integer, allocatable :: d_nnz(:) ! vector of diagonal preallocation
-    integer, allocatable :: o_nnz(:) ! vector of off-diagonal preallocation
-  end type loss_operator
-
-  logical :: adjoint_calc = .false. ! adjoint calculation
+  public :: init_loss_matrix, build_loss_matrix
 
 contains
 
 !===============================================================================
-! INIT_M_OPERATOR
+! INIT_LOSS_MATRIX
 !===============================================================================
 
-  subroutine init_M_operator(this)
+  subroutine init_loss_matrix(loss_matrix)
 
-    type(loss_operator) :: this
+    type(Matrix) :: loss_matrix ! cmfd loss matrix
 
-    ! get indices
-    call get_M_indices(this)
-
-    ! get preallocation
-    call preallocate_loss_matrix(this)
-
-    ! set up M operator
-    call MatCreateAIJ(PETSC_COMM_WORLD, this%localn, this%localn, PETSC_DECIDE,&
-         PETSC_DECIDE, PETSC_NULL_INTEGER, this%d_nnz, PETSC_NULL_INTEGER, &
-         this%o_nnz, this%M,ierr)
-    call MatSetOption(this%M, MAT_NEW_NONZERO_LOCATIONS, PETSC_TRUE, ierr)
-    call MatSetOption(this%M, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE, ierr)
-
-  end subroutine init_M_operator
-
-!===============================================================================
-! GET_M_INDICES
-!===============================================================================
-
-  subroutine get_M_indices(this)
-
-    use global,  only: cmfd, cmfd_coremap
-
-    type(loss_operator) :: this
+    integer :: nx   ! maximum number of x cells
+    integer :: ny   ! maximum number of y cells
+    integer :: nz   ! maximum number of z cells
+    integer :: ng   ! maximum number of groups
+    integer :: n    ! total length of matrix
+    integer :: nnz  ! number of nonzeros in matrix
+    integer :: n_i  ! number of interior cells
+    integer :: n_c  ! number of corner cells
+    integer :: n_s  ! number side cells
+    integer :: nz_c ! number of non-zero corner cells
+    integer :: nz_s ! number of non-zero side cells
+    integer :: nz_i ! number of non-zero interior cells
 
     ! get maximum number of cells in each direction
     nx = cmfd%indices(1)
@@ -66,38 +37,50 @@ contains
     nz = cmfd%indices(3)
     ng = cmfd%indices(4)
 
-    ! get number of nonzeros
-    this%nnz = 7 + ng - 1
-
     ! calculate dimensions of matrix
     if (cmfd_coremap) then
-      this%n = cmfd % mat_dim * ng
+      n = cmfd % mat_dim * ng
     else
-      this%n = nx*ny*nz*ng
+      n = nx*ny*nz*ng
     end if
 
-  end subroutine get_M_indices
+    ! calculate number of nonzeros, if core map -> need to determine manually
+    if (cmfd_coremap) then
+      nnz = preallocate_loss_matrix(nx, ny, nz, ng, n)
+    else ! structured Cartesian grid
+      n_c  = 4                   ! define # of corners
+      n_s  = 2*(nx + ny) - 8     ! define # of sides
+      n_i  = nx*ny - (n_c + n_S) ! define # of interiors  
+      nz_c = ng*n_c*(3 + ng - 1) ! define # nonzero corners
+      nz_s = ng*n_s*(4 + ng - 1) ! define # nonzero sides
+      nz_i = ng*n_i*(5 + ng - 1) ! define # nonzero interiors
+      nnz  = nz_c + nz_s + nz_i
+    end if
+
+    ! configure loss matrix
+    call loss_matrix % create(n, nnz)
+
+  end subroutine init_loss_matrix
 
 !===============================================================================
 ! PREALLOCATE_LOSS_MATRIX
 !===============================================================================
 
-  subroutine preallocate_loss_matrix(this)
+  function preallocate_loss_matrix(nx, ny, nz, ng, n) result(nnz)
 
-    use constants,  only: CMFD_NOACCEL
-    use global,     only: cmfd, cmfd_coremap
+    integer, intent(in)    :: nx   ! maximum number of x cells
+    integer, intent(in)    :: ny   ! maximum number of y cells
+    integer, intent(in)    :: nz   ! maximum number of z cells
+    integer, intent(in)    :: ng   ! maximum number of groups
+    integer, intent(in)    :: n    ! total length of matrix
+    integer                :: nnz  ! number of nonzeros
 
-    type(loss_operator) :: this
-
-    integer :: rank          ! rank of processor
-    integer :: sizen         ! number of procs
     integer :: i             ! iteration counter for x
     integer :: j             ! iteration counter for y
     integer :: k             ! iteration counter for z
     integer :: g             ! iteration counter for groups
     integer :: l             ! iteration counter for leakages
     integer :: h             ! energy group when doing scattering
-    integer :: n             ! the extent of the matrix
     integer :: irow          ! row counter
     integer :: bound(6)      ! vector for comparing when looking for bound
     integer :: xyz_idx       ! index for determining if x,y or z leakage
@@ -105,45 +88,11 @@ contains
     integer :: neig_idx(3)   ! spatial indices of neighbour
     integer :: nxyz(3,2)     ! single vector containing bound. locations
     integer :: shift_idx     ! parameter to shift index by +1 or -1
-    integer :: row_start     ! index of local starting row
-    integer :: row_end       ! index of local final row
     integer :: neig_mat_idx  ! matrix index of neighbor cell
     integer :: scatt_mat_idx ! matrix index for h-->g scattering terms
 
-    ! initialize size and rank
-    sizen = 0
-    rank = 0
-
-    ! get rank and max rank of procs
-    call MPI_COMM_RANK(PETSC_COMM_WORLD, rank, ierr)
-    call MPI_COMM_SIZE(PETSC_COMM_WORLD, sizen, ierr)
-
-    ! get local problem size
-    n = this%n
-
-    ! determine local size, divide evenly between all other procs
-    this%localn = n/(sizen)
-        
-    ! add 1 more if less proc id is less than mod
-    if (rank < mod(n,sizen)) this%localn = this%localn + 1
-
-    ! determine local starting row
-    row_start = 0
-    if (rank < mod(n,sizen)) then
-      row_start = rank*(n/sizen+1)
-    else
-      row_start = min(mod(n,sizen)*(n/sizen+1) + &
-           (rank - mod(n,sizen))*(n/sizen),n) 
-    end if
-
-    ! determine local final row
-    row_end = row_start + this%localn - 1
-
-    ! allocate counters
-    if (.not. allocated(this%d_nnz)) allocate(this%d_nnz(row_start:row_end))
-    if (.not. allocated(this%o_nnz)) allocate(this%o_nnz(row_start:row_end))
-    this % d_nnz = 0
-    this % o_nnz = 0
+    ! reset to 0
+    nnz = 0
 
     ! create single vector of these indices for boundary calculation
     nxyz(1,:) = (/1,nx/)
@@ -151,14 +100,13 @@ contains
     nxyz(3,:) = (/1,nz/)
 
     ! begin loop around local rows
-    ROWS: do irow = row_start,row_end
+    ROWS: do irow = 1, n
 
-      ! initialize counters 
-      this%d_nnz(irow) = 1 ! already add in matrix diagonal
-      this%o_nnz(irow) = 0
+      ! set a nonzero for diagonal
+      nnz = nnz + 1
 
       ! get location indices
-      call matrix_to_indices(irow, g, i, j, k)
+      call matrix_to_indices(irow, g, i, j, k, ng, nx, ny, nz)
 
       ! create boundary vector
       bound = (/i,i,j,j,k,k/)
@@ -187,15 +135,10 @@ contains
 
               ! get neighbor matrix index
               call indices_to_matrix(g,neig_idx(1), neig_idx(2), & 
-                   neig_idx(3), neig_mat_idx)
+                   neig_idx(3), neig_mat_idx, ng, nx, ny)
 
               ! record nonzero
-              if (((neig_mat_idx-1) >= row_start) .and. &
-                   ((neig_mat_idx-1) <= row_end)) then
-                this%d_nnz(irow) = this%d_nnz(irow) + 1
-              else
-                this%o_nnz(irow) = this%o_nnz(irow) + 1
-              end if
+              nnz = nnz + 1 
 
             end if
 
@@ -203,15 +146,10 @@ contains
 
             ! get neighbor matrix index
             call indices_to_matrix(g, neig_idx(1), neig_idx(2), neig_idx(3), &
-                 neig_mat_idx)
+                 neig_mat_idx, ng, nx, ny)
 
             ! record nonzero
-            if (((neig_mat_idx-1) >= row_start) .and. &
-                 ((neig_mat_idx-1) <= row_end)) then
-              this%d_nnz(irow) = this%d_nnz(irow) + 1
-            else
-              this%o_nnz(irow) = this%o_nnz(irow) + 1
-            end if
+            nnz = nnz + 1
 
           end if
 
@@ -226,32 +164,24 @@ contains
         if (h == g) cycle
 
         ! get neighbor matrix index
-        call indices_to_matrix(h, i, j, k, scatt_mat_idx)
+        call indices_to_matrix(h, i, j, k, scatt_mat_idx, ng, nx, ny)
 
         ! record nonzero
-        if (((scatt_mat_idx-1) >= row_start) .and. &
-             ((scatt_mat_idx-1) <= row_end)) then
-          this%d_nnz(irow) = this%d_nnz(irow) + 1
-        else
-          this%o_nnz(irow) = this%o_nnz(irow) + 1
-        end if
+        nnz = nnz + 1
 
       end do SCATTR
 
     end do ROWS
 
-  end subroutine preallocate_loss_matrix
+  end function preallocate_loss_matrix
 
 !===============================================================================
 ! BUILD_LOSS_MATRIX creates the matrix representing loss of neutrons
 !===============================================================================
 
-  subroutine build_loss_matrix(this, adjoint)
+  subroutine build_loss_matrix(loss_matrix, adjoint)
 
-    use constants,  only: CMFD_NOACCEL, ZERO
-    use global,     only: cmfd, cmfd_coremap, cmfd_write_matrices
-
-    type(loss_operator) :: this
+    type(Matrix) :: loss_matrix     ! cmfd loss matrix
     logical, optional :: adjoint    ! set up the adjoint
 
     integer :: nxyz(3,2)            ! single vector containing bound. locations
@@ -261,6 +191,10 @@ contains
     integer :: g                    ! iteration counter for groups
     integer :: l                    ! iteration counter for leakages
     integer :: h                    ! energy group when doing scattering
+    integer :: nx                   ! maximum number of x cells
+    integer :: ny                   ! maximum number of y cells
+    integer :: nz                   ! maximum number of z cells
+    integer :: ng                   ! maximum number of groups
     integer :: neig_mat_idx         ! matrix index of neighbor cell
     integer :: scatt_mat_idx        ! matrix index for h-->g scattering terms
     integer :: bound(6)             ! vector for comparing when looking for bound
@@ -268,9 +202,8 @@ contains
     integer :: dir_idx              ! index for determining - or + face of cell
     integer :: neig_idx(3)          ! spatial indices of neighbour
     integer :: shift_idx            ! parameter to shift index by +1 or -1
-    integer :: row_start            ! the first local row on the processor
-    integer :: row_finish           ! the last local row on the processor
     integer :: irow                 ! iteration counter over row
+    logical :: adjoint_calc         ! is this a physical adjoint calculation?
     real(8) :: totxs                ! total macro cross section
     real(8) :: scattxsgg            ! scattering macro cross section g-->g
     real(8) :: scattxshg            ! scattering macro cross section h-->g
@@ -283,25 +216,28 @@ contains
     real(8) :: val                  ! temporary variable before saving to 
 
     ! check for adjoint
+    adjoint_calc = .false.
     if (present(adjoint)) adjoint_calc = adjoint 
 
+    ! get maximum number of cells in each direction
+    nx = cmfd%indices(1)
+    ny = cmfd%indices(2)
+    nz = cmfd%indices(3)
+    ng = cmfd%indices(4)
+
     ! create single vector of these indices for boundary calculation
     nxyz(1,:) = (/1,nx/)
     nxyz(2,:) = (/1,ny/)
     nxyz(3,:) = (/1,nz/)
 
-    ! initialize row start and finish
-    row_start = 0
-    row_finish = 0
-
-    ! get row bounds for this processor
-    call MatGetOwnershipRange(this%M, row_start, row_finish, ierr)
-
     ! begin iteration loops
-    ROWS: do irow = row_start, row_finish-1
+    ROWS: do irow = 1, loss_matrix % n
+
+      ! set up a new row in matrix
+      call loss_matrix % new_row
 
       ! get indices for that row
-      call matrix_to_indices(irow, g, i, j, k)
+      call matrix_to_indices(irow, g, i, j, k, ng, nx, ny, nz)
 
       ! retrieve cell data
       totxs = cmfd%totalxs(g,i,j,k)
@@ -347,14 +283,13 @@ contains
 
               ! get neighbor matrix index
               call indices_to_matrix(g, neig_idx(1), neig_idx(2), neig_idx(3), &
-                   neig_mat_idx)
+                   neig_mat_idx, ng, nx, ny)
 
               ! compute value and record to bank
               val = jn/hxyz(xyz_idx)
 
               ! record value in matrix
-              call MatSetValue(this%M, irow, neig_mat_idx-1, val, &
-                   INSERT_VALUES, ierr)
+              call loss_matrix % add_value(neig_mat_idx, val)
 
             end if
 
@@ -365,14 +300,13 @@ contains
 
             ! get neighbor matrix index
             call indices_to_matrix(g, neig_idx(1), neig_idx(2), neig_idx(3), &
-                 neig_mat_idx)
+                 neig_mat_idx, ng, nx, ny)
 
             ! compute value and record to bank
             val = jn/hxyz(xyz_idx)
 
             ! record value in matrix
-            call MatSetValue(this%M, irow, neig_mat_idx-1, val, &
-                 INSERT_VALUES, ierr)
+            call loss_matrix % add_value(neig_mat_idx, val)
 
           end if
 
@@ -391,7 +325,7 @@ contains
       val = jnet + totxs - scattxsgg
 
       ! record diagonal term
-      call MatSetValue(this%M, irow, irow, val, INSERT_VALUES, ierr)
+      call loss_matrix % add_value(irow, val)
 
       ! begin loop over off diagonal in-scattering
       SCATTR: do h = 1, ng
@@ -400,7 +334,7 @@ contains
         if (h == g) cycle
 
         ! get neighbor matrix index
-        call indices_to_matrix(h, i, j, k, scatt_mat_idx)
+        call indices_to_matrix(h, i, j, k, scatt_mat_idx, ng, nx, ny)
 
         ! get scattering macro xs
         scattxshg = cmfd%scattxs(h, g, i, j, k)
@@ -410,39 +344,49 @@ contains
 
         ! check for adjoint and bank value
         if (adjoint_calc) then
-          call MatSetValue(this%M, scatt_mat_idx-1, irow, val, &
-               INSERT_VALUES, ierr)
+
+          ! get scattering macro xs, transposed!
+          scattxshg = cmfd%scattxs(g, h, i, j, k)
+
+          ! negate the scattering xs 
+          val = -scattxshg
+
+          ! record value in matrix
+          call loss_matrix % add_value(scatt_mat_idx, val)
+
         else
-          call MatSetValue(this%M, irow, scatt_mat_idx-1, val, &
-               INSERT_VALUES, ierr)
+
+          ! get scattering macro xs
+          scattxshg = cmfd%scattxs(h, g, i, j, k)
+
+          ! negate the scattering xs 
+          val = -scattxshg
+
+          ! record value in matrix
+          call loss_matrix % add_value(scatt_mat_idx, val)
+
         end if
 
       end do SCATTR
 
     end do ROWS 
 
-    ! assemble matrix
-    call MatAssemblyBegin(this%M, MAT_FLUSH_ASSEMBLY, ierr)
-    call MatAssemblyEnd(this%M, MAT_FINAL_ASSEMBLY, ierr)
-
-    ! print out operator to file
-    if (cmfd_write_matrices) call print_M_operator(this)
-
   end subroutine build_loss_matrix
 
 !===============================================================================
 ! INDICES_TO_MATRIX takes (x,y,z,g) indices and computes location in matrix
 !===============================================================================
 
-  subroutine indices_to_matrix(g, i, j, k, matidx)
-
-    use global,  only: cmfd, cmfd_coremap
+  subroutine indices_to_matrix(g, i, j, k, matidx, ng, nx, ny)
 
     integer :: matidx ! the index location in matrix
     integer :: i      ! current x index
     integer :: j      ! current y index
     integer :: k      ! current z index
     integer :: g      ! current group index
+    integer :: nx     ! maximum number of x cells
+    integer :: ny     ! maximum number of y cells
+    integer :: ng     ! maximum number of groups
 
     ! check if coremap is used
     if (cmfd_coremap) then
@@ -463,15 +407,17 @@ contains
 ! MATRIX_TO_INDICES
 !===============================================================================
 
-  subroutine matrix_to_indices(irow, g, i, j, k)
-
-    use global,  only: cmfd, cmfd_coremap
+  subroutine matrix_to_indices(irow, g, i, j, k, ng, nx, ny, nz)
 
     integer :: i     ! iteration counter for x
     integer :: j     ! iteration counter for y
     integer :: k     ! iteration counter for z
     integer :: g     ! iteration counter for groups
     integer :: irow  ! iteration counter over row (0 reference)
+    integer :: nx    ! maximum number of x cells
+    integer :: ny    ! maximum number of y cells
+    integer :: nz    ! maximum number of z cells
+    integer :: ng    ! maximum number of groups
 
     ! check for core map
     if (cmfd_coremap) then
@@ -494,46 +440,4 @@ contains
 
   end subroutine matrix_to_indices
 
-!===============================================================================
-! PRINT_M_OPERATOR
-!===============================================================================
-
-  subroutine print_M_operator(this)
-
-    type(loss_operator) :: this
-
-    PetscViewer :: viewer
-
-    ! write out matrix in binary file (debugging)
-    if (adjoint_calc) then
-      call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'adj_lossmat.bin', &
-           FILE_MODE_WRITE, viewer, ierr)
-    else
-      call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'lossmat.bin', &
-           FILE_MODE_WRITE, viewer, ierr)
-    end if
-    call MatView(this%M, viewer, ierr)
-    call PetscViewerDestroy(viewer, ierr)
-
-  end subroutine print_M_operator
-
-!==============================================================================
-! DESTROY_M_OPERATOR
-!==============================================================================
-
-  subroutine destroy_M_operator(this)
-
-    type(loss_operator) :: this
-
-    ! deallocate matrix
-    call MatDestroy(this%M, ierr)
-
-    ! deallocate other parameters
-    if (allocated(this%d_nnz)) deallocate(this%d_nnz)
-    if (allocated(this%o_nnz)) deallocate(this%o_nnz)
-
-  end subroutine destroy_M_operator
-
-# endif
-
 end module cmfd_loss_operator
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index 2b53facd96..82a80ad396 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -7,9 +7,9 @@ module matrix_header
 #  include <finclude/petsc.h90>
 # endif
 
-  type, public :: matrix
+  type, public :: Matrix
     integer :: n        ! number of rows/cols in matrix
-    integer :: nz       ! number of nonzeros in matrix
+    integer :: nnz      ! number of nonzeros in matrix
     integer :: n_kount  ! counter for length of matrix
     integer :: nz_kount ! counter for number of non zeros
     integer, allocatable :: row(:) ! csr row vector
@@ -34,21 +34,25 @@ contains
 ! MATRIX_CREATE allocates CSR vectors
 !===============================================================================
 
-  subroutine matrix_create(self, n, nz)
+  subroutine matrix_create(self, n, nnz)
 
     integer       :: n
-    integer       :: nz
-    class(matrix) :: self
+    integer       :: nnz
+    class(Matrix) :: self
 
     ! preallocate vectors
     if (.not.allocated(self % row)) allocate(self % row(n+1))
-    if (.not.allocated(self % col)) allocate(self % col(nz))
-    if (.not.allocated(self % val)) allocate(self % val(nz))
+    if (.not.allocated(self % col)) allocate(self % col(nnz))
+    if (.not.allocated(self % val)) allocate(self % val(nnz))
 
     ! set counters to 1
     self % n_kount  = 1
     self % nz_kount = 1
 
+    ! set n and nnz
+    self % n = n
+    self % nnz = nnz
+
   end subroutine matrix_create
 
 !===============================================================================
@@ -57,7 +61,7 @@ contains
 
   subroutine matrix_destroy(self)
 
-    class(matrix) :: self
+    class(Matrix) :: self
 
     if (allocated(self % row)) deallocate(self % row)
     if (allocated(self % col)) deallocate(self % col)
@@ -73,7 +77,7 @@ contains
 
     integer       :: col
     real(8)       :: val
-    class(matrix) :: self
+    class(Matrix) :: self
 
     self % col(self % nz_kount) = col
     self % val(self % nz_kount) = val
@@ -87,7 +91,7 @@ contains
 
   subroutine matrix_new_row(self)
 
-    class(matrix) :: self
+    class(Matrix) :: self
 
     self % row(self % n_kount) = self % nz_kount
     self % n_kount = self % n_kount + 1
@@ -102,7 +106,7 @@ contains
 
   subroutine matrix_setup_petsc(self)
 
-    class(matrix) :: self
+    class(Matrix) :: self
 
     integer :: petsc_err
 

From 3749332218df54e885417e837ae9a20794b418e9 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 8 Jul 2013 12:01:18 -0400
Subject: [PATCH 04/69] fixed some redundant statements and simplified

---
 src/cmfd_loss_operator.F90 | 30 +++++++-----------------------
 1 file changed, 7 insertions(+), 23 deletions(-)

diff --git a/src/cmfd_loss_operator.F90 b/src/cmfd_loss_operator.F90
index 258fa650c4..0e86d38a63 100644
--- a/src/cmfd_loss_operator.F90
+++ b/src/cmfd_loss_operator.F90
@@ -336,37 +336,21 @@ contains
         ! get neighbor matrix index
         call indices_to_matrix(h, i, j, k, scatt_mat_idx, ng, nx, ny)
 
-        ! get scattering macro xs
-        scattxshg = cmfd%scattxs(h, g, i, j, k)
-
-        ! record value in matrix (negate it)
-        val = -scattxshg
-
-        ! check for adjoint and bank value
+        ! check for adjoint
         if (adjoint_calc) then
-
           ! get scattering macro xs, transposed!
           scattxshg = cmfd%scattxs(g, h, i, j, k)
-
-          ! negate the scattering xs 
-          val = -scattxshg
-
-          ! record value in matrix
-          call loss_matrix % add_value(scatt_mat_idx, val)
-
         else
-
           ! get scattering macro xs
           scattxshg = cmfd%scattxs(h, g, i, j, k)
-
-          ! negate the scattering xs 
-          val = -scattxshg
-
-          ! record value in matrix
-          call loss_matrix % add_value(scatt_mat_idx, val)
-
         end if
 
+        ! negate the scattering xs 
+        val = -scattxshg
+
+        ! record value in matrix
+        call loss_matrix % add_value(scatt_mat_idx, val)
+
       end do SCATTR
 
     end do ROWS 

From 9703a55c684159fb0a0fb21b0b6a349e2bd8ddea Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 8 Jul 2013 12:02:26 -0400
Subject: [PATCH 05/69] change prod operator construction

---
 src/DEPENDENCIES           |   1 +
 src/cmfd_prod_operator.F90 | 319 ++++++++-----------------------------
 2 files changed, 65 insertions(+), 255 deletions(-)

diff --git a/src/DEPENDENCIES b/src/DEPENDENCIES
index 973386c406..ab014e45fd 100644
--- a/src/DEPENDENCIES
+++ b/src/DEPENDENCIES
@@ -87,6 +87,7 @@ source.o: string.o
 
 cmfd_prod_operator.o: constants.o
 cmfd_prod_operator.o: global.o
+cmfd_prod_operator.o: matrix_header.o
 
 cmfd_slepc_solver.o: cmfd_loss_operator.o
 cmfd_slepc_solver.o: cmfd_prod_operator.o
diff --git a/src/cmfd_prod_operator.F90 b/src/cmfd_prod_operator.F90
index a5eac6df74..02f2559a36 100644
--- a/src/cmfd_prod_operator.F90
+++ b/src/cmfd_prod_operator.F90
@@ -1,66 +1,29 @@
 module cmfd_prod_operator
 
-# ifdef PETSC
+  use constants,      only: CMFD_NOACCEL
+  use global,         only: cmfd, cmfd_coremap
+  use matrix_header,  only: Matrix
 
   implicit none
   private
-  public :: init_F_operator,build_prod_matrix,destroy_F_operator
-
-# include <finclude/petsc.h90>
-
-  integer  :: nx   ! maximum number of x cells
-  integer  :: ny   ! maximum number of y cells
-  integer  :: nz   ! maximum number of z cells
-  integer  :: ng   ! maximum number of groups
-  integer  :: ierr ! petsc error code
-
-  type, public :: prod_operator
-
-    Mat      :: F    ! petsc matrix for neutronic prod operator
-    integer  :: n    ! dimensions of matrix
-    integer  :: nnz  ! max number of nonzeros
-    integer  :: localn ! local size on proc
-    integer, allocatable :: d_nnz(:) ! vector of diagonal preallocation
-    integer, allocatable :: o_nnz(:) ! vector of off-diagonal preallocation
-
-  end type prod_operator
-
-  logical :: adjoint_calc = .false. ! adjoint calculation 
+  public :: init_prod_matrix, build_prod_matrix
 
 contains
 
 !==============================================================================
-! INIT_F_OPERATOR
+! INIT_PROD_MATRIX
 !==============================================================================
 
-  subroutine init_F_operator(this)
+  subroutine init_prod_matrix(prod_matrix)
 
-    type(prod_operator) :: this
+    type(Matrix) :: prod_matrix
 
-    ! get indices
-    call get_F_indices(this)
-
-    ! get preallocation
-    call preallocate_prod_matrix(this)
-
-    ! set up M operator
-    call MatCreateAIJ(PETSC_COMM_WORLD, this%localn, this%localn, PETSC_DECIDE, &
-         PETSC_DECIDE, PETSC_NULL_INTEGER, this%d_nnz, PETSC_NULL_INTEGER, &
-         this%o_nnz, this%F, ierr)
-    call MatSetOption(this%F, MAT_NEW_NONZERO_LOCATIONS, PETSC_TRUE, ierr)
-    call MatSetOption(this%F, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE, ierr)
-
-  end subroutine init_F_operator
-
-!==============================================================================
-! GET_F_INDICES
-!==============================================================================
-
-  subroutine get_F_indices(this)
-
-    use global,  only: cmfd, cmfd_coremap
-
-    type(prod_operator) :: this
+    integer :: nx   ! maximum number of x cells
+    integer :: ny   ! maximum number of y cells
+    integer :: nz   ! maximum number of z cells
+    integer :: ng   ! maximum number of groups
+    integer :: n    ! total length of matrix
+    integer :: nnz  ! number of nonzeros in matrix
 
     ! get maximum number of cells in each direction
     nx = cmfd%indices(1)
@@ -68,132 +31,26 @@ contains
     nz = cmfd%indices(3)
     ng = cmfd%indices(4)
 
-    ! get number of nonzeros
-    this%nnz = 7 + ng - 1
-
-    ! calculate dimensions of matrix
+    ! calculate dimensions and number of nonzeros in matrix
     if (cmfd_coremap) then
-      this%n = cmfd % mat_dim * ng
+      n = cmfd % mat_dim * ng
     else
-      this%n = nx*ny*nz*ng
+      n = nx*ny*nz*ng
     end if
+    nz = n * ng
 
-  end subroutine get_F_indices
+    ! configure prod matrix
+    call prod_matrix % create(n, nnz)
+
+  end subroutine init_prod_matrix
 
 !===============================================================================
-! PREALLOCATE_PROD_MATRIX
+! BUILD_PROD_MATRIX creates the matrix representing production of neutrons
 !===============================================================================
 
-  subroutine preallocate_prod_matrix(this)
+  subroutine build_prod_matrix(prod_matrix, adjoint)
 
-    use constants,  only: CMFD_NOACCEL
-    use global,     only: cmfd, cmfd_coremap
-
-    type(prod_operator) :: this
-
-    integer :: rank          ! rank of processor
-    integer :: sizen         ! number of procs
-    integer :: i             ! iteration counter for x
-    integer :: j             ! iteration counter for y
-    integer :: k             ! iteration counter for z
-    integer :: g             ! iteration counter for groups
-    integer :: h             ! energy group when doing scattering
-    integer :: n             ! the extent of the matrix
-    integer :: irow          ! row counter
-    integer :: row_start     ! index of local starting row
-    integer :: row_end       ! index of local final row
-    integer :: hmat_idx      ! index in matrix for energy group h
-
-    ! initialize size and rank
-    rank = 0
-    sizen = 0
-
-    ! get rank and max rank of procs
-    call MPI_COMM_RANK(PETSC_COMM_WORLD, rank, ierr)
-    call MPI_COMM_SIZE(PETSC_COMM_WORLD, sizen, ierr)
-
-    ! get local problem size
-    n = this%n
-
-    ! determine local size, divide evenly between all other procs
-    this%localn = n/(sizen)
-
-    ! add 1 more if less proc id is less than mod
-    if (rank < mod(n, sizen)) this%localn = this%localn + 1
-
-    ! determine local starting row
-    row_start = 0
-    if (rank < mod(n, sizen)) then
-      row_start = rank*(n/sizen+1)
-    else
-      row_start = min(mod(n, sizen)*(n/sizen+1) + &
-           (rank - mod(n, sizen))*(n/sizen), n)
-    end if
-
-    ! determine local final row
-    row_end = row_start + this%localn - 1
-
-    ! allocate counters
-    if (.not. allocated(this%d_nnz)) allocate(this%d_nnz(row_start:row_end))
-    if (.not. allocated(this%o_nnz)) allocate(this%o_nnz(row_start:row_end))
-    this % d_nnz = 0
-    this % o_nnz = 0
-
-    ! begin loop around local rows
-    ROWS: do irow = row_start, row_end
-
-      ! initialize counters 
-      this%d_nnz(irow) = 1 ! already add in matrix diagonal
-      this%o_nnz(irow) = 0
-
-      ! get location indices
-      call matrix_to_indices(irow, g, i, j, k)
-
-      ! check if not including reflector
-      if (cmfd_coremap) then
-
-        ! check if at a reflector
-        if (cmfd % coremap(i,j,k) == CMFD_NOACCEL) then
-          cycle
-        end if
-
-      end if
-
-      ! begin loop over off diagonal in-scattering
-      NFISS: do h = 1, ng
-
-        ! cycle though if h=g
-        if (h == g) then
-          cycle
-        end if
-
-        ! get neighbor matrix index
-        call indices_to_matrix(h, i, j, k, hmat_idx)
-
-        ! record nonzero
-        if (((hmat_idx-1) >= row_start) .and. &
-             ((hmat_idx-1) <= row_end)) then
-          this%d_nnz(irow) = this%d_nnz(irow) + 1
-        else
-          this%o_nnz(irow) = this%o_nnz(irow) + 1
-        end if
-
-      end do NFISS
-
-    end do ROWS
-
-  end subroutine preallocate_prod_matrix
-
-!===============================================================================
-! BUILD_PROD_MATRIX creates the matrix representing loss of neutrons
-!===============================================================================
-
-  subroutine build_prod_matrix(this, adjoint)
-
-    use constants,  only: CMFD_NOACCEL
-    use global,     only: cmfd, cmfd_coremap, cmfd_write_matrices
-
-    type(prod_operator) :: this
+    type(Matrix) :: prod_matrix
     logical, optional :: adjoint
 
     integer :: i                  ! iteration counter for x
@@ -201,29 +58,25 @@ contains
     integer :: k                  ! iteration counter for z
     integer :: g                  ! iteration counter for groups
     integer :: h                  ! energy group when doing scattering
+    integer :: nx                 ! maximum number of x cells
+    integer :: ny                 ! maximum number of y cells
+    integer :: nz                 ! maximum number of z cells
+    integer :: ng                 ! maximum number of groups
     integer :: hmat_idx           ! index in matrix for energy group h
-    integer :: ierr               ! Petsc error code
-    integer :: row_start          ! the first local row on the processor
-    integer :: row_finish         ! the last local row on the processor
     integer :: irow               ! iteration counter over row
+    logical :: adjoint_calc       ! is this a physical adjoint?
     real(8) :: nfissxs            ! nufission cross section h-->g
     real(8) :: val                ! temporary variable for nfissxs
 
     ! check for adjoint
+    adjoint_calc = .false.
     if (present(adjoint)) adjoint_calc = adjoint
 
-    ! initialize row start and finish
-    row_start = 0
-    row_finish = 0
-
-    ! get row bounds for this processor
-    call MatGetOwnershipRange(this%F, row_start, row_finish, ierr)
-
     ! begin iteration loops
-    ROWS: do irow = row_start, row_finish-1
+    ROWS: do irow = 1, prod_matrix % n
 
       ! get indices for that row
-      call matrix_to_indices(irow, g, i, j, k)
+      call matrix_to_indices(irow, g, i, j, k, ng, nx, ny, nz)
 
       ! check if not including reflector
       if (cmfd_coremap) then
@@ -238,52 +91,48 @@ contains
       ! loop around all other groups 
       NFISS: do h = 1, ng
 
-        ! get cell data
-        nfissxs = cmfd%nfissxs(h,g,i,j,k)
-
         ! get matrix column location
-        call indices_to_matrix(h, i, j, k, hmat_idx)
-
-        ! reocrd value in matrix
-        val = nfissxs
+        call indices_to_matrix(h, i, j, k, hmat_idx, ng, nx, ny)
 
         ! check for adjoint and bank val
         if (adjoint_calc) then
-          call MatSetValue(this%F, hmat_idx-1, irow, val, INSERT_VALUES, ierr)
+          ! get nu-fission cross section from cell
+          nfissxs = cmfd%nfissxs(g,h,i,j,k)
         else
-          call MatSetValue(this%F, irow, hmat_idx-1, val, INSERT_VALUES, ierr)
+          ! get nu-fission cross section from cell
+          nfissxs = cmfd%nfissxs(h,g,i,j,k)
         end if
 
+        ! set as value to be recorded
+        val = nfissxs
+
+        ! record value in matrix
+        call prod_matrix % add_value(hmat_idx, val)
+
       end do NFISS
 
     end do ROWS 
 
-    ! assemble matrix 
-    call MatAssemblyBegin(this%F, MAT_FLUSH_ASSEMBLY, ierr)
-    call MatAssemblyEnd(this%F, MAT_FINAL_ASSEMBLY, ierr)
-
-    ! print out operator to file
-    if (cmfd_write_matrices) call print_F_operator(this)
-
   end subroutine build_prod_matrix
 
 !===============================================================================
 ! INDICES_TO_MATRIX takes (x,y,z,g) indices and computes location in matrix
 !===============================================================================
 
-  subroutine indices_to_matrix(g, i, j, k, matidx)
+  subroutine indices_to_matrix(g, i, j, k, matidx, ng, nx, ny)
 
-    use global,  only: cmfd, cmfd_coremap
+    integer :: matidx ! the index location in matrix
+    integer :: i      ! current x index
+    integer :: j      ! current y index
+    integer :: k      ! current z index
+    integer :: g      ! current group index
+    integer :: nx     ! maximum number of x cells
+    integer :: ny     ! maximum number of y cells
+    integer :: ng     ! maximum number of groups
 
-    integer :: matidx         ! the index location in matrix
-    integer :: i              ! current x index
-    integer :: j              ! current y index
-    integer :: k              ! current z index
-    integer :: g              ! current group index
-    
     ! check if coremap is used
     if (cmfd_coremap) then
-    
+
       ! get idx from core map
       matidx = ng*(cmfd % coremap(i,j,k)) - (ng - g)
 
@@ -300,21 +149,23 @@ contains
 ! MATRIX_TO_INDICES
 !===============================================================================
 
-  subroutine matrix_to_indices(irow, g, i, j, k)
+  subroutine matrix_to_indices(irow, g, i, j, k, ng, nx, ny, nz)
 
-    use global,  only: cmfd, cmfd_coremap
-
-    integer :: i                    ! iteration counter for x
-    integer :: j                    ! iteration counter for y
-    integer :: k                    ! iteration counter for z
-    integer :: g                    ! iteration counter for groups
-    integer :: irow                 ! iteration counter over row (0 reference)
+    integer :: i     ! iteration counter for x
+    integer :: j     ! iteration counter for y
+    integer :: k     ! iteration counter for z
+    integer :: g     ! iteration counter for groups
+    integer :: irow  ! iteration counter over row (0 reference)
+    integer :: nx    ! maximum number of x cells
+    integer :: ny    ! maximum number of y cells
+    integer :: nz    ! maximum number of z cells
+    integer :: ng    ! maximum number of groups
 
     ! check for core map
     if (cmfd_coremap) then
 
       ! get indices from indexmap
-      g = mod(irow,ng) + 1
+      g = mod(irow, ng) + 1
       i = cmfd % indexmap(irow/ng+1,1)
       j = cmfd % indexmap(irow/ng+1,2)
       k = cmfd % indexmap(irow/ng+1,3)
@@ -331,46 +182,4 @@ contains
 
   end subroutine matrix_to_indices
 
-!===============================================================================
-! PRINT_F_OPERATOR
-!===============================================================================
-
-  subroutine print_F_operator(this)
-
-    type(prod_operator) :: this
-
-    PetscViewer :: viewer
-
-    ! write out matrix in binary file (debugging)
-    if (adjoint_calc) then
-      call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'adj_prodmat.bin', &   
-           FILE_MODE_WRITE, viewer, ierr)
-    else
-      call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'prodmat.bin', &
-           FILE_MODE_WRITE, viewer, ierr)
-    end if
-    call MatView(this%F, viewer, ierr)
-    call PetscViewerDestroy(viewer, ierr)
-
-  end subroutine print_F_operator
-
-!==============================================================================
-! DESTROY_F_OPERATOR
-!==============================================================================
-
-  subroutine destroy_F_operator(this)
-
-    type(prod_operator) :: this
-
-    ! deallocate matrix
-    call MatDestroy(this%F, ierr)
-
-    ! deallocate other parameters
-    if (allocated(this%d_nnz)) deallocate(this%d_nnz)
-    if (allocated(this%o_nnz)) deallocate(this%o_nnz)
-
-  end subroutine destroy_F_operator
-
-# endif 
-
 end module cmfd_prod_operator

From 9f5255e2be3d6308b8e3fcaff1e6e439b5161162 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 8 Jul 2013 13:47:01 -0400
Subject: [PATCH 06/69] added a vector class

---
 src/DEPENDENCIES      |  2 +
 src/OBJECTS           |  3 +-
 src/vector_header.F90 | 95 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 99 insertions(+), 1 deletion(-)
 create mode 100644 src/vector_header.F90

diff --git a/src/DEPENDENCIES b/src/DEPENDENCIES
index ab014e45fd..7dfbfe54f3 100644
--- a/src/DEPENDENCIES
+++ b/src/DEPENDENCIES
@@ -264,6 +264,8 @@ cross_section.o: material_header.o
 cross_section.o: random_lcg.o
 cross_section.o: search.o
 
+vector_header.o: constants.o
+
 state_point.o: constants.o
 state_point.o: error.o
 state_point.o: global.o
diff --git a/src/OBJECTS b/src/OBJECTS
index 030679cca6..af668b4b5b 100644
--- a/src/OBJECTS
+++ b/src/OBJECTS
@@ -61,4 +61,5 @@ string.o \
 tally.o \
 tally_header.o \
 tally_initialize.o \
-timer_header.o
+timer_header.o \
+vector_header.o
diff --git a/src/vector_header.F90 b/src/vector_header.F90
new file mode 100644
index 0000000000..7f2abd7113
--- /dev/null
+++ b/src/vector_header.F90
@@ -0,0 +1,95 @@
+module vector_header
+
+  use constants,  only: ZERO
+
+  implicit none
+  private
+
+# ifdef PETSC
+#  include <finclude/petsc.h90>
+# endif
+
+  type, public :: Vector 
+    integer :: n        ! number of rows/cols in matrix
+    real(8), allocatable :: val(:) ! matrix value vector
+#  ifdef PETSC
+    Vec :: petsc_vec
+#  endif
+   contains
+     procedure :: create       => vector_create
+     procedure :: destroy      => vector_destroy
+     procedure :: add_value    => vector_add_value
+#   ifdef PETSC
+     procedure :: setup_petsc  => vector_setup_petsc
+#   endif
+  end type Vector
+
+contains
+
+!===============================================================================
+! VECTOR_CREATE allocates and initializes a vector 
+!===============================================================================
+
+  subroutine vector_create(self, n)
+
+    integer       :: n
+    class(Vector) :: self
+
+    ! preallocate vector
+    if (.not.allocated(self % val)) allocate(self % val(n))
+
+    ! set n
+    self % n = n
+
+    ! initialize to zero
+    self % val = ZERO
+
+  end subroutine vector_create
+
+!===============================================================================
+! VECTOR_DESTROY deallocates all space associated with a vector
+!===============================================================================
+
+  subroutine vector_destroy(self)
+
+    class(Vector) :: self
+
+    if (allocated(self % val)) deallocate(self % val)
+
+  end subroutine vector_destroy
+
+!===============================================================================
+! VECTOR_ADD_VALUE adds a value to the vector
+!===============================================================================
+
+  subroutine vector_add_value(self, idx, val)
+
+    integer       :: idx
+    real(8)       :: val
+    class(Vector) :: self
+
+    self % val(idx) = val
+
+  end subroutine vector_add_value
+
+# ifdef PETSC
+
+!===============================================================================
+! VECTOR_SETUP_PETSC
+!===============================================================================
+
+  subroutine vector_setup_petsc(self)
+
+    class(Vector) :: self
+
+    integer :: petsc_err
+
+    ! link to petsc
+    call VecCreateSeqWithArray(PETSC_COMM_WORLD, 1, self % n, self % val, &
+         self % petsc_vec, petsc_err) 
+
+  end subroutine vector_setup_petsc
+
+# endif
+
+end module vector_header

From e06137938cbc014d4e44b65653fb4f1b35d3b10c Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 8 Jul 2013 17:01:44 -0400
Subject: [PATCH 07/69] changed over power iteration and added more methods

---
 src/DEPENDENCIES           |   6 +-
 src/cmfd_loss_operator.F90 |  19 ++-
 src/cmfd_power_solver.F90  | 292 ++++++++++---------------------------
 src/cmfd_prod_operator.F90 |  30 +++-
 src/matrix_header.F90      | 161 +++++++++++++++++++-
 src/petsc_solver.F90       |  51 ++++---
 src/vector_header.F90      |  19 +++
 7 files changed, 321 insertions(+), 257 deletions(-)

diff --git a/src/DEPENDENCIES b/src/DEPENDENCIES
index 7dfbfe54f3..f74355babb 100644
--- a/src/DEPENDENCIES
+++ b/src/DEPENDENCIES
@@ -39,7 +39,9 @@ cmfd_power_solver.o: cmfd_loss_operator.o
 cmfd_power_solver.o: cmfd_prod_operator.o
 cmfd_power_solver.o: constants.o
 cmfd_power_solver.o: global.o
-cmfd_power_solver.o: string.o
+cmfd_power_solver.o: matrix_header.o
+cmfd_power_solver.o: petsc_solver.o
+cmfd_power_solver.o: vector_header.o
 
 cmfd_message_passing.o: cmfd_header.o
 cmfd_message_passing.o: global.o
@@ -101,8 +103,6 @@ fission.o: global.o
 fission.o: interpolation.o
 fission.o: search.o
 
-cmfd_jacobian_operator.o: cmfd_loss_operator.o
-cmfd_jacobian_operator.o: cmfd_prod_operator.o
 cmfd_jacobian_operator.o: constants.o
 cmfd_jacobian_operator.o: global.o
 
diff --git a/src/cmfd_loss_operator.F90 b/src/cmfd_loss_operator.F90
index 0e86d38a63..67551f78b9 100644
--- a/src/cmfd_loss_operator.F90
+++ b/src/cmfd_loss_operator.F90
@@ -234,7 +234,7 @@ contains
     ROWS: do irow = 1, loss_matrix % n
 
       ! set up a new row in matrix
-      call loss_matrix % new_row
+      call loss_matrix % new_row()
 
       ! get indices for that row
       call matrix_to_indices(irow, g, i, j, k, ng, nx, ny, nz)
@@ -355,6 +355,9 @@ contains
 
     end do ROWS 
 
+    ! CSR requires n+1 row
+    call loss_matrix % new_row()
+
   end subroutine build_loss_matrix
 
 !===============================================================================
@@ -408,17 +411,17 @@ contains
 
       ! get indices from indexmap
       g = mod(irow, ng) + 1
-      i = cmfd % indexmap(irow/ng+1,1)
-      j = cmfd % indexmap(irow/ng+1,2)
-      k = cmfd % indexmap(irow/ng+1,3)
+      i = cmfd % indexmap((irow-1)/ng+1,1)
+      j = cmfd % indexmap((irow-1)/ng+1,2)
+      k = cmfd % indexmap((irow-1)/ng+1,3)
 
     else
 
       ! compute indices
-      g = mod(irow, ng) + 1
-      i = mod(irow, ng*nx)/ng + 1
-      j = mod(irow, ng*nx*ny)/(ng*nx)+ 1
-      k = mod(irow, ng*nx*ny*nz)/(ng*nx*ny) + 1
+      g = mod(irow-1, ng) + 1
+      i = mod(irow-1, ng*nx)/ng + 1
+      j = mod(irow-1, ng*nx*ny)/(ng*nx)+ 1
+      k = mod(irow-1, ng*nx*ny*nz)/(ng*nx*ny) + 1
 
     end if
 
diff --git a/src/cmfd_power_solver.F90 b/src/cmfd_power_solver.F90
index 8376b964d0..0a258aac13 100644
--- a/src/cmfd_power_solver.F90
+++ b/src/cmfd_power_solver.F90
@@ -1,41 +1,30 @@
 module cmfd_power_solver
 
-# ifdef PETSC
-
 ! This module contains routines to execute the power iteration solver
 
-  use cmfd_loss_operator, only: loss_operator,init_M_operator, &
-                                build_loss_matrix,destroy_M_operator
-  use cmfd_prod_operator, only: prod_operator,init_F_operator, &
-                                build_prod_matrix,destroy_F_operator
+  use cmfd_loss_operator, only: init_loss_matrix, build_loss_matrix
+  use cmfd_prod_operator, only: init_prod_matrix, build_prod_matrix
+  use matrix_header,      only: Matrix
+  use petsc_solver,       only: Petsc_gmres
+  use vector_header,      only: Vector
 
   implicit none
   private
   public :: cmfd_power_execute 
 
-# include <finclude/petsc.h90>
-
-  type(loss_operator) :: loss   ! M loss matrix
-  type(prod_operator) :: prod   ! F production matrix
-
-  Vec     :: phi_n                ! new flux eigenvector
-  Vec     :: phi_o                ! old flux eigenvector
-  Vec     :: S_n                  ! new source vector
-  Vec     :: S_o                  ! old source vector
-  KSP     :: krylov               ! krylov solver
-  KSP     :: sub_krylov           ! sub-ksp for bjacobi
-  PC      :: prec                 ! preconditioner for krylov
-  PC      :: sub_prec             ! sub-prec for bjacobi
-  integer :: ierr                 ! error flag
-  integer :: nlocal
-  integer :: first
+  logical :: iconv                ! did the problem converged
   real(8) :: k_n                  ! new k-eigenvalue
   real(8) :: k_o                  ! old k-eigenvalue
   real(8) :: ktol = 1.e-8_8       ! tolerance on keff
   real(8) :: stol = 1.e-8_8       ! tolerance on source
-  logical :: iconv                ! did the problem converged
-
   logical :: adjoint_calc = .false. ! run an adjoint calculation
+  type(Matrix) :: loss            ! cmfd loss matrix
+  type(Matrix) :: prod            ! cmfd prod matrix
+  type(Vector) :: phi_n           ! new flux vector
+  type(Vector) :: phi_o           ! old flux vector
+  type(Vector) :: S_n             ! new source vector
+  type(Vector) :: S_o             ! old flux vector
+  type(Petsc_gmres) :: gmres      ! gmres solver
 
 contains
 
@@ -45,7 +34,7 @@ contains
 
   subroutine cmfd_power_execute(k_tol, s_tol, adjoint)
 
-    use global,  only: cmfd_adjoint_type, n_procs_cmfd
+    use global,  only: cmfd_adjoint_type
 
     real(8), optional :: k_tol    ! tolerance on keff
     real(8), optional :: s_tol    ! tolerance on source
@@ -65,26 +54,20 @@ contains
         physical_adjoint = .true.
 
     ! initialize solver
-    call init_solver()
+    call gmres % create()
 
     ! initialize matrices and vectors
-    call init_data()
-
-    ! set up M loss matrix
-    call build_loss_matrix(loss, adjoint=physical_adjoint) 
-
-    ! set up F production matrix
-    call build_prod_matrix(prod, adjoint=physical_adjoint)
+    call init_data(physical_adjoint)
 
     ! check for adjoint calculation
     if (adjoint_calc .and. trim(cmfd_adjoint_type) == 'math') &
         call compute_adjoint()
 
     ! set up krylov info
-    call KSPSetOperators(krylov, loss%M, loss%M, SAME_NONZERO_PATTERN, ierr)
+    call gmres % set_oper(loss % petsc_mat, loss % petsc_mat)
 
     ! precondition matrix
-    call precondition_matrix()
+    call gmres % precondition(loss % petsc_mat)
 
     ! begin power iteration 
     call execute_power_iter()
@@ -98,122 +81,58 @@ contains
   end subroutine cmfd_power_execute
 
 !===============================================================================
-! INIT_DATA allocates matrices vectors for CMFD solution
+! INIT_DATA allocates matrices and vectors for CMFD solution
 !===============================================================================
 
-  subroutine init_data()
+  subroutine init_data(adjoint)
 
     use constants, only: ONE
 
+    logical :: adjoint
+
     integer :: n      ! problem size
     real(8) :: guess  ! initial guess
 
     ! set up matrices
-    call init_M_operator(loss)
-    call init_F_operator(prod)
+    call init_loss_matrix(loss)
+    call init_prod_matrix(prod)
 
     ! get problem size
-    n = loss%localn
+    n = loss % n
 
     ! set up flux vectors
-    call VecCreateMPI(PETSC_COMM_WORLD, n, PETSC_DECIDE, phi_n, ierr)
-    call VecCreateMPI(PETSC_COMM_WORLD, n, PETSC_DECIDE, phi_o, ierr)
+    call phi_n % create(n)
+    call phi_o % create(n)
 
     ! set up source vectors
-    call VecCreateMPI(PETSC_COMM_WORLD, n, PETSC_DECIDE, S_n, ierr)
-    call VecCreateMPI(PETSC_COMM_WORLD, n, PETSC_DECIDE, S_o, ierr)
+    call S_n % create(n)
+    call S_o % create(n)
 
     ! set initial guess
     guess = ONE
-    call VecSet(phi_n, guess, ierr)
-    call VecSet(phi_o, guess, ierr)
+    phi_n % val = guess
+    phi_o % val = guess
     k_n = guess
     k_o = guess
 
+    ! set up loss matrix
+    call build_loss_matrix(loss, adjoint=adjoint) 
+
+    ! set up production matrix
+    call build_prod_matrix(prod, adjoint=adjoint)
+
+    ! setup petsc for everything
+    call loss % assemble()
+    call prod % assemble()
+    call loss % setup_petsc()
+    call prod % setup_petsc()
+    call phi_n % setup_petsc()
+    call phi_o % setup_petsc()
+    call S_o % setup_petsc()
+    call S_n % setup_petsc()
+
   end subroutine init_data
 
-!===============================================================================
-! INIT_SOLVER
-!===============================================================================
-
-  subroutine init_solver()
-
-    real(8)  :: rtol ! relative tolerance
-    real(8)  :: atol ! absolute tolerance  
-
-    ! set tolerance
-    rtol = 1.0e-10_8
-    atol = 1.0e-10_8
-
-    ! set up krylov solver
-    call KSPCreate(PETSC_COMM_WORLD, krylov, ierr)
-    call KSPSetTolerances(krylov, rtol, atol, PETSC_DEFAULT_DOUBLE_PRECISION, &
-         PETSC_DEFAULT_INTEGER, ierr)
-    call KSPSetType(krylov, KSPGMRES, ierr)
-    call KSPSetInitialGuessNonzero(krylov, PETSC_TRUE, ierr)
-
-  end subroutine init_solver
-
-!===============================================================================
-! PRECONDITION_MATRIX 
-!===============================================================================
-
-  subroutine precondition_matrix()
-
-    use global, only: cmfd_power_monitor, cmfd_solver_type, &
-                      n_procs_cmfd, cmfd_ilu_levels, master
-    use string, only: to_str
-    use, intrinsic :: ISO_FORTRAN_ENV
-
-    character(len=20) :: ksptype,pctype
- 
-    ! set up preconditioner
-    call KSPGetPC(krylov, prec, ierr)
-    if (n_procs_cmfd == 1) then
-      call PCSetType(prec, PCILU, ierr)
-      call PCFactorSetLevels(prec, cmfd_ilu_levels, ierr)
-      call KSPSetUp(krylov, ierr)
-      call PCFactorGetMatrix(prec, loss%M, ierr)
-    else
-      call PetscOptionsSetValue("-pc_type", "bjacobi", ierr)
-      call PetscOptionsSetValue("-sub_pc_type", "ilu", ierr)  
-      call PetscOptionsSetValue("-sub_pc_factor_levels", &
-           trim(to_str(cmfd_ilu_levels)), ierr)
-      call PCSetFromOptions(prec, ierr)
-      call KSPSetUp(krylov, ierr)
-    end if
-
-    ! get options
-    if (trim(cmfd_solver_type) == 'power') call KSPSetFromOptions(krylov, ierr)
-
-    ! get all types and print
-    call KSPGetType(krylov, ksptype, ierr)
-    call PCGetType(prec, pctype, ierr)
-
-    ! print heading information
-    if (cmfd_power_monitor .and. master) then
-      write(OUTPUT_UNIT,'(A)') ''
-      write(OUTPUT_UNIT,'(A)') '########################################################'
-      write(OUTPUT_UNIT,'(A)') '################ Power Iteration Solver ################'
-      write(OUTPUT_UNIT,'(A)') '########################################################'
-      write(OUTPUT_UNIT,'(A)')
-      write(OUTPUT_UNIT,100)   'Eigenvalue Tolerance:', ktol
-      write(OUTPUT_UNIT,100)   'Source Tolerance:    ', stol
-      write(OUTPUT_UNIT,'(A)') ''
-      write(OUTPUT_UNIT,102)   'Linear Solver Type:  ', ksptype
-      write(OUTPUT_UNIT,102)   'Preconditioner Type: ', pctype
-      write(OUTPUT_UNIT,101)   'ILU Fill Levels:', cmfd_ilu_levels
-      write(OUTPUT_UNIT,'(A)') ''
-      write(OUTPUT_UNIT,'(A)') '---------------------------------------------'
-      write(OUTPUT_UNIT,'(A)') ''
-    end if
-
- 100 FORMAT(A,1X,1PE11.4)
- 101 FORMAT(A,1X,I0)
- 102 FORMAT(A,1X,A)
-
-  end subroutine precondition_matrix
-
 !===============================================================================
 ! COMPUTE_ADJOINT 
 !===============================================================================
@@ -222,23 +141,14 @@ contains
 
     use global,  only: cmfd_write_matrices
 
-    PetscViewer :: viewer
-
     ! transpose matrices
-    call MatTranspose(loss%M, MAT_REUSE_MATRIX, loss%M, ierr)
-    call MatTranspose(prod%F, MAT_REUSE_MATRIX, prod%F, ierr)
+    call loss % transpose()
+    call prod % transpose()
 
     ! write out matrix in binary file (debugging)
     if (cmfd_write_matrices) then
-      call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'adj_lossmat.bin', &
-           FILE_MODE_WRITE, viewer, ierr)
-      call MatView(loss%M, viewer, ierr)
-      call PetscViewerDestroy(viewer, ierr)
-
-      call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'adj_prodmat.bin', &
-           FILE_MODE_WRITE, viewer, ierr)
-      call MatView(prod%F, viewer, ierr)
-      call PetscViewerDestroy(viewer, ierr)
+      call loss % write_petsc_binary('adj_lossmat.bin')
+      call prod % write_petsc_binary('adj_prodmat.bin')
     end if
 
   end subroutine compute_adjoint
@@ -250,10 +160,6 @@ contains
 
   subroutine execute_power_iter()
 
-    use constants, only: ONE
-
-    real(8)     :: num       ! numerator for eigenvalue update
-    real(8)     :: den       ! denominator for eigenvalue update
     integer     :: i         ! iteration counter
 
     ! reset convergence flag
@@ -263,24 +169,22 @@ contains
     do i = 1, 10000
 
       ! compute source vector
-      call MatMult(prod%F, phi_o, S_o, ierr)
+      call prod % vector_multiply(phi_o % petsc_vec, S_o % petsc_vec)
 
       ! normalize source vector
-      call VecScale(S_o, ONE/k_o, ierr)
+      S_o % val = S_o % val / k_o
 
       ! compute new flux vector
-      call KSPSolve(krylov, S_o, phi_n, ierr)
+      call gmres % solve(S_o % petsc_vec, phi_n % petsc_vec)
 
       ! compute new source vector
-      call MatMult(prod%F, phi_n, S_n, ierr)
+      call prod % vector_multiply(phi_n % petsc_vec, S_n % petsc_vec)
 
       ! compute new k-eigenvalue
-      call VecSum(S_n, num, ierr)
-      call VecSum(S_o, den, ierr)
-      k_n = num/den
+      k_n = sum(S_n % val) / sum(S_o % val)
 
       ! renormalize the old source
-      call VecScale(S_o, k_o, ierr)
+      S_o % val = S_o % val * k_o
 
       ! check convergence
       call convergence(i)
@@ -289,7 +193,7 @@ contains
       if (iconv) exit
 
       ! record old values
-      call VecCopy(phi_n, phi_o, ierr)
+      phi_o % val = phi_n % val
       k_o = k_n
 
     end do
@@ -302,16 +206,14 @@ contains
 
   subroutine convergence(iter)
 
-    use global,  only: cmfd_power_monitor, master
+    use constants,  only: ONE
+    use global,     only: cmfd_power_monitor, master
     use, intrinsic :: ISO_FORTRAN_ENV
 
     integer     :: iter           ! iteration number
 
     real(8)     :: kerr           ! error in keff
     real(8)     :: serr           ! error in source
-    real(8)     :: norm_n         ! L2 norm of new source
-    real(8)     :: norm_o         ! L2 norm of old source
-    integer     :: ierr           ! petsc error code
 
     ! reset convergence flag
     iconv = .false.
@@ -320,9 +222,7 @@ contains
     kerr = abs(k_o - k_n)/k_n
 
     ! calculate max error in source
-    call VecNorm(S_n, NORM_2, norm_n, ierr)
-    call VecNorm(S_o, NORM_2, norm_o, ierr)
-    serr = abs(norm_n-norm_o)/norm_n
+    serr = sqrt(ONE/dble(S_n % n) * sum(((S_n % val - S_o % val)/S_n % val)**2))
 
     ! check for convergence
     if(kerr < ktol .and. serr < stol) iconv = .true.
@@ -341,18 +241,13 @@ contains
 
   subroutine extract_results()
 
-    use constants,        only: ZERO
-    use global,           only: cmfd, n_procs_cmfd, cmfd_write_matrices
+    use global, only: cmfd, cmfd_write_matrices
 
+    character(len=25)    :: filename  ! name of file to write data 
     integer              :: n         ! problem size
-    integer              :: row_start ! local row start
-    integer              :: row_end   ! local row end
-    real(8),allocatable  :: mybuf(:)  ! temp buffer
-    PetscScalar, pointer :: phi_v(:)  ! pointer to eigenvector info
-    PetscViewer          :: viewer    ! petsc viewer for binary write
 
     ! get problem size
-    n = loss%n
+    n = loss % n
 
     ! also allocate in cmfd object
     if (adjoint_calc) then
@@ -360,19 +255,13 @@ contains
     else
       if (.not. allocated(cmfd%phi)) allocate(cmfd%phi(n))
     end if
-    if (.not. allocated(mybuf)) allocate(mybuf(n))
 
-    ! get ownership range
-    call VecGetOwnershipRange(phi_n, row_start, row_end, ierr)
-
-    ! convert petsc phi_object to cmfd_obj
-    call VecGetArrayF90(phi_n, phi_v, ierr)
+    ! save values 
     if (adjoint_calc) then
-      cmfd%adj_phi(row_start+1:row_end) = phi_v
+      cmfd % adj_phi = phi_n % val
     else
-      cmfd%phi(row_start+1:row_end) = phi_v 
+      cmfd % phi = phi_n % val 
     end if
-    call VecRestoreArrayF90(phi_n, phi_v, ierr)
 
     ! save eigenvalue
     if(adjoint_calc) then
@@ -381,16 +270,6 @@ contains
       cmfd%keff = k_n
     end if
 
-    ! reduce result to all 
-    mybuf = ZERO
-    if (adjoint_calc) then
-      call MPI_ALLREDUCE(cmfd%adj_phi, mybuf, n, MPI_REAL8, MPI_SUM, &
-           PETSC_COMM_WORLD, ierr)
-    else
-      call MPI_ALLREDUCE(cmfd%phi, mybuf, n, MPI_REAL8, MPI_SUM, &
-           PETSC_COMM_WORLD, ierr)
-    end if
-
     ! normalize phi to 1
     if (adjoint_calc) then
       cmfd%adj_phi = cmfd%adj_phi/sqrt(sum(cmfd%adj_phi*cmfd%adj_phi))
@@ -401,20 +280,13 @@ contains
     ! write out results
     if (cmfd_write_matrices) then
       if (adjoint_calc) then
-        call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'adj_fluxvec.bin', &
-             FILE_MODE_WRITE, viewer, ierr)
+        filename = 'adj_fluxvec.bin'
       else
-        call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'fluxvec.bin', &
-             FILE_MODE_WRITE, viewer, ierr)
+        filename = 'fluxvec.bin'
       end if
-      call VecView(phi_n, viewer, ierr)
-      call PetscViewerDestroy(viewer, ierr)
+      call phi_n % write_petsc_binary(filename)
     end if
 
-    ! nullify pointer and deallocate local vars
-    if (associated(phi_v)) nullify(phi_v)
-    if (allocated(mybuf)) deallocate(mybuf)
-
   end subroutine extract_results
 
 !===============================================================================
@@ -423,23 +295,15 @@ contains
 
   subroutine finalize()
 
-    use global,  only: n_procs_cmfd
-
-    ! finalize solver objects
-    call KSPDestroy(krylov, ierr)
-    call KSPDestroy(sub_krylov, ierr)
-
-    ! finalize data objects
-    if (n_procs_cmfd > 1) call destroy_M_operator(loss) ! only destroy for jacobi
-    call destroy_F_operator(prod)
-
-    call VecDestroy(phi_n, ierr)
-    call VecDestroy(phi_o, ierr)
-    call VecDestroy(S_n, ierr)
-    call VecDestroy(S_o, ierr)
+    ! destroy all objects 
+    call gmres % destroy() 
+    call loss  % destroy() 
+    call prod  % destroy()
+    call phi_n % destroy()
+    call phi_o % destroy()
+    call S_n   % destroy()
+    call S_o   % destroy()
 
   end subroutine finalize
 
-# endif
-
 end module cmfd_power_solver
diff --git a/src/cmfd_prod_operator.F90 b/src/cmfd_prod_operator.F90
index 02f2559a36..9e017013f8 100644
--- a/src/cmfd_prod_operator.F90
+++ b/src/cmfd_prod_operator.F90
@@ -37,7 +37,7 @@ contains
     else
       n = nx*ny*nz*ng
     end if
-    nz = n * ng
+    nnz = n * ng
 
     ! configure prod matrix
     call prod_matrix % create(n, nnz)
@@ -68,6 +68,12 @@ contains
     real(8) :: nfissxs            ! nufission cross section h-->g
     real(8) :: val                ! temporary variable for nfissxs
 
+    ! get maximum number of cells in each direction
+    nx = cmfd%indices(1)
+    ny = cmfd%indices(2)
+    nz = cmfd%indices(3)
+    ng = cmfd%indices(4)
+
     ! check for adjoint
     adjoint_calc = .false.
     if (present(adjoint)) adjoint_calc = adjoint
@@ -75,6 +81,9 @@ contains
     ! begin iteration loops
     ROWS: do irow = 1, prod_matrix % n
 
+      ! add a new row to matrix
+      call prod_matrix % new_row()
+
       ! get indices for that row
       call matrix_to_indices(irow, g, i, j, k, ng, nx, ny, nz)
 
@@ -89,6 +98,7 @@ contains
       end if
 
       ! loop around all other groups 
+
       NFISS: do h = 1, ng
 
         ! get matrix column location
@@ -107,12 +117,16 @@ contains
         val = nfissxs
 
         ! record value in matrix
+
         call prod_matrix % add_value(hmat_idx, val)
 
       end do NFISS
 
     end do ROWS 
 
+    ! CSR requires n+1 row
+    call prod_matrix % new_row()
+
   end subroutine build_prod_matrix
 
 !===============================================================================
@@ -166,17 +180,17 @@ contains
 
       ! get indices from indexmap
       g = mod(irow, ng) + 1
-      i = cmfd % indexmap(irow/ng+1,1)
-      j = cmfd % indexmap(irow/ng+1,2)
-      k = cmfd % indexmap(irow/ng+1,3)
+      i = cmfd % indexmap((irow-1)/ng+1,1)
+      j = cmfd % indexmap((irow-1)/ng+1,2)
+      k = cmfd % indexmap((irow-1)/ng+1,3)
 
     else
 
       ! compute indices
-      g = mod(irow, ng) + 1
-      i = mod(irow, ng*nx)/ng + 1
-      j = mod(irow, ng*nx*ny)/(ng*nx)+ 1
-      k = mod(irow, ng*nx*ny*nz)/(ng*nx*ny) + 1
+      g = mod(irow-1, ng) + 1
+      i = mod(irow-1, ng*nx)/ng + 1
+      j = mod(irow-1, ng*nx*ny)/(ng*nx)+ 1
+      k = mod(irow-1, ng*nx*ny*nz)/(ng*nx*ny) + 1
 
     end if
 
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index 82a80ad396..f402f32428 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -23,8 +23,12 @@ module matrix_header
      procedure :: destroy      => matrix_destroy
      procedure :: add_value    => matrix_add_value
      procedure :: new_row      => matrix_new_row
+     procedure :: assemble     => matrix_assemble
 #   ifdef PETSC
-     procedure :: setup_petsc  => matrix_setup_petsc
+     procedure :: setup_petsc        => matrix_setup_petsc
+     procedure :: write_petsc_binary => matrix_write_petsc_binary
+     procedure :: transpose          => matrix_transpose
+     procedure :: vector_multiply    => matrix_vector_multiply
 #   endif
   end type matrix
 
@@ -100,6 +104,111 @@ contains
 
 # ifdef PETSC
 
+!===============================================================================
+! MATRIX_ASSEMBLE
+!===============================================================================
+
+  subroutine matrix_assemble(self)
+
+    class(Matrix) :: self
+
+    integer :: i
+    integer :: first
+    integer :: last
+
+    ! loop around row vector
+    do i = 1, self % n
+
+      ! get bounds
+      first = self % row(i)
+      last  = self % row(i+1) - 1
+
+      ! sort a row
+      call sort_csr(self % col, self % val, first, last)
+
+    end do
+
+  end subroutine matrix_assemble
+
+!===============================================================================
+! SORT_CSR
+!===============================================================================
+
+  recursive subroutine sort_csr(col, val, first, last)
+
+    integer :: col(:)
+    integer :: first
+    integer :: last
+    real(8) :: val(:)
+
+    integer :: mid
+
+    if (first < last) then
+      call split(col, val, first, last, mid)
+      call sort_csr(col, val, first, mid-1)
+      call sort_csr(col, val, mid+1, last)
+    end if
+
+  end subroutine sort_csr
+
+!===============================================================================
+! SPLIT
+!===============================================================================
+
+  subroutine split(col, val, low, high, mid)
+
+    integer :: col(:)
+    integer :: low
+    integer :: high
+    integer :: mid
+    real(8) :: val(:)
+
+    integer :: left
+    integer :: right
+    integer :: iswap
+    integer :: pivot
+    real(8) :: rswap
+    real(8) :: val0
+
+    left = low
+    right = high
+    pivot = col(low)
+    val0 = val(low)
+
+    ! repeat the following while left and right havent met
+    do while (left < right)
+
+      ! scan right to left to find element < pivot
+      do while (left < right .and. col(right) >= pivot)
+        right = right - 1
+      end do
+
+      ! scan left to right to find element > pivot
+      do while (left < right .and. col(left) <= pivot)
+        left = left + 1
+      end do
+
+      ! if left and right havent met, exchange the items
+      if (left < right) then
+        iswap = col(left)
+        col(left) = col(right)
+        col(right) = iswap
+        rswap = val(left)
+        val(left) = val(right)
+        val(right) = rswap
+      end if
+
+    end do
+
+    ! switch the element in split position with pivot
+    col(low) = col(right)
+    col(right) = pivot
+    mid = right
+    val(low) = val(right)
+    val(right) = val0
+
+  end subroutine split
+
 !===============================================================================
 ! MATRIX_SETUP_PETSC
 !===============================================================================
@@ -120,6 +229,56 @@ contains
 
   end subroutine matrix_setup_petsc
 
+!===============================================================================
+! MATRIX_WRITE_PETSC_BINARY
+!===============================================================================
+
+  subroutine matrix_write_petsc_binary(self, filename)
+
+    character(*) :: filename
+    class(Matrix) :: self
+
+    integer :: petsc_err
+    PetscViewer :: viewer
+
+    call PetscViewerBinaryOpen(PETSC_COMM_WORLD, trim(filename), &
+         FILE_MODE_WRITE, viewer, petsc_err)
+    call MatView(self % petsc_mat, viewer, petsc_err)
+    call PetscViewerDestroy(viewer, petsc_err)
+
+  end subroutine matrix_write_petsc_binary
+
+!===============================================================================
+! MATRIX_TRANSPOSE
+!===============================================================================
+
+  subroutine matrix_transpose(self)
+
+    class(Matrix) :: self
+
+    integer :: petsc_err
+
+    call MatTranspose(self % petsc_mat, MAT_REUSE_MATRIX, self % petsc_mat, &
+         petsc_err)
+
+  end subroutine matrix_transpose
+
+!===============================================================================
+! MATRIX_VECTOR_MULTIPLY
+!===============================================================================
+
+  subroutine matrix_vector_multiply(self, vec_in, vec_out)
+
+    class(Matrix) :: self
+    Vec           :: vec_in
+    Vec           :: vec_out
+
+    integer :: petsc_err
+    call self % write_petsc_binary('prod_mat.bin')
+    call MatMult(self % petsc_mat, vec_in, vec_out, petsc_err)
+
+  end subroutine matrix_vector_multiply
+
 # endif
 
 end module matrix_header
diff --git a/src/petsc_solver.F90 b/src/petsc_solver.F90
index c9136e4c70..8f42f53202 100644
--- a/src/petsc_solver.F90
+++ b/src/petsc_solver.F90
@@ -3,23 +3,20 @@ module petsc_solver
   implicit none
   private
 
-# ifdef PETSC
 #  include <finclude/petsc.h90>
-# endif
 
-  type, public :: gmres
-#  ifdef PETSC
+  type, public :: Petsc_gmres 
     KSP :: ksp
     PC  :: pc
-#  endif
    contains
      procedure :: create       => gmres_create
      procedure :: precondition => gmres_precondition
      procedure :: set_oper     => gmres_set_oper
      procedure :: destroy      => gmres_destroy
-  end type gmres
+     procedure :: solve        => gmres_solve
+  end type Petsc_gmres
 
-  integer :: petsc_err
+  integer :: Petsc_err
 
 contains
 
@@ -29,13 +26,12 @@ contains
 
   subroutine gmres_create(self)
 
-    class(gmres) :: self
+    class(Petsc_gmres) :: self
 
     integer :: ilu_levels = 5
     real(8) :: rtol = 1.0e-10_8
     real(8) :: atol = 1.0e-10_8
 
-#  ifdef PETSC
     call KSPCreate(PETSC_COMM_WORLD, self % ksp, petsc_err)
     call KSPSetTolerances(self % ksp, rtol, atol, &
          PETSC_DEFAULT_DOUBLE_PRECISION, PETSC_DEFAULT_INTEGER, petsc_err)
@@ -43,8 +39,6 @@ contains
     call KSPSetInitialGuessNonzero(self % ksp, PETSC_TRUE, petsc_err)
     call KSPGetPC(self % ksp, self % pc, petsc_err)
     call PCFactorSetLevels(self % pc, ilu_levels, petsc_err)
-    call KSPSetUp(self % ksp, petsc_err) 
-#  endif
 
   end subroutine gmres_create
 
@@ -54,12 +48,11 @@ contains
 
   subroutine gmres_precondition(self, matrix)
 
-    class(gmres) :: self
+    class(Petsc_gmres) :: self
     Mat          :: matrix
 
-#  ifdef PETSC
+    call KSPSetUp(self % ksp, petsc_err) 
     call PCFactorGetMatrix(self % pc, matrix, petsc_err) 
-#  endif
 
   end subroutine gmres_precondition
 
@@ -69,14 +62,12 @@ contains
 
   subroutine gmres_set_oper(self, prec_matrix, matrix)
 
-    class(gmres) :: self
-    Mat          :: prec_matrix
-    Mat          :: matrix
+    class(Petsc_gmres) :: self
+    Mat                :: prec_matrix
+    Mat                :: matrix
 
-#  ifdef PETSC
     call KSPSetOperators(self % ksp, matrix, prec_matrix, &
          SAME_NONZERO_PATTERN, petsc_err)
-#  endif
 
   end subroutine gmres_set_oper
 
@@ -86,12 +77,26 @@ contains
 
   subroutine gmres_destroy(self)
 
-   class(gmres) :: self
+   class(Petsc_gmres) :: self
 
-#  ifdef PETSC
     call KSPDestroy(self % ksp, petsc_err)
-#  endif 
 
   end subroutine gmres_destroy
 
-end module petsc_solver 
+!===============================================================================
+! GMRES_SOLVE
+!===============================================================================
+
+  subroutine gmres_solve(self, b, x)
+
+    class(Petsc_gmres) :: self
+    Vec :: b
+    Vec :: x
+
+    integer :: petsc_err
+
+    call KSPSolve(self % ksp, b, x, petsc_err)
+
+  end subroutine gmres_solve
+
+end module petsc_solver
diff --git a/src/vector_header.F90 b/src/vector_header.F90
index 7f2abd7113..53d3bb6fe5 100644
--- a/src/vector_header.F90
+++ b/src/vector_header.F90
@@ -21,6 +21,7 @@ module vector_header
      procedure :: add_value    => vector_add_value
 #   ifdef PETSC
      procedure :: setup_petsc  => vector_setup_petsc
+     procedure :: write_petsc_binary => vector_write_petsc_binary
 #   endif
   end type Vector
 
@@ -90,6 +91,24 @@ contains
 
   end subroutine vector_setup_petsc
 
+!===============================================================================
+! VECTOR_WRITE_PETSC_BINARY
+!===============================================================================
+
+  subroutine vector_write_petsc_binary(self, filename)
+
+    character(*) :: filename
+    class(Vector) :: self
+
+    integer :: petsc_err
+    PetscViewer :: viewer
+
+    call PetscViewerBinaryOpen(PETSC_COMM_WORLD, trim(filename), &
+         FILE_MODE_WRITE, viewer, petsc_err)
+    call VecView(self % petsc_vec, viewer, petsc_err)
+    call PetscViewerDestroy(viewer, petsc_err)
+
+  end subroutine vector_write_petsc_binary
 # endif
 
 end module vector_header

From cb203502e5a48ef3174c24f59688b57f1b8370f0 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 9 Jul 2013 09:10:00 -0400
Subject: [PATCH 08/69] removed cmfd message passing and re-made dependencies

---
 src/DEPENDENCIES | 9 ---------
 src/OBJECTS      | 1 -
 2 files changed, 10 deletions(-)

diff --git a/src/DEPENDENCIES b/src/DEPENDENCIES
index f74355babb..1fdb68cfcc 100644
--- a/src/DEPENDENCIES
+++ b/src/DEPENDENCIES
@@ -43,9 +43,6 @@ cmfd_power_solver.o: matrix_header.o
 cmfd_power_solver.o: petsc_solver.o
 cmfd_power_solver.o: vector_header.o
 
-cmfd_message_passing.o: cmfd_header.o
-cmfd_message_passing.o: global.o
-
 cmfd_data.o: cmfd_header.o
 cmfd_data.o: constants.o
 cmfd_data.o: error.o
@@ -106,15 +103,10 @@ fission.o: search.o
 cmfd_jacobian_operator.o: constants.o
 cmfd_jacobian_operator.o: global.o
 
-cmfd_snes_solver.o: cmfd_jacobian_operator.o
-cmfd_snes_solver.o: cmfd_loss_operator.o
-cmfd_snes_solver.o: cmfd_power_solver.o
-cmfd_snes_solver.o: cmfd_prod_operator.o
 cmfd_snes_solver.o: constants.o
 cmfd_snes_solver.o: global.o
 cmfd_snes_solver.o: string.o
 
-cmfd_input.o: cmfd_message_passing.o
 cmfd_input.o: error.o
 cmfd_input.o: global.o
 cmfd_input.o: mesh_header.o
@@ -319,7 +311,6 @@ plot_header.o: constants.o
 cmfd_header.o: constants.o
 
 cmfd_execute.o: cmfd_data.o
-cmfd_execute.o: cmfd_message_passing.o
 cmfd_execute.o: cmfd_power_solver.o
 cmfd_execute.o: cmfd_snes_solver.o
 cmfd_execute.o: constants.o
diff --git a/src/OBJECTS b/src/OBJECTS
index af668b4b5b..fe309726d7 100644
--- a/src/OBJECTS
+++ b/src/OBJECTS
@@ -8,7 +8,6 @@ cmfd_header.o \
 cmfd_input.o \
 cmfd_jacobian_operator.o \
 cmfd_loss_operator.o \
-cmfd_message_passing.o \
 cmfd_output.o \
 cmfd_power_solver.o \
 cmfd_prod_operator.o \

From f6e33c5db70397f233a920b45698ca3a4b2d1892 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 9 Jul 2013 09:10:38 -0400
Subject: [PATCH 09/69] got rid of 2 group fixed data

---
 src/cmfd_data.F90 | 37 ++-----------------------------------
 1 file changed, 2 insertions(+), 35 deletions(-)

diff --git a/src/cmfd_data.F90 b/src/cmfd_data.F90
index 9943379784..8d44b3d0d7 100644
--- a/src/cmfd_data.F90
+++ b/src/cmfd_data.F90
@@ -22,7 +22,7 @@ contains
 
     use cmfd_header,         only: allocate_cmfd
     use constants,           only: CMFD_NOACCEL
-    use global,              only: cmfd, cmfd_coremap, cmfd_run_2grp
+    use global,              only: cmfd, cmfd_coremap
 
     ! initialize cmfd object
     if (.not.allocated(cmfd%flux)) call allocate_cmfd(cmfd)
@@ -31,14 +31,11 @@ contains
     if ((cmfd_coremap) .and. (cmfd%mat_dim == CMFD_NOACCEL)) call set_coremap()
 
     ! calculate all cross sections based on reaction rates from last batch
-    if (.not. cmfd_run_2grp) call compute_xs()
+    call compute_xs()
 
     ! check neutron balance
 !   call neutron_balance(670)
 
-    ! fix 2 grp cross sections
-    if (cmfd_run_2grp) call fix_2_grp()
-
     ! calculate dtilde
     call compute_dtilde()
 
@@ -807,36 +804,6 @@ contains
 
   end function get_reflector_albedo
 
-!===============================================================================
-! FIX_2_GRP modifies xs for benchmark with stand alone (homogeneous)
-!===============================================================================
-
-  subroutine fix_2_grp()
-
-    use global,  only: cmfd
-
-    ! overwrite cross sections
-    cmfd % totalxs(1,:,:,:) = 0.02597_8
-    cmfd % totalxs(2,:,:,:) = 0.06669_8
-    cmfd % scattxs(1,1,:,:,:) = 0.0_8
-    cmfd % scattxs(1,2,:,:,:) = 0.01742_8
-    cmfd % scattxs(2,1,:,:,:) = 0.0_8
-    cmfd % scattxs(2,2,:,:,:) = 0.0_8
-    cmfd % nfissxs(1,1,:,:,:) = 0.00536_8
-    cmfd % nfissxs(1,2,:,:,:) = 0.0_8
-    cmfd % nfissxs(2,1,:,:,:) = 0.10433_8
-    cmfd % nfissxs(2,2,:,:,:) = 0.0_8
-    cmfd % diffcof(1,:,:,:) = 1.4176_8
-    cmfd % diffcof(2,:,:,:) = 0.37336_8
-    cmfd % hxyz(1,:,:,:) = 0.5_8
-    cmfd % hxyz(2,:,:,:) = 0.5_8
-    cmfd % hxyz(3,:,:,:) = 0.5_8
-
-    ! set dhat reset to true
-    dhat_reset = .true.
-
-  end subroutine fix_2_grp
-
 !===============================================================================
 ! FIX_NEUTRON_BALANCE
 !===============================================================================

From 51c78ada0feb7f8be7e92e5b64bc36e277ac2e5a Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 9 Jul 2013 09:31:17 -0400
Subject: [PATCH 10/69] changed comments and updated some parts of code

---
 src/DEPENDENCIES  |   1 +
 src/cmfd_data.F90 | 444 +++++++++++++++++++++++-----------------------
 2 files changed, 221 insertions(+), 224 deletions(-)

diff --git a/src/DEPENDENCIES b/src/DEPENDENCIES
index 1fdb68cfcc..433ed14828 100644
--- a/src/DEPENDENCIES
+++ b/src/DEPENDENCIES
@@ -49,6 +49,7 @@ cmfd_data.o: error.o
 cmfd_data.o: global.o
 cmfd_data.o: mesh.o
 cmfd_data.o: mesh_header.o
+cmfd_data.o: string.o
 cmfd_data.o: tally_header.o
 
 plot.o: constants.o
diff --git a/src/cmfd_data.F90 b/src/cmfd_data.F90
index 8d44b3d0d7..4e81c2aaf4 100644
--- a/src/cmfd_data.F90
+++ b/src/cmfd_data.F90
@@ -15,31 +15,34 @@ module cmfd_data
 contains
 
 !==============================================================================
-! SET_UP_CMFD
+! SET_UP_CMFD configures cmfd object for a CMFD eigenvalue calculation
 !==============================================================================
 
   subroutine set_up_cmfd() 
 
     use cmfd_header,         only: allocate_cmfd
     use constants,           only: CMFD_NOACCEL
-    use global,              only: cmfd, cmfd_coremap
+    use global,              only: cmfd, cmfd_coremap, cmfd_downscatter
 
-    ! initialize cmfd object
+    ! Initialize cmfd object
     if (.not.allocated(cmfd%flux)) call allocate_cmfd(cmfd)
 
-    ! check for core map and set it up
+    ! Check for core map and set it up
     if ((cmfd_coremap) .and. (cmfd%mat_dim == CMFD_NOACCEL)) call set_coremap()
 
-    ! calculate all cross sections based on reaction rates from last batch
+    ! Calculate all cross sections based on reaction rates from last batch
     call compute_xs()
 
-    ! check neutron balance
+    ! Compute effective downscatter cross section
+    if (cmfd_downscatter) call compute_effective_downscatter()
+
+    ! Check neutron balance
 !   call neutron_balance(670)
 
-    ! calculate dtilde
+    ! Calculate dtilde
     call compute_dtilde()
 
-    ! calucate dhat
+    ! Calculate dhat
     call compute_dhat()
 
   end subroutine set_up_cmfd 
@@ -52,11 +55,13 @@ contains
 
     use constants,    only: FILTER_MESH, FILTER_ENERGYIN, FILTER_ENERGYOUT,     &
                             FILTER_SURFACE, IN_RIGHT, OUT_RIGHT, IN_FRONT,      &
-                            OUT_FRONT, IN_TOP, OUT_TOP, CMFD_NOACCEL, ZERO, ONE
+                            OUT_FRONT, IN_TOP, OUT_TOP, CMFD_NOACCEL, ZERO,     &
+                            ONE, TINY_BIT
     use error,        only: fatal_error
     use global,       only: cmfd, message, n_cmfd_tallies, cmfd_tallies, meshes
     use mesh,         only: mesh_indices_to_bin
     use mesh_header,  only: StructuredMesh
+    use string,       only: to_str
     use tally_header, only: TallyObject
 
     integer :: nx            ! number of mesh cells in x direction
@@ -80,30 +85,30 @@ contains
     type(TallyObject),    pointer :: t => null() ! pointer for tally object
     type(StructuredMesh), pointer :: m => null() ! pointer for mesh object
 
-    ! extract spatial and energy indices from object
+    ! Extract spatial and energy indices from object
     nx = cmfd % indices(1)
     ny = cmfd % indices(2)
     nz = cmfd % indices(3)
     ng = cmfd % indices(4)
 
-    ! set flux object and source distribution to all zeros
+    ! Set flux object and source distribution to all zeros
     cmfd % flux = ZERO
     cmfd % openmc_src = ZERO
 
-    ! associate tallies and mesh
+    ! Associate tallies and mesh
     t => cmfd_tallies(1)
     i_mesh = t % filters(t % find_filter(FILTER_MESH)) % int_bins(1)
     m => meshes(i_mesh)
 
-    ! set mesh widths
+    ! Set mesh widths
     cmfd % hxyz(1,:,:,:) = m % width(1) ! set x width
     cmfd % hxyz(2,:,:,:) = m % width(2) ! set y width
     cmfd % hxyz(3,:,:,:) = m % width(3) ! set z width
 
-   ! begin loop around tallies
+   ! Begin loop around tallies
    TAL: do ital = 1, n_cmfd_tallies
 
-     ! associate tallies and mesh
+     ! Associate tallies and mesh
      t => cmfd_tallies(ital)
      i_mesh = t % filters(t % find_filter(FILTER_MESH)) % int_bins(1)
      m => meshes(i_mesh)
@@ -113,98 +118,99 @@ contains
      i_filter_eout = t % find_filter(FILTER_ENERGYOUT)
      i_filter_surf = t % find_filter(FILTER_SURFACE)
 
-     ! begin loop around space
+     ! Begin loop around space
      ZLOOP: do k = 1,nz
 
        YLOOP: do j = 1,ny
 
           XLOOP: do i = 1,nx
  
-            ! check for active mesh cell
+            ! Check for active mesh cell
             if (allocated(cmfd%coremap)) then
               if (cmfd%coremap(i,j,k) == CMFD_NOACCEL) then
                 cycle
               end if
             end if
 
-            ! loop around energy groups
+            ! Loop around energy groups
             OUTGROUP: do h = 1,ng
 
-              ! start tally 1
+              ! Start tally 1
               TALLY: if (ital == 1) then
 
-                ! reset all bins to 1
+                ! Reset all bins to 1
                 t % matching_bins = 1
 
-                ! set ijk as mesh indices
+                ! Set ijk as mesh indices
                 ijk = (/ i, j, k /)
 
-                ! get bin number for mesh indices
+                ! Get bin number for mesh indices
                 t % matching_bins(i_filter_mesh) = mesh_indices_to_bin(m,ijk)
 
-                ! apply energy in filter
+                ! Apply energy in filter
                 if (i_filter_ein > 0) then
                   t % matching_bins(i_filter_ein) = ng - h + 1
                 end if
 
-                ! calculate score index from bins
+                ! Calculate score index from bins
                 score_index = sum((t % matching_bins - 1) * t%stride) + 1
 
-                ! get flux
+                ! Get flux
                 flux = t % results(1,score_index) % sum
                 cmfd % flux(h,i,j,k) = flux
 
-                ! detect zero flux
-                if ((flux - 0.0D0) < 1.0E-10_8) then
-                  print *,h,i,j,k,flux
-                  message = 'Detected zero flux without coremap overlay'
+                ! Detect zero flux, abort if located
+                if ((flux - ZERO) < TINY_BIT) then
+                  message = 'Detected zero flux without coremap overlay at: (' &
+                          // to_str(i) // ',' // to_str(j) // ',' // to_str(k) &
+                          // ') in group ' // to_str(h)
                   call fatal_error()
                 end if
 
-                ! get total rr and convert to total xs
+                ! Get total rr and convert to total xs
                 cmfd % totalxs(h,i,j,k) = t % results(2,score_index) % sum / flux
 
-                ! get p1 scatter rr and convert to p1 scatter xs
+                ! Get p1 scatter rr and convert to p1 scatter xs
                 cmfd % p1scattxs(h,i,j,k) = t % results(3,score_index) % sum / flux
 
-                ! calculate diffusion coefficient
+                ! Calculate diffusion coefficient
                 cmfd % diffcof(h,i,j,k) = ONE/(3.0_8*(cmfd % totalxs(h,i,j,k) - &
                      cmfd % p1scattxs(h,i,j,k)))
 
               else if (ital == 2) then
 
-                ! begin loop to get energy out tallies
+                ! Begin loop to get energy out tallies
                 INGROUP: do g = 1, ng
 
-                  ! reset all bins to 1
+                  ! Reset all bins to 1
                   t % matching_bins = 1
 
-                  ! set ijk as mesh indices
+                  ! Set ijk as mesh indices
                   ijk = (/ i, j, k /)
 
-                  ! get bin number for mesh indices
+                  ! Get bin number for mesh indices
                   t % matching_bins(i_filter_mesh) = mesh_indices_to_bin(m,ijk)
 
                   if (i_filter_ein > 0) then
-                    ! apply energy in filter
+                    ! Apply energy in filter
                     t % matching_bins(i_filter_ein) = ng - h + 1
 
-                    ! set energy out bin
+                    ! Set energy out bin
                     t % matching_bins(i_filter_eout) = ng - g + 1
                   end if
 
-                  ! calculate score index from bins
+                  ! Calculate score index from bins
                   score_index = sum((t % matching_bins - 1) * t%stride) + 1
 
-                  ! get scattering
+                  ! Get scattering
                   cmfd % scattxs(h,g,i,j,k) = t % results(1,score_index) % sum /&
                        cmfd % flux(h,i,j,k)
 
-                  ! get nu-fission
+                  ! Get nu-fission
                   cmfd % nfissxs(h,g,i,j,k) = t % results(2,score_index) % sum /&
                        cmfd % flux(h,i,j,k)
 
-                  ! bank source
+                  ! Bank source
                   cmfd % openmc_src(g,i,j,k) = cmfd % openmc_src(g,i,j,k) + &
                        t % results(2,score_index) % sum
 
@@ -212,13 +218,13 @@ contains
 
               else if (ital == 3) then
 
-                ! initialize and filter for energy
+                ! Initialize and filter for energy
                 t % matching_bins = 1
                 if (i_filter_ein > 0) then
                   t % matching_bins(i_filter_ein) = ng - h + 1
                 end if
 
-                ! left surface
+                ! Left surface
                 t % matching_bins(i_filter_mesh) = mesh_indices_to_bin(m, &
                      (/ i-1, j, k /) + 1, .true.)
                 t % matching_bins(i_filter_surf) = IN_RIGHT
@@ -228,7 +234,7 @@ contains
                 score_index = sum((t % matching_bins - 1) * t % stride) + 1 ! incoming
                 cmfd % current(2,h,i,j,k) = t % results(1,score_index) % sum
 
-                ! right surface
+                ! Right surface
                 t % matching_bins(i_filter_mesh) = mesh_indices_to_bin(m, &
                      (/ i, j, k /) + 1, .true.)
                 t % matching_bins(i_filter_surf) = IN_RIGHT
@@ -238,7 +244,7 @@ contains
                 score_index = sum((t % matching_bins - 1) * t % stride) + 1 ! outgoing
                 cmfd % current(4,h,i,j,k) = t % results(1,score_index) % sum
 
-                ! back surface
+                ! Back surface
                 t % matching_bins(i_filter_mesh) = mesh_indices_to_bin(m, &
                      (/ i, j-1, k /) + 1, .true.)
                 t % matching_bins(i_filter_surf) = IN_FRONT
@@ -248,7 +254,7 @@ contains
                 score_index = sum((t % matching_bins - 1) * t % stride) + 1 ! incoming
                 cmfd % current(6,h,i,j,k) = t % results(1,score_index) % sum
 
-                ! front surface
+                ! Front surface
                 t % matching_bins(i_filter_mesh) = mesh_indices_to_bin(m, &
                      (/ i, j, k /) + 1, .true.)
                 t % matching_bins(i_filter_surf) = IN_FRONT
@@ -258,7 +264,7 @@ contains
                 score_index = sum((t % matching_bins - 1) * t % stride) + 1 ! outgoing
                 cmfd % current(8,h,i,j,k) = t % results(1,score_index) % sum
 
-                ! bottom surface
+                ! Bottom surface
                 t % matching_bins(i_filter_mesh) = mesh_indices_to_bin(m, &
                      (/ i, j, k-1 /) + 1, .true.)
                 t % matching_bins(i_filter_surf) = IN_TOP
@@ -268,7 +274,7 @@ contains
                 score_index = sum((t % matching_bins - 1) * t % stride) + 1 ! incoming
                 cmfd % current(10,h,i,j,k) = t % results(1,score_index) % sum
 
-                ! top surface
+                ! Top surface
                 t % matching_bins(i_filter_mesh) = mesh_indices_to_bin(m, &
                      (/ i, j, k /) + 1, .true.)
                 t % matching_bins(i_filter_surf) = IN_TOP
@@ -290,10 +296,10 @@ contains
 
     end do TAL
 
-    ! normalize openmc source distribution
+    ! Normalize openmc source distribution
     cmfd % openmc_src = cmfd % openmc_src/sum(cmfd % openmc_src)*cmfd%norm
 
-    ! nullify all pointers
+    ! Nullify all pointers
     if (associated(t)) nullify(t)
     if (associated(m)) nullify(m)
 
@@ -308,42 +314,42 @@ contains
     use constants,  only: CMFD_NOACCEL
     use global,     only: cmfd
 
-    integer :: kount=1           ! counter for unique fuel assemblies
-    integer :: nx                ! number of mesh cells in x direction
-    integer :: ny                ! number of mesh cells in y direction
-    integer :: nz                ! number of mesh cells in z direction
-    integer :: i                 ! iteration counter for x
-    integer :: j                 ! iteration counter for y
-    integer :: k                 ! iteration counter for z
+    integer :: kount=1 ! counter for unique fuel assemblies
+    integer :: nx      ! number of mesh cells in x direction
+    integer :: ny      ! number of mesh cells in y direction
+    integer :: nz      ! number of mesh cells in z direction
+    integer :: i       ! iteration counter for x
+    integer :: j       ! iteration counter for y
+    integer :: k       ! iteration counter for z
 
-    ! extract spatial indices from object
+    ! Extract spatial indices from object
     nx = cmfd % indices(1)
     ny = cmfd % indices(2)
     nz = cmfd % indices(3)
 
-    ! count how many fuel assemblies exist
+    ! Count how many fuel assemblies exist
     cmfd % mat_dim = sum(cmfd % coremap - 1)
 
-    ! allocate indexmap
+    ! Allocate indexmap
     if (.not. allocated(cmfd % indexmap)) &
          allocate(cmfd % indexmap(cmfd % mat_dim,3))
 
-    ! begin loops over spatial indices
+    ! Begin loops over spatial indices
     ZLOOP: do k = 1, nz
 
       YLOOP: do j = 1, ny
 
         XLOOP: do i = 1, nx
 
-          ! check for reflector
+          ! Check for reflector
           if (cmfd % coremap(i,j,k) == 1) then
 
-            ! reset value to 99999
+            ! reset value to CMFD no acceleration constant 
             cmfd % coremap(i,j,k) = CMFD_NOACCEL 
 
           else
 
-            ! must be a fuel --> give unique id number
+            ! Must be a fuel --> give unique id number
             cmfd % coremap(i,j,k) = kount
             cmfd % indexmap(kount,1) = i
             cmfd % indexmap(kount,2) = j
@@ -386,19 +392,19 @@ contains
     real(8) :: fission      ! fission term in neutron balance
     real(8) :: res          ! residual of neutron balance
 
-    ! check if keff is close to 0 (happens on first active batch)
+    ! Check if keff is close to 0 (happens on first active batch)
     if (keff < 1e-8_8) return
 
-    ! extract spatial and energy indices from object
+    ! Extract spatial and energy indices from object
     nx = cmfd % indices(1)
     ny = cmfd % indices(2)
     nz = cmfd % indices(3)
     ng = cmfd % indices(4)
 
-    ! allocate res dataspace
+    ! Allocate res dataspace
     if (.not. allocated(cmfd%resnb)) allocate(cmfd%resnb(ng,nx,ny,nz))
 
-    ! begin loop around space and energy groups
+    ! Begin loop around space and energy groups
     ZLOOP: do k = 1, nz
 
       YLOOP: do j = 1, ny
@@ -407,7 +413,7 @@ contains
 
           GROUPG: do g = 1, ng
 
-            ! check for active mesh
+            ! Check for active mesh
             if (allocated(cmfd%coremap)) then
               if (cmfd%coremap(i,j,k) == CMFD_NOACCEL) then
                 cmfd%resnb(g,i,j,k) = CMFD_NORES
@@ -415,7 +421,7 @@ contains
               end if
             end if
 
-            ! get leakage
+            ! Get leakage
             leakage = ZERO
             LEAK: do l = 1, 3
 
@@ -426,10 +432,10 @@ contains
 
             end do LEAK
 
-            ! interactions
+            ! Interactions
             interactions = cmfd % totalxs(g,i,j,k) * cmfd % flux(g,i,j,k)
 
-            ! get scattering and fission
+            ! Get scattering and fission
             scattering = ZERO
             fission = ZERO
             GROUPH: do h = 1, ng
@@ -442,16 +448,16 @@ contains
 
             end do GROUPH
 
-            ! compute residual
+            ! Compute residual
             res = leakage + interactions - scattering - (ONE/keff)*fission
 
-            ! normalize by flux
+            ! Normalize by flux
             res = res/cmfd%flux(g,i,j,k)
 
-            ! bank res in cmfd object
+            ! Bank res in cmfd object
             cmfd%resnb(g,i,j,k) = res
 
-            ! write out info to file
+            ! Write out info to file
             write(uid,'(A,1X,I0,1X,I0,1X,I0,1X,A,1X,I0,1X,A,1PE11.4)') &
                           'Location',i,j,k,' Group:',g,'Balance:',res
             write(uid,100) 'Leakage:',leakage
@@ -475,7 +481,7 @@ contains
   end subroutine neutron_balance
 
 !===============================================================================
-! COMPUTE_DTILDE computes the diffusion coupling coefficient
+! COMPUTE_DTILDE precomputes the diffusion coupling coefficient
 !===============================================================================
 
   subroutine compute_dtilde()
@@ -483,44 +489,44 @@ contains
     use constants,  only: CMFD_NOACCEL, ZERO_FLUX, TINY_BIT
     use global,     only: cmfd, cmfd_coremap
 
-    integer :: nx                 ! maximum number of cells in x direction
-    integer :: ny                 ! maximum number of cells in y direction
-    integer :: nz                 ! maximum number of cells in z direction
-    integer :: ng                 ! maximum number of energy groups
-    integer :: nxyz(3,2)          ! single vector containing boundary locations
-    integer :: i                  ! iteration counter for x
-    integer :: j                  ! iteration counter for y
-    integer :: k                  ! iteration counter for z
-    integer :: g                  ! iteration counter for groups
-    integer :: l                  ! iteration counter for leakages
-    integer :: xyz_idx            ! index for determining if x,y or z leakage
-    integer :: dir_idx            ! index for determining - or + face of cell
-    integer :: shift_idx          ! parameter to shift index by +1 or -1
-    integer :: neig_idx(3)        ! spatial indices of neighbour
-    integer :: bound(6)           ! vector containing indices for boudary check
-    real(8) :: albedo(6)          ! albedo vector with global boundaries
-    real(8) :: cell_dc            ! diffusion coef of current cell
-    real(8) :: cell_hxyz(3)       ! cell dimensions of current ijk cell
-    real(8) :: neig_dc            ! diffusion coefficient of neighbor cell
-    real(8) :: neig_hxyz(3)       ! cell dimensions of neighbor cell
-    real(8) :: dtilde             ! finite difference coupling parameter 
-    real(8) :: ref_albedo         ! albedo to reflector
+    integer :: nx           ! maximum number of cells in x direction
+    integer :: ny           ! maximum number of cells in y direction
+    integer :: nz           ! maximum number of cells in z direction
+    integer :: ng           ! maximum number of energy groups
+    integer :: nxyz(3,2)    ! single vector containing boundary locations
+    integer :: i            ! iteration counter for x
+    integer :: j            ! iteration counter for y
+    integer :: k            ! iteration counter for z
+    integer :: g            ! iteration counter for groups
+    integer :: l            ! iteration counter for leakages
+    integer :: xyz_idx      ! index for determining if x,y or z leakage
+    integer :: dir_idx      ! index for determining - or + face of cell
+    integer :: shift_idx    ! parameter to shift index by +1 or -1
+    integer :: neig_idx(3)  ! spatial indices of neighbour
+    integer :: bound(6)     ! vector containing indices for boudary check
+    real(8) :: albedo(6)    ! albedo vector with global boundaries
+    real(8) :: cell_dc      ! diffusion coef of current cell
+    real(8) :: cell_hxyz(3) ! cell dimensions of current ijk cell
+    real(8) :: neig_dc      ! diffusion coefficient of neighbor cell
+    real(8) :: neig_hxyz(3) ! cell dimensions of neighbor cell
+    real(8) :: dtilde       ! finite difference coupling parameter 
+    real(8) :: ref_albedo   ! albedo to reflector
 
-    ! get maximum of spatial and group indices
+    ! Get maximum of spatial and group indices
     nx = cmfd%indices(1)
     ny = cmfd%indices(2)
     nz = cmfd%indices(3)
     ng = cmfd%indices(4)
 
-    ! create single vector of these indices for boundary calculation
+    ! Create single vector of these indices for boundary calculation
     nxyz(1,:) = (/1,nx/)
     nxyz(2,:) = (/1,ny/)
     nxyz(3,:) = (/1,nz/)
 
-    ! get boundary condition information
+    ! Get boundary condition information
     albedo = cmfd%albedo
 
-    ! geting loop over group and spatial indices
+    ! Loop over group and spatial indices
     ZLOOP: do k = 1, nz
 
       YLOOP: do j = 1, ny
@@ -529,63 +535,63 @@ contains
 
           GROUP: do g = 1, ng
 
-            ! check for active mesh cell
+            ! Check for active mesh cell
             if (allocated(cmfd%coremap)) then
               if (cmfd%coremap(i,j,k) == CMFD_NOACCEL) cycle
             end if
 
-            ! get cell data
+            ! Get cell data
             cell_dc = cmfd%diffcof(g,i,j,k)
             cell_hxyz = cmfd%hxyz(:,i,j,k)
 
-            ! setup of vector to identify boundary conditions
+            ! Setup of vector to identify boundary conditions
             bound = (/i,i,j,j,k,k/)
 
-            ! begin loop around sides of cell for leakage
+            ! Begin loop around sides of cell for leakage
             LEAK: do l = 1, 6
 
-              ! define xyz and +/- indices
+              ! Define xyz and +/- indices
               xyz_idx = int(ceiling(real(l)/real(2)))  ! x=1, y=2, z=3
               dir_idx = 2 - mod(l,2) ! -=1, +=2
-              shift_idx = -2*mod(l,2) + 1          ! shift neig by -1 or +1
+              shift_idx = -2*mod(l,2) + 1 ! shift neig by -1 or +1
 
-              ! check if at a boundary
+              ! Check if at a boundary
               if (bound(l) == nxyz(xyz_idx,dir_idx)) then
 
-                ! compute dtilde
+                ! Compute dtilde with albedo boundary condition
                 dtilde = (2*cell_dc*(1-albedo(l)))/(4*cell_dc*(1+albedo(l)) + &
                      (1-albedo(l))*cell_hxyz(xyz_idx))
 
-                ! check for zero flux
+                ! Check for zero flux
                 if (abs(albedo(l) - ZERO_FLUX) < TINY_BIT) dtilde = 2*cell_dc / &
                      cell_hxyz(xyz_idx)
 
               else  ! not a boundary
 
-                ! compute neighboring cell indices
+                ! Compute neighboring cell indices
                 neig_idx = (/i,j,k/)                ! begin with i,j,k
                 neig_idx(xyz_idx) = shift_idx + neig_idx(xyz_idx)
 
-                ! get neigbor cell data
+                ! Get neigbor cell data
                 neig_dc = cmfd%diffcof(g,neig_idx(1),neig_idx(2),neig_idx(3))
                 neig_hxyz = cmfd%hxyz(:,neig_idx(1),neig_idx(2),neig_idx(3))
 
-                ! check for fuel-reflector interface
+                ! Check for fuel-reflector interface
                 if (cmfd_coremap) then
 
                   if (cmfd % coremap(neig_idx(1),neig_idx(2),neig_idx(3)) == &
                        CMFD_NOACCEL .and. cmfd % coremap(i,j,k) /= CMFD_NOACCEL) then
 
-                    ! get albedo
+                    ! Get albedo
                     ref_albedo = get_reflector_albedo(l,g,i,j,k)
 
-                    ! compute dtilde
+                    ! Compute dtilde
                     dtilde = (2*cell_dc*(1-ref_albedo))/(4*cell_dc*(1+ &
                          ref_albedo)+(1-ref_albedo)*cell_hxyz(xyz_idx))
 
-                  else ! not next to a reflector or no core map
+                  else ! Not next to a reflector or no core map
 
-                    ! compute dtilde
+                    ! Compute dtilde to neighbor cell
                     dtilde = (2*cell_dc*neig_dc)/(neig_hxyz(xyz_idx)*cell_dc + &
                          cell_hxyz(xyz_idx)*neig_dc)
 
@@ -593,7 +599,7 @@ contains
 
                 else ! no core map
 
-                  ! compute dtilde
+                  ! Compute dtilde to neighbor cell
                   dtilde = (2*cell_dc*neig_dc)/(neig_hxyz(xyz_idx)*cell_dc + &
                        cell_hxyz(xyz_idx)*neig_dc)
 
@@ -601,7 +607,7 @@ contains
 
               end if
 
-              ! record dtilde in cmfd object
+              ! Record dtilde in cmfd object
               cmfd%dtilde(l,g,i,j,k) = dtilde
 
             end do LEAK
@@ -625,40 +631,40 @@ contains
     use constants,  only: CMFD_NOACCEL, ZERO
     use global,     only: cmfd, cmfd_coremap
 
-    integer :: nx                 ! maximum number of cells in x direction
-    integer :: ny                 ! maximum number of cells in y direction
-    integer :: nz                 ! maximum number of cells in z direction
-    integer :: ng                 ! maximum number of energy groups
-    integer :: nxyz(3,2)          ! single vector containing boundary locations
-    integer :: i                  ! iteration counter for x
-    integer :: j                  ! iteration counter for y
-    integer :: k                  ! iteration counter for z
-    integer :: g                  ! iteration counter for groups
-    integer :: l                  ! iteration counter for leakages
-    integer :: xyz_idx            ! index for determining if x,y or z leakage
-    integer :: dir_idx            ! index for determining - or + face of cell
-    integer :: shift_idx          ! parameter to shift index by +1 or -1
-    integer :: neig_idx(3)        ! spatial indices of neighbour
-    integer :: bound(6)           ! vector containing indices for boudary check
-    real(8) :: cell_dtilde(6)     ! cell dtilde for each face
-    real(8) :: cell_flux          ! flux in current cell
-    real(8) :: current(12)        ! area integrated cell current at each face
-    real(8) :: net_current        ! net current on a face
-    real(8) :: neig_flux          ! flux in neighbor cell
-    real(8) :: dhat               ! dhat equivalence parameter
+    integer :: nx             ! maximum number of cells in x direction
+    integer :: ny             ! maximum number of cells in y direction
+    integer :: nz             ! maximum number of cells in z direction
+    integer :: ng             ! maximum number of energy groups
+    integer :: nxyz(3,2)      ! single vector containing boundary locations
+    integer :: i              ! iteration counter for x
+    integer :: j              ! iteration counter for y
+    integer :: k              ! iteration counter for z
+    integer :: g              ! iteration counter for groups
+    integer :: l              ! iteration counter for leakages
+    integer :: xyz_idx        ! index for determining if x,y or z leakage
+    integer :: dir_idx        ! index for determining - or + face of cell
+    integer :: shift_idx      ! parameter to shift index by +1 or -1
+    integer :: neig_idx(3)    ! spatial indices of neighbour
+    integer :: bound(6)       ! vector containing indices for boudary check
+    real(8) :: cell_dtilde(6) ! cell dtilde for each face
+    real(8) :: cell_flux      ! flux in current cell
+    real(8) :: current(12)    ! area integrated cell current at each face
+    real(8) :: net_current    ! net current on a face
+    real(8) :: neig_flux      ! flux in neighbor cell
+    real(8) :: dhat           ! dhat equivalence parameter
 
-    ! get maximum of spatial and group indices
+    ! Get maximum of spatial and group indices
     nx = cmfd%indices(1)
     ny = cmfd%indices(2)
     nz = cmfd%indices(3)
     ng = cmfd%indices(4)
 
-    ! create single vector of these indices for boundary calculation
+    ! Create single vector of these indices for boundary calculation
     nxyz(1,:) = (/1,nx/)
     nxyz(2,:) = (/1,ny/)
     nxyz(3,:) = (/1,nz/)
 
-    ! geting loop over group and spatial indices
+    ! Geting loop over group and spatial indices
     ZLOOP:  do k = 1,nz
 
       YLOOP: do j = 1,ny
@@ -667,51 +673,51 @@ contains
 
           GROUP: do g = 1,ng
 
-            ! check for active mesh cell
+            ! Check for active mesh cell
             if (allocated(cmfd%coremap)) then
               if (cmfd%coremap(i,j,k) == CMFD_NOACCEL) then
                 cycle
               end if
             end if
 
-            ! get cell data
+            ! Get cell data
             cell_dtilde = cmfd%dtilde(:,g,i,j,k)
             cell_flux = cmfd%flux(g,i,j,k)/product(cmfd%hxyz(:,i,j,k))
             current = cmfd%current(:,g,i,j,k)
 
-            ! setup of vector to identify boundary conditions
+            ! Setup of vector to identify boundary conditions
             bound = (/i,i,j,j,k,k/)
 
-            ! begin loop around sides of cell for leakage
+            ! Begin loop around sides of cell for leakage
             LEAK: do l = 1,6
 
-              ! define xyz and +/- indices
+              ! Define xyz and +/- indices
               xyz_idx = int(ceiling(real(l)/real(2)))  ! x=1, y=2, z=3
               dir_idx = 2 - mod(l,2) ! -=1, +=2
               shift_idx = -2*mod(l,2) +1          ! shift neig by -1 or +1
 
-              ! calculate net current on l face (divided by surf area)
+              ! Calculate net current on l face (divided by surf area)
               net_current = (current(2*l) - current(2*l-1)) / &
                    product(cmfd%hxyz(:,i,j,k)) * cmfd%hxyz(xyz_idx,i,j,k)
 
-              ! check if at a boundary
+              ! Check if at a boundary
               if (bound(l) == nxyz(xyz_idx,dir_idx)) then
 
-                ! compute dhat
+                ! Compute dhat
                 dhat = (net_current - shift_idx*cell_dtilde(l)*cell_flux) / &
                      cell_flux
 
               else  ! not a boundary
 
-                ! compute neighboring cell indices
+                ! Compute neighboring cell indices
                 neig_idx = (/i,j,k/)                ! begin with i,j,k
                 neig_idx(xyz_idx) = shift_idx + neig_idx(xyz_idx)
 
-                ! get neigbor flux 
+                ! Get neigbor flux 
                 neig_flux = cmfd%flux(g,neig_idx(1),neig_idx(2),neig_idx(3)) / &
                      product(cmfd%hxyz(:,neig_idx(1),neig_idx(2),neig_idx(3)))
 
-                ! check for fuel-reflector interface
+                ! Check for fuel-reflector interface
                 if (cmfd_coremap) then
 
                   if (cmfd % coremap(neig_idx(1),neig_idx(2),neig_idx(3)) == &
@@ -723,7 +729,7 @@ contains
 
                   else ! not a fuel-reflector interface
 
-                    ! compute dhat 
+                    ! Compute dhat 
                     dhat = (net_current + shift_idx*cell_dtilde(l)* &
                          (neig_flux - cell_flux))/(neig_flux + cell_flux)
 
@@ -731,7 +737,7 @@ contains
 
                 else ! not for fuel-reflector case
 
-                  ! compute dhat 
+                  ! Compute dhat 
                   dhat = (net_current + shift_idx*cell_dtilde(l)* &
                        (neig_flux - cell_flux))/(neig_flux + cell_flux)
 
@@ -739,13 +745,6 @@ contains
 
               end if
 
-              ! check for zero net current
-!             if ((abs(current(2*l)-current(2*l-1)) < 1e-8_8).and.xyz_idx/=3) then
-!               print *,'Zero net current interface',g,i,j,k
-!               print *,current(2*l),current(2*l-1),net_current
-!               dhat = ZERO
-!             end if
-
               ! record dhat in cmfd object
               cmfd%dhat(l,g,i,j,k) = dhat
 
@@ -784,13 +783,13 @@ contains
     real(8) :: current(12)          ! partial currents for all faces of mesh cell            
     real(8) :: albedo               ! the albedo
 
-    ! get partial currents from object
+    ! Get partial currents from object
     current = cmfd%current(:,g,i,j,k)
 
-    ! define xyz and +/- indices
+    ! Define xyz and +/- indices
     shift_idx = -2*mod(l,2) + 1          ! shift neig by -1 or +1
 
-    ! calculate albedo
+    ! Calculate albedo
     if ((shift_idx ==  1 .and. current(2*l  ) < 1.0e-10_8) .or. &
         (shift_idx == -1 .and. current(2*l-1) < 1.0e-10_8)) then
       albedo = ALBEDO_REJECT 
@@ -799,7 +798,7 @@ contains
       albedo = (current(2*l-1)/current(2*l))**(shift_idx)
     end if
 
-    ! assign to function variable
+    ! Assign to function variable
     get_reflector_albedo = albedo
 
   end function get_reflector_albedo
@@ -814,54 +813,54 @@ contains
     use global,     only: cmfd, keff
     use, intrinsic :: ISO_FORTRAN_ENV
 
-    integer :: nx                ! number of mesh cells in x direction
-    integer :: ny                ! number of mesh cells in y direction
-    integer :: nz                ! number of mesh cells in z direction
-    integer :: ng                ! number of energy groups
-    integer :: i                 ! iteration counter for x
-    integer :: j                 ! iteration counter for y
-    integer :: k                 ! iteration counter for z
-    integer :: l                 ! iteration counter for surface
-    real(8) :: leak1             ! leakage rate in group 1
-    real(8) :: leak2             ! leakage rate in group 2
-    real(8) :: flux1             ! group 1 volume int flux
-    real(8) :: flux2             ! group 2 volume int flux
-    real(8) :: sigt1             ! group 1 total xs
-    real(8) :: sigt2             ! group 2 total xs
-    real(8) :: sigs11            ! scattering transfer 1 --> 1
-    real(8) :: sigs21            ! scattering transfer 2 --> 1
-    real(8) :: sigs12            ! scattering transfer 1 --> 2
-    real(8) :: sigs22            ! scattering transfer 2 --> 2
-    real(8) :: nsigf11           ! fission transfer 1 --> 1
-    real(8) :: nsigf21           ! fission transfer 2 --> 1
-    real(8) :: nsigf12           ! fission transfer 1 --> 2
-    real(8) :: nsigf22           ! fission transfer 2 --> 2
-    real(8) :: siga1             ! group 1 abs xs
-    real(8) :: siga2             ! group 2 abs xs
-    real(8) :: sigs12_eff        ! effective downscatter xs
+    integer :: nx         ! number of mesh cells in x direction
+    integer :: ny         ! number of mesh cells in y direction
+    integer :: nz         ! number of mesh cells in z direction
+    integer :: ng         ! number of energy groups
+    integer :: i          ! iteration counter for x
+    integer :: j          ! iteration counter for y
+    integer :: k          ! iteration counter for z
+    integer :: l          ! iteration counter for surface
+    real(8) :: leak1      ! leakage rate in group 1
+    real(8) :: leak2      ! leakage rate in group 2
+    real(8) :: flux1      ! group 1 volume int flux
+    real(8) :: flux2      ! group 2 volume int flux
+    real(8) :: sigt1      ! group 1 total xs
+    real(8) :: sigt2      ! group 2 total xs
+    real(8) :: sigs11     ! scattering transfer 1 --> 1
+    real(8) :: sigs21     ! scattering transfer 2 --> 1
+    real(8) :: sigs12     ! scattering transfer 1 --> 2
+    real(8) :: sigs22     ! scattering transfer 2 --> 2
+    real(8) :: nsigf11    ! fission transfer 1 --> 1
+    real(8) :: nsigf21    ! fission transfer 2 --> 1
+    real(8) :: nsigf12    ! fission transfer 1 --> 2
+    real(8) :: nsigf22    ! fission transfer 2 --> 2
+    real(8) :: siga1      ! group 1 abs xs
+    real(8) :: siga2      ! group 2 abs xs
+    real(8) :: sigs12_eff ! effective downscatter xs
 
-    ! extract spatial and energy indices from object
+    ! Extract spatial and energy indices from object
     nx = cmfd % indices(1)
     ny = cmfd % indices(2)
     nz = cmfd % indices(3)
     ng = cmfd % indices(4)
 
-    ! return if not two groups
+    ! Return if not two groups
     if (ng /= 2) return
 
-    ! begin loop around space and energy groups
+    ! Begin loop around space and energy groups
     ZLOOP: do k = 1, nz
 
       YLOOP: do j = 1, ny
 
         XLOOP: do i = 1, nx
 
-          ! check for active mesh
+          ! Check for active mesh
           if (allocated(cmfd%coremap)) then
             if (cmfd%coremap(i,j,k) == CMFD_NOACCEL) cycle 
           end if
 
-          ! compute leakage in groups 1 and 2
+          ! Compute leakage in groups 1 and 2
           leak1 = ZERO 
           leak2 = ZERO
           LEAK: do l = 1, 3
@@ -879,7 +878,7 @@ contains
 
           end do LEAK
 
-          ! extract cross sections and flux from object
+          ! Extract cross sections and flux from object
           flux1 = cmfd % flux(1,i,j,k)
           flux2 = cmfd % flux(2,i,j,k)
           sigt1 = cmfd % totalxs(1,i,j,k)
@@ -893,37 +892,37 @@ contains
           nsigf12 = cmfd % nfissxs(1,2,i,j,k)
           nsigf22 = cmfd % nfissxs(2,2,i,j,k)
 
-          ! check for no fission into group 2
+          ! Check for no fission into group 2
           if (.not.(nsigf12 < 1e-6_8 .and. nsigf22 < 1e-6_8)) then
             write(OUTPUT_UNIT,'(A,1PE11.4,1X,1PE11.4)') 'Fission in G=2', &
                   nsigf12,nsigf22
           end if
 
-          ! compute absorption xs
+          ! Compute absorption xs
           siga1 = sigt1 - sigs11 - sigs12
           siga2 = sigt2 - sigs22 - sigs21
 
-          ! compute effective downscatter xs
+          ! Compute effective downscatter xs
           sigs12_eff = (ONE/keff*nsigf11*flux1 - leak1 - siga1*flux1 &
                - ONE/keff*nsigf21/siga2*leak2 ) / ( flux1*(ONE &
                - ONE/keff*nsigf21/siga2))
 
-          ! redefine flux 2
+          ! Redefine flux 2
           flux2 = (sigs12_eff*flux1 - leak2)/siga2
           cmfd % flux(2,i,j,k) = flux2
  
-          ! recompute total cross sections (use effective and no upscattering)
+          ! Recompute total cross sections (use effective and no upscattering)
           sigt1 = siga1 + sigs11 + sigs12_eff
           sigt2 = siga2 + sigs22
 
-          ! record total xs
+          ! Record total xs
           cmfd % totalxs(1,i,j,k) = sigt1
           cmfd % totalxs(2,i,j,k) = sigt2
 
-          ! record effective downscatter xs
+          ! Record effective downscatter xs
           cmfd % scattxs(1,2,i,j,k) = sigs12_eff
 
-          ! zero out upscatter cross section
+          ! Zero out upscatter cross section
           cmfd % scattxs(2,1,i,j,k) = ZERO 
 
         end do XLOOP
@@ -935,7 +934,7 @@ contains
   end subroutine fix_neutron_balance
 
 !===============================================================================
-! FIX_NEUTRON_BALANCE
+! COMPUTE_EFFECTIVE_DOWNSCATTER changes downscatter rate for zero upscatter
 !===============================================================================
 
   subroutine compute_effective_downscatter()
@@ -962,31 +961,28 @@ contains
     real(8) :: siga2             ! group 2 abs xs
     real(8) :: sigs12_eff        ! effective downscatter xs
 
-    ! check for balance
-    if (.not. cmfd_downscatter) return
-
-    ! extract spatial and energy indices from object
+    ! Extract spatial and energy indices from object
     nx = cmfd % indices(1)
     ny = cmfd % indices(2)
     nz = cmfd % indices(3)
     ng = cmfd % indices(4)
 
-    ! return if not two groups
+    ! Return if not two groups
     if (ng /= 2) return
 
-    ! begin loop around space and energy groups
+    ! Begin loop around space and energy groups
     ZLOOP: do k = 1, nz
 
       YLOOP: do j = 1, ny
 
         XLOOP: do i = 1, nx
 
-          ! check for active mesh
+          ! Check for active mesh
           if (allocated(cmfd%coremap)) then
             if (cmfd%coremap(i,j,k) == CMFD_NOACCEL) cycle
           end if
 
-          ! extract cross sections and flux from object
+          ! Extract cross sections and flux from object
           flux1 = cmfd % flux(1,i,j,k)
           flux2 = cmfd % flux(2,i,j,k)
           sigt1 = cmfd % totalxs(1,i,j,k)
@@ -996,25 +992,25 @@ contains
           sigs12 = cmfd % scattxs(1,2,i,j,k)
           sigs22 = cmfd % scattxs(2,2,i,j,k)
 
-          ! compute absorption xs
+          ! Compute absorption xs
           siga1 = sigt1 - sigs11 - sigs12
           siga2 = sigt2 - sigs22 - sigs21
 
-          ! compute effective downscatter xs
+          ! Compute effective downscatter xs
           sigs12_eff = sigs12 - sigs21*flux2/flux1 
 
-          ! recompute total cross sections (use effective and no upscattering)
+          ! Recompute total cross sections (use effective and no upscattering)
           sigt1 = siga1 + sigs11 + sigs12_eff
           sigt2 = siga2 + sigs22
     
-          ! record total xs
+          ! Record total xs
           cmfd % totalxs(1,i,j,k) = sigt1
           cmfd % totalxs(2,i,j,k) = sigt2
 
-          ! record effective downscatter xs
+          ! Record effective downscatter xs
           cmfd % scattxs(1,2,i,j,k) = sigs12_eff
 
-          ! zero out upscatter cross section
+          ! Zero out upscatter cross section
           cmfd % scattxs(2,1,i,j,k) = ZERO 
 
         end do XLOOP

From 989e3b067c7d93c748bbaf6ea8b7127adfab253d Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 9 Jul 2013 09:51:16 -0400
Subject: [PATCH 11/69] cleaned up comments and some bits of code

---
 src/cmfd_execute.F90 | 313 ++++++++++++++++++++-----------------------
 1 file changed, 145 insertions(+), 168 deletions(-)

diff --git a/src/cmfd_execute.F90 b/src/cmfd_execute.F90
index 7588d11a52..8f94bbe18a 100644
--- a/src/cmfd_execute.F90
+++ b/src/cmfd_execute.F90
@@ -5,60 +5,39 @@ module cmfd_execute
 ! cross section generation, diffusion calculation, and source re-weighting
 !==============================================================================
 
+  use global
+
   implicit none
   private
   public :: execute_cmfd, cmfd_init_batch
 
-# ifdef PETSC
-#   include <finclude/petsc.h90>
-# endif
-
 contains
 
 !==============================================================================
-! EXECUTE_CMFD
+! EXECUTE_CMFD runs the CMFD calculation
 !==============================================================================
 
   subroutine execute_cmfd()
 
-# ifdef PETSC
-
     use cmfd_data,              only: set_up_cmfd
-    use cmfd_message_passing,   only: petsc_init_mpi, cmfd_bcast
     use cmfd_power_solver,      only: cmfd_power_execute
     use cmfd_snes_solver,       only: cmfd_snes_execute
     use error,                  only: warning, fatal_error 
-    use global,                 only: n_procs_cmfd, cmfd,                       &
-                                      cmfd_solver_type, time_cmfd,              &
-                                      cmfd_run_adjoint, cmfd_write_hdf5,        &
-                                      cmfd_feedback,cmfd_hold_weights,          &
-                                      cmfd_inact_flush, cmfd_keff_tol,          &
-                                      cmfd_act_flush, current_batch, keff,      &
-                                      n_batches, message, master, mpi_err, rank
 
-    ! stop cmfd timer
+    ! CMFD single processor on master
     if (master) then
+
+      ! Start cmfd timer
       call time_cmfd % start()
-    end if
 
-    ! filter processors (lowest PETSc group)
-    if (rank < n_procs_cmfd) then
+      ! Create cmfd data from OpenMC tallies 
+      call set_up_cmfd()
 
-      ! set up cmfd data (master only)
-      if (master) call set_up_cmfd()
-
-      ! broadcast cmfd to all petsc procs
-      call cmfd_bcast()
-
-      ! process solver options
+      ! Process solver options
       call process_cmfd_options()
 
-    end if
-
-    ! filter processors (lowest PETSc group)
-    if (rank < n_procs_cmfd) then
-
-      ! call solver
+      ! Call solver
+      call time_solver % start()
       if (trim(cmfd_solver_type) == 'power') then
         call cmfd_power_execute()
       elseif (trim(cmfd_solver_type) == 'jfnk') then
@@ -67,20 +46,18 @@ contains
         message = 'solver type became invalid after input processing'
         call fatal_error() 
       end if
+      call time_solver % stop()
 
-      ! perform any last batch tasks 
-      if (current_batch == n_batches) then
-
-        ! check for adjoint run
-        if (cmfd_run_adjoint) then
-          if (trim(cmfd_solver_type) == 'power') then
-            call cmfd_power_execute(adjoint = .true.)
-          elseif (trim(cmfd_solver_type) == 'jfnk') then
-            call cmfd_snes_execute(adjoint = .true.)
-          end if
+      ! check to perform adjoint on last batch
+      call time_solver % start()
+      if (current_batch == n_batches .and. cmfd_run_adjoint) then
+        if (trim(cmfd_solver_type) == 'power') then
+          call cmfd_power_execute(adjoint = .true.)
+        elseif (trim(cmfd_solver_type) == 'jfnk') then
+          call cmfd_snes_execute(adjoint = .true.)
         end if
-
       end if
+      call time_solver % stop()
 
     end if
 
@@ -91,19 +68,12 @@ contains
     if (cmfd_feedback) call cmfd_reweight(.true.)
 
     ! stop cmfd timer
-    if (master) then
-      call time_cmfd % stop()
-    end if
-
-    ! wait here for all procs
-    call MPI_Barrier(MPI_COMM_WORLD, mpi_err)
-
-# endif
+    if (master) call time_cmfd % stop()
 
   end subroutine execute_cmfd
 
 !==============================================================================
-! CMFD_INIT_BATCH
+! CMFD_INIT_BATCH handles cmfd options at the start of every batch
 !==============================================================================
 
   subroutine cmfd_init_batch()
@@ -112,7 +82,7 @@ contains
                                  cmfd_inact_flush, cmfd_act_flush, cmfd_run, &
                                  current_batch, cmfd_hold_weights
 
-    ! check to activate CMFD diffusion and possible feedback
+    ! Check to activate CMFD diffusion and possible feedback
     ! this guarantees that when cmfd begins at least one batch of tallies are
     ! accumulated
     if (cmfd_run .and. cmfd_begin == current_batch) then
@@ -120,18 +90,16 @@ contains
       cmfd_tally_on = .true.
     end if
 
-    ! check to flush cmfd tallies for active batches, no more inactive flush
+    ! Check to flush cmfd tallies for active batches, no more inactive flush
     if (cmfd_run .and. cmfd_act_flush == current_batch) then
       call cmfd_tally_reset()
       cmfd_tally_on = .true.
       cmfd_inact_flush(2) = -1
     end if
 
-    ! check to flush cmfd tallies during inactive batches (>= on number of
+    ! Check to flush cmfd tallies during inactive batches (>= on number of
     ! flushes important as the code will flush on the first batch which we
     ! dont want to count)
-!    if (cmfd_run .and. current_batch < n_inactive .and. mod(current_batch-1,cmfd_inact_flush(1))   &
-!       == 0 .and. cmfd_inact_flush(2) >= 0) then
     if (cmfd_run .and. mod(current_batch,cmfd_inact_flush(1))   &
        == 0 .and. cmfd_inact_flush(2) > 0 .and. cmfd_begin < current_batch) then
         cmfd_hold_weights = .true.
@@ -144,20 +112,22 @@ contains
 # ifdef PETSC
 
 !==============================================================================
-! PROCESS_CMFD_OPTIONS 
+! PROCESS_CMFD_OPTIONS processes user options that interface with PETSc 
 !==============================================================================
 
   subroutine process_cmfd_options()
 
     use global,       only: cmfd_snes_monitor, cmfd_ksp_monitor, mpi_err
 
-    ! check for snes monitor
+#ifdef PETSC
+    ! Check for snes monitor
     if (cmfd_snes_monitor) call PetscOptionsSetValue("-snes_monitor", &
          "stdout", mpi_err)
 
-    ! check for ksp monitor
+    ! Check for ksp monitor
     if (cmfd_ksp_monitor) call PetscOptionsSetValue("-ksp_monitor", &
          "stdout", mpi_err)
+#endif
 
     end subroutine process_cmfd_options
 
@@ -169,38 +139,42 @@ contains
 
     use constants,  only: CMFD_NOACCEL, ZERO, TWO
     use global,     only: cmfd, cmfd_coremap, master, mpi_err, entropy_on
+#ifdef MPI
+    use mpi
+#endif
 
-    integer :: nx ! maximum number of cells in x direction
-    integer :: ny ! maximum number of cells in y direction
-    integer :: nz ! maximum number of cells in z direction
-    integer :: ng ! maximum number of energy groups
-    integer :: n  ! total size
-    integer :: i ! iteration counter for x
-    integer :: j ! iteration counter for y
-    integer :: k ! iteration counter for z
-    integer :: g ! iteration counter for groups
-    integer :: idx ! index in vector
+    integer :: nx      ! maximum number of cells in x direction
+    integer :: ny      ! maximum number of cells in y direction
+    integer :: nz      ! maximum number of cells in z direction
+    integer :: ng      ! maximum number of energy groups
+    integer :: n       ! total size
+    integer :: i       ! iteration counter for x
+    integer :: j       ! iteration counter for y
+    integer :: k       ! iteration counter for z
+    integer :: g       ! iteration counter for groups
+    integer :: idx     ! index in vector
     real(8) :: hxyz(3) ! cell dimensions of current ijk cell
     real(8) :: vol     ! volume of cell
     real(8),allocatable :: source(:,:,:,:)  ! tmp source array for entropy
 
-    ! get maximum of spatial and group indices
-    nx = cmfd%indices(1)
-    ny = cmfd%indices(2)
-    nz = cmfd%indices(3)
-    ng = cmfd%indices(4)
+    ! Get maximum of spatial and group indices
+    nx = cmfd % indices(1)
+    ny = cmfd % indices(2)
+    nz = cmfd % indices(3)
+    ng = cmfd % indices(4)
     n  = ng*nx*ny*nz
 
-    ! allocate cmfd source if not already allocated and allocate buffer
-    if (.not. allocated(cmfd%cmfd_src)) allocate(cmfd%cmfd_src(ng,nx,ny,nz))
+    ! Allocate cmfd source if not already allocated and allocate buffer
+    if (.not. allocated(cmfd % cmfd_src)) &
+       allocate(cmfd % cmfd_src(ng,nx,ny,nz))
 
-    ! reset cmfd source to 0
-    cmfd%cmfd_src = ZERO
+    ! Reset cmfd source to 0
+    cmfd % cmfd_src = ZERO
 
-    ! only perform for master
+    ! Only perform for master
     if (master) then
 
-      ! loop around indices to map to cmfd object
+      ! Loop around indices to map to cmfd object
       ZLOOP: do k = 1, nz
 
         YLOOP: do j = 1, ny
@@ -209,25 +183,25 @@ contains
 
             GROUP: do g = 1, ng
 
-              ! check for core map
+              ! Check for core map
               if (cmfd_coremap) then
-                if (cmfd%coremap(i,j,k) == CMFD_NOACCEL) then
+                if (cmfd % coremap(i,j,k) == CMFD_NOACCEL) then
                   cycle
                 end if
               end if
 
-              ! get dimensions of cell
-              hxyz = cmfd%hxyz(:,i,j,k)
+              ! Get dimensions of cell
+              hxyz = cmfd % hxyz(:,i,j,k)
 
-              ! calculate volume
+              ! Calculate volume
               vol = hxyz(1)*hxyz(2)*hxyz(3)
 
-              ! get first index
+              ! Get first index
               idx = get_matrix_idx(1,i,j,k,ng,nx,ny)
 
-              ! compute fission source
-              cmfd%cmfd_src(g,i,j,k) = sum(cmfd%nfissxs(:,g,i,j,k) * &
-                   cmfd%phi(idx:idx+(ng-1)))*vol
+              ! Compute fission source
+              cmfd % cmfd_src(g,i,j,k) = sum(cmfd % nfissxs(:,g,i,j,k) * &
+                     cmfd % phi(idx:idx + (ng - 1)))*vol
 
             end do GROUP
 
@@ -237,38 +211,38 @@ contains
 
       end do ZLOOP
 
-      ! normalize source such that it sums to 1.0
-      cmfd%cmfd_src = cmfd%cmfd_src/sum(cmfd%cmfd_src)
+      ! Normalize source such that it sums to 1.0
+      cmfd % cmfd_src = cmfd % cmfd_src/sum(cmfd % cmfd_src)
 
-      ! compute entropy
+      ! Compute entropy
       if (entropy_on) then
 
-        ! allocate tmp array
+        ! Allocate tmp array
         if (.not.allocated(source)) allocate(source(ng,nx,ny,nz))
 
-        ! initialize the source
+        ! Initialize the source
         source = ZERO
 
-        ! compute log
-        where (cmfd%cmfd_src > ZERO)
-          source = cmfd%cmfd_src*log(cmfd%cmfd_src)/log(TWO)
+        ! Compute log
+        where (cmfd % cmfd_src > ZERO)
+          source = cmfd % cmfd_src*log(cmfd % cmfd_src)/log(TWO)
         end where
 
-        ! sum that source
-        cmfd%entropy = -sum(source)
+        ! Sum that source
+        cmfd % entropy = -sum(source)
 
-        ! deallocate tmp array
+        ! Deallocate tmp array
         if (allocated(source)) deallocate(source)
 
       end if
 
-      ! normalize source so average is 1.0
-      cmfd%cmfd_src = cmfd%cmfd_src/sum(cmfd%cmfd_src)*cmfd%norm
+      ! Normalize source so average is 1.0
+      cmfd % cmfd_src = cmfd % cmfd_src/sum(cmfd % cmfd_src)*cmfd % norm
 
     end if
 
-    ! broadcast full source to all procs
-    call MPI_BCAST(cmfd%cmfd_src, n, MPI_REAL8, 0, MPI_COMM_WORLD, mpi_err)
+    ! Broadcast full source to all procs
+    call MPI_BCAST(cmfd % cmfd_src, n, MPI_REAL8, 0, MPI_COMM_WORLD, mpi_err)
 
   end subroutine calc_fission_source
 
@@ -287,114 +261,117 @@ contains
     use mesh,        only: count_bank_sites, get_mesh_indices
     use search,      only: binary_search
 
-    ! local variables
-    integer :: nx ! maximum number of cells in x direction
-    integer :: ny ! maximum number of cells in y direction
-    integer :: nz ! maximum number of cells in z direction
-    integer :: ng ! maximum number of energy groups
-    integer :: i ! iteration counter
-    integer :: ijk(3) ! spatial bin location
-    integer :: e_bin ! energy bin of source particle
-    integer :: n_groups ! number of energy groups
-    integer(8) :: size_bank ! size of source bank
-    logical :: outside ! any source sites outside mesh
-    logical :: in_mesh ! source site is inside mesh
-    logical :: new_weights ! calcualte new weights
-    type(StructuredMesh), pointer :: m ! point to mesh
-    real(8), allocatable :: egrid(:)
+#ifdef MPI
+    use mpi
+#endif
 
-    ! associate pointer
+    integer :: nx                      ! maximum number of cells in x direction
+    integer :: ny                      ! maximum number of cells in y direction
+    integer :: nz                      ! maximum number of cells in z direction
+    integer :: ng                      ! maximum number of energy groups
+    integer :: i                       ! iteration counter
+    integer :: ijk(3)                  ! spatial bin location
+    integer :: e_bin                   ! energy bin of source particle
+    integer :: n_groups                ! number of energy groups
+    integer(8) :: size_bank            ! size of source bank
+    logical :: outside                 ! any source sites outside mesh
+    logical :: in_mesh                 ! source site is inside mesh
+    logical :: new_weights             ! calcualte new weights
+    type(StructuredMesh), pointer :: m ! point to mesh
+    real(8), allocatable :: egrid(:)   ! energy grid
+
+    ! Associate pointer
     m => meshes(n_user_meshes + 1)
 
-    ! get maximum of spatial and group indices
-    nx = cmfd%indices(1)
-    ny = cmfd%indices(2)
-    nz = cmfd%indices(3)
-    ng = cmfd%indices(4)
+    ! Get maximum of spatial and group indices
+    nx = cmfd % indices(1)
+    ny = cmfd % indices(2)
+    nz = cmfd % indices(3)
+    ng = cmfd % indices(4)
 
-    ! compute size of source bank
+    ! Compute size of source bank
     size_bank = bank_last - bank_first + 1_8 
 
-    ! allocate arrays in cmfd object (can take out later extend to multigroup)
-    if (.not.allocated(cmfd%sourcecounts)) then 
-      allocate(cmfd%sourcecounts(ng,nx,ny,nz))
+    ! Allocate arrays in cmfd object (can take out later extend to multigroup)
+    if (.not.allocated(cmfd % sourcecounts)) then 
+      allocate(cmfd % sourcecounts(ng,nx,ny,nz))
       cmfd % sourcecounts = 0
     end if
-    if (.not.allocated(cmfd%weightfactors)) then 
-      allocate(cmfd%weightfactors(ng,nx,ny,nz))
+    if (.not.allocated(cmfd % weightfactors)) then 
+      allocate(cmfd % weightfactors(ng,nx,ny,nz))
       cmfd % weightfactors = ONE
     end if
 
-    ! allocate energy grid and reverse cmfd energy grid
-    if (.not. allocated(egrid)) allocate(egrid(ng+1))
-    egrid = (/(cmfd%egrid(ng-i+2),i = 1,ng+1)/)
+    ! Allocate energy grid and reverse cmfd energy grid
+    if (.not. allocated(egrid)) allocate(egrid(ng + 1))
+    egrid = (/(cmfd % egrid(ng - i + 2), i = 1, ng + 1)/)
 
-    ! compute new weight factors
+    ! Compute new weight factors
     if (new_weights) then
 
-      ! zero out weights
+      ! Zero out weights
       cmfd%weightfactors = ZERO
 
-      ! count bank sites in mesh
+      ! Count bank sites in mesh
       call count_bank_sites(m, source_bank, cmfd%sourcecounts, egrid, &
            sites_outside=outside, size_bank = size_bank)
 
-      ! check for sites outside of the mesh
+      ! Check for sites outside of the mesh
       if (master .and. outside) then
         message = "Source sites outside of the CMFD mesh!"
         call fatal_error()
       end if
 
-      ! have master compute weight factors
+      ! Have master compute weight factors (watch for 0s)
       if (master) then
-        where(cmfd%cmfd_src > ZERO .and. cmfd%sourcecounts > ZERO)
-          cmfd%weightfactors = cmfd%cmfd_src/sum(cmfd%cmfd_src)* &
-                               sum(cmfd%sourcecounts) / cmfd%sourcecounts
+        where(cmfd % cmfd_src > ZERO .and. cmfd % sourcecounts > ZERO)
+          cmfd % weightfactors = cmfd % cmfd_src/sum(cmfd % cmfd_src)* &
+                               sum(cmfd % sourcecounts) / cmfd % sourcecounts
         end where
       end if
 
-      ! broadcast weight factors to all procs
-      call MPI_BCAST(cmfd%weightfactors, ng*nx*ny*nz, MPI_REAL8, 0, &
+      ! Broadcast weight factors to all procs
+      call MPI_BCAST(cmfd % weightfactors, ng*nx*ny*nz, MPI_REAL8, 0, &
            MPI_COMM_WORLD, mpi_err)
 
    end if
 
-    ! begin loop over source bank
+    ! Begin loop over source bank
     do i = 1, int(size_bank, 4) 
 
-      ! determine spatial bin
-      call get_mesh_indices(m, source_bank(i)%xyz, ijk, in_mesh)
+      ! Determine spatial bin
+      call get_mesh_indices(m, source_bank(i) % xyz, ijk, in_mesh)
 
-      ! determine energy bin
-      n_groups = size(cmfd%egrid) - 1
-      if (source_bank(i) % E < cmfd%egrid(1)) then
+      ! Determine energy bin
+      n_groups = size(cmfd % egrid) - 1
+      if (source_bank(i) % E < cmfd % egrid(1)) then
         e_bin = 1
-        message = 'source pt below energy grid'
+        message = 'Source pt below energy grid'
         call warning()
-      elseif (source_bank(i) % E > cmfd%egrid(n_groups+1)) then
+      elseif (source_bank(i) % E > cmfd % egrid(n_groups + 1)) then
         e_bin = n_groups
-        message = 'source pt above energy grid'
+        message = 'Source pt above energy grid'
         call warning()
       else
-        e_bin = binary_search(cmfd%egrid, n_groups + 1, source_bank(i) % E)
+        e_bin = binary_search(cmfd % egrid, n_groups + 1, source_bank(i) % E)
       end if
 
-      ! reverese energy bin (lowest grp is highest energy bin)
+      ! Reverese energy bin (lowest grp is highest energy bin)
       e_bin = n_groups - e_bin + 1
 
-      ! check for outside of mesh
+      ! Check for outside of mesh
       if (.not. in_mesh) then
-        message = 'Source site found outside of CMFD mesh!'
+        message = 'Source site found outside of CMFD mesh'
         call fatal_error()
       end if
 
-      ! reweight particle
-      source_bank(i)%wgt = source_bank(i)%wgt * &
-           cmfd%weightfactors(e_bin,ijk(1),ijk(2),ijk(3))
+      ! Reweight particle
+      source_bank(i) % wgt = source_bank(i) % wgt * &
+             cmfd % weightfactors(e_bin, ijk(1), ijk(2), ijk(3))
 
     end do
 
-    ! deallocate
+    ! Deallocate all
     if (allocated(egrid)) deallocate(egrid)
 
   end subroutine cmfd_reweight
@@ -416,15 +393,15 @@ contains
     integer :: ny     ! maximum number of cells in y direction
     integer :: ng     ! maximum number of energy groups
 
-    ! check if coremap is used
+    ! Check if coremap is used
     if (cmfd_coremap) then
 
-      ! get idx from core map
+      ! Get idx from core map
       matidx = ng*(cmfd % coremap(i,j,k)) - (ng - g)
 
     else
 
-      ! compute index
+      ! Compute index
       matidx = g + ng*(i - 1) + ng*nx*(j - 1) + ng*nx*ny*(k - 1)
 
     end if
@@ -445,14 +422,14 @@ contains
 
     integer :: i ! loop counter
 
-    ! print message
+    ! Print message
     message = "CMFD tallies reset"
     call write_message(7)
 
-    ! begin loop around CMFD tallies
+    ! Begin loop around CMFD tallies
     do i = 1, n_cmfd_tallies
 
-      ! reset that tally
+      ! Reset that tally
       cmfd_tallies(i) % n_realizations = 0
       call reset_result(cmfd_tallies(i) % results)
 

From 9cf923e63797d1c22b71a43acf8fbd7c740f3cc7 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 9 Jul 2013 09:56:57 -0400
Subject: [PATCH 12/69] cleaned up cmfd input comments and bits of code

---
 src/cmfd_input.F90 | 221 ++++++++++++++++++++++++---------------------
 1 file changed, 116 insertions(+), 105 deletions(-)

diff --git a/src/cmfd_input.F90 b/src/cmfd_input.F90
index c228890d1e..d251ec379a 100644
--- a/src/cmfd_input.F90
+++ b/src/cmfd_input.F90
@@ -1,29 +1,53 @@
 module cmfd_input
 
+  use global
+
   implicit none
   private
   public :: configure_cmfd 
 
+# ifdef PETSC
+#  include <finclude/petsc.h90>
+# endif
+
 contains
 
 !===============================================================================
-! CONFIGURE_CMFD
+! CONFIGURE_CMFD initializes PETSc and CMFD parameters
 !===============================================================================
 
   subroutine configure_cmfd()
 
-# ifdef PETSC
-    use cmfd_message_passing,   only: petsc_init_mpi
-# endif
+    integer :: new_comm ! new mpi communicator
+    integer :: color    ! color group of processor
 
-    ! read in cmfd input file
+    ! Read in cmfd input file
     call read_cmfd_xml()
 
-    ! initialize petsc on mpi
-# ifdef PETSC
-    call petsc_init_mpi()
+    ! Assign color
+    if (master) then
+      color = 1
+    else
+      color = 2
+    end if
+
+    ! Split up procs
+# ifdef MPI
+    call MPI_COMM_SPLIT(MPI_COMM_WORLD,color,0,new_comm,mpi_err)
 # endif
 
+    ! assign to PETSc
+# ifdef PETSC
+    PETSC_COMM_WORLD = new_comm
+
+    ! Initialize petsc on all procs
+    call PetscInitialize(PETSC_NULL_CHARACTER,mpi_err)
+# endif
+
+    ! initialize timers
+    call time_cmfd % reset()
+    call time_solver % reset()
+
   end subroutine configure_cmfd
 
 !===============================================================================
@@ -33,17 +57,16 @@ contains
   subroutine read_cmfd_xml()
     
     use error,   only: fatal_error
-    use global
     use output,  only: write_message
     use string,  only: lower_case
     use xml_data_cmfd_t
     use, intrinsic :: ISO_FORTRAN_ENV
 
-    integer :: ng
-    logical :: file_exists ! does cmfd.xml exist?
-    character(MAX_LINE_LEN) :: filename
+    integer :: ng                       ! number of energy groups
+    logical :: file_exists              ! does cmfd.xml exist?
+    character(MAX_LINE_LEN) :: filename ! name of input file
 
-    ! read cmfd infput file
+    ! Read cmfd input file
     filename = trim(path_input) // "cmfd.xml"
     inquire(FILE=filename, EXIST=file_exists)
     if (.not. file_exists) then
@@ -55,79 +78,78 @@ contains
       return
     else
 
-      ! tell user
+      ! Tell user
       message = "Reading CMFD XML file..."
       call write_message(5)
 
     end if
 
-    ! parse cmfd.xml file
+    ! Parse cmfd.xml file
     call read_xml_file_cmfd_t(filename)
 
-    ! set spatial dimensions in cmfd object
-    cmfd % indices(1:3) = mesh_ % dimension(1:3) ! sets spatial dimensions
+    ! Set spatial dimensions in cmfd object (structed Cartesian mesh)
+    cmfd % indices(1:3) = mesh_ % dimension(1:3)
 
-    ! get number of energy groups
+    ! Get number of energy groups or set to 1 group default
     if (associated(mesh_ % energy)) then
       ng = size(mesh_ % energy)
-      if(.not.allocated(cmfd%egrid)) allocate(cmfd%egrid(ng))
-      cmfd%egrid = mesh_ % energy 
+      if(.not.allocated(cmfd % egrid)) allocate(cmfd % egrid(ng))
+      cmfd % egrid = mesh_ % energy 
       cmfd % indices(4) = ng - 1 ! sets energy group dimension
     else
-      if(.not.allocated(cmfd%egrid)) allocate(cmfd%egrid(2))
-      cmfd%egrid = (/0.0_8,20.0_8/)
+      if(.not.allocated(cmfd % egrid)) allocate(cmfd % egrid(2))
+      cmfd % egrid = (/0.0_8,20.0_8/)
       cmfd % indices(4) = 1 ! one energy group
     end if
     
-    ! set global albedo
+    ! Set global albedo, these can be overwritten by coremap
     if (associated(mesh_ % albedo)) then
       cmfd % albedo = mesh_ % albedo
     else
       cmfd % albedo = (/1.0, 1.0, 1.0, 1.0, 1.0, 1.0/)
     end if
 
-    ! get acceleration map
+    ! Get core map overlay for a subset mesh for CMFD 
     if (associated(mesh_ % map)) then
+
+      ! Allocate a core map with appropriate dimensions
       allocate(cmfd % coremap(cmfd % indices(1), cmfd % indices(2), &
            cmfd % indices(3)))
+
+      ! Check and make sure it is of correct size
       if (size(mesh_ % map) /= product(cmfd % indices(1:3))) then
-        message = 'FATAL==>CMFD coremap not to correct dimensions'
+        message = 'CMFD coremap not to correct dimensions'
         call fatal_error() 
       end if
-      cmfd % coremap = reshape(mesh_ % map,(cmfd % indices(1:3)))
-      cmfd_coremap = .true.
-   end if
 
-    ! check for core map activation by printing note
-    if (cmfd_coremap .and. master) then
+      ! Reshape core map vector into (x,y,z) array
+      cmfd % coremap = reshape(mesh_ % map,(cmfd % indices(1:3)))
+
+      ! Indicate to cmfd that a core map overlay is active
+      cmfd_coremap = .true.
+
+      ! Write to stdout that a core map overlay is active
       message = "Core Map Overlay Activated"
       call write_message()
+
     end if
 
-    ! check for normalization constant
+    ! Get normalization constant for source (default is 1.0 from XML) 
     cmfd % norm = norm_
 
-    ! set feedback logical
+    ! Set feedback logical
     call lower_case(feedback_)
     if (feedback_ == 'true' .or. feedback_ == '1') cmfd_feedback = .true.
 
-    ! set balance logical
-    ! call lower_case(balance_)
-    ! if (balance_ == 'true' .or. balance == '1') cmfd_balance = .true.
+    ! Set downscatter logical
+    call lower_case(downscatter_)
+    if (downscatter_ == 'true' .or. downscatter_ == '1') &
+         cmfd_downscatter = .true.
 
-    ! set downscatter logical
-    ! call lower_case(downscatter_)
-    ! if (downscatter_ == 'true' .or. downscatter == '1') &
-    !      cmfd_downscatter = downscatter_
-
-    ! set 2 group fix
-    call lower_case(run_2grp_)
-    if (run_2grp_ == 'true' .or. run_2grp_ == '1') cmfd_run_2grp = .true.
-
-    ! set the solver type
+    ! Set the solver type (default power from XML)
     cmfd_solver_type = solver_(1:10)
 
-    ! set monitoring 
+    ! Set convergence monitoring 
     call lower_case(snes_monitor_)
     call lower_case(ksp_monitor_)
     call lower_case(power_monitor_)
@@ -138,47 +160,36 @@ contains
     if (power_monitor_ == 'true' .or. power_monitor_ == '1') &
          cmfd_power_monitor = .true.
 
-    ! output logicals
+    ! Output logicals
     call lower_case(write_balance_)
     call lower_case(write_matrices_)
-    ! call lower_case(write_hdf5_)
     if (write_balance_ == 'true' .or. write_balance_ == '1') &
          cmfd_write_balance = .true.
     if (write_matrices_ == 'true' .or. write_matrices_ == '1') &
          cmfd_write_matrices = .true.
-    ! if (write_hdf5_ == 'true' .or. write_hdf5_ == '1') &
-    !      cmfd_write_hdf5 = .true.
 
-    ! run an adjoint calc
+    ! Run an adjoint calc
     call lower_case(run_adjoint_)
     if (run_adjoint_ == 'true' .or. run_adjoint_ == '1') &
          cmfd_run_adjoint = .true.
 
-    ! batch to begin cmfd
+    ! Batch to begin cmfd (default is 1 from XML)
     cmfd_begin = begin_
 
-    ! tally during inactive batches
+    ! Tally during inactive batches (by default we will always tally from 1)
     call lower_case(inactive_)
     if (inactive_ == 'false' .or. inactive_ == '0') cmfd_tally_on = .false.
 
-    ! inactive batch flush window
-    cmfd_inact_flush(1) = inactive_flush_
-    cmfd_inact_flush(2) = num_flushes_
+    ! Inactive batch flush window
+    cmfd_inact_flush(1) = inactive_flush_ ! the interval of batches
+    cmfd_inact_flush(2) = num_flushes_    ! number of times to do this
 
-    ! last flush before active batches
+    ! Last flush before active batches
     cmfd_act_flush = active_flush_
 
-    ! tolerance on keff
-    cmfd_keff_tol = keff_tol_
-    
-    ! create tally objects
+    ! Create tally objects
     call create_cmfd_tally()
 
-    ! set number of CMFD processors and report to user
-    n_procs_cmfd = n_cmfd_procs_ 
-    if (master) write(OUTPUT_UNIT,'(A,1X,I0,1X,A)') "CMFD Running on", &
-       n_procs_cmfd," processors."
-
   end subroutine read_cmfd_xml
 
 !===============================================================================
@@ -193,7 +204,6 @@ contains
   subroutine create_cmfd_tally()
 
     use error,            only: fatal_error, warning
-    use global
     use mesh_header,      only: StructuredMesh
     use string
     use tally,            only: setup_active_cmfdtallies
@@ -211,22 +221,22 @@ contains
     type(StructuredMesh), pointer :: m => null()
     type(TallyFilter) :: filters(N_FILTER_TYPES) ! temporary filters
 
-    ! parse cmfd.xml file
+    ! Parse cmfd.xml file
      filename = trim(path_input) // "cmfd.xml"
      call read_xml_file_cmfd_t(filename)
 
-    ! set global variables if they are 0 (this can happen if there is no tally
+    ! Set global variables if they are 0 (this can happen if there is no tally
     ! file)
     if (n_meshes == 0) n_meshes = n_user_meshes + n_cmfd_meshes
 
-    ! allocate mesh
+    ! Allocate mesh
     if (.not. allocated(meshes)) allocate(meshes(n_meshes))
     m => meshes(n_user_meshes+1)
 
-    ! set mesh id
+    ! Set mesh id
     m % id = n_user_meshes + 1 
 
-    ! set mesh type to rectangular
+    ! Set mesh type to rectangular
     m % type = LATTICE_RECT
 
     ! Determine number of dimensions for mesh
@@ -321,20 +331,20 @@ contains
     ! Add mesh to dictionary
     call mesh_dict % add_key(m % id, n_user_meshes + 1)
 
-    ! allocate tallies
+    ! Allocate tallies
     call add_tallies("cmfd", n_cmfd_tallies)
 
-    ! begin loop around tallies
+    ! Begin loop around tallies
     do i = 1, n_cmfd_tallies
 
-      ! point t to tally variable
+      ! Point t to tally variable
       t => cmfd_tallies(i)
 
-      ! set reset property
+      ! Set reset property
       call lower_case(reset_)
       if (reset_ == 'true' .or. reset_ == '1') t % reset = .true.
 
-      ! set up mesh filter
+      ! Set up mesh filter
       n_filters = 1
       filters(n_filters) % type = FILTER_MESH
       filters(n_filters) % n_bins = product(m % dimension)
@@ -342,7 +352,7 @@ contains
       filters(n_filters) % int_bins(1) = n_user_meshes + 1
       t % find_filter(FILTER_MESH) = n_filters
 
-      ! read and set incoming energy mesh filter
+      ! Read and set incoming energy mesh filter
       if (associated(mesh_ % energy)) then
         n_filters = n_filters + 1
         filters(n_filters) % type = FILTER_ENERGYIN
@@ -353,40 +363,40 @@ contains
         t % find_filter(FILTER_ENERGYIN) = n_filters
       end if
 
-      ! set number of nucilde bins
+      ! Set number of nucilde bins
       allocate(t % nuclide_bins(1))
       t % nuclide_bins(1) = -1
       t % n_nuclide_bins = 1
 
-      ! record tally id which is equivalent to loop number
+      ! Record tally id which is equivalent to loop number
       t % id = i_cmfd_tallies + i
 
       if (i == 1) then
 
-        ! set label
+        ! Set label
         t % label = "CMFD flux, total, scatter-1"
 
-        ! set tally estimator to analog
+        ! Set tally estimator to analog
         t % estimator = ESTIMATOR_ANALOG
 
-        ! set tally type to volume
+        ! Set tally type to volume
         t % type = TALLY_VOLUME
 
-        ! allocate and set filters
+        ! Allocate and set filters
         t % n_filters = n_filters
         allocate(t % filters(n_filters))
         t % filters = filters(1:n_filters)
 
-        ! allocate scoring bins 
+        ! Allocate scoring bins 
         allocate(t % score_bins(3))
         t % n_score_bins = 3
         t % n_user_score_bins = 3
 
-        ! allocate scattering order data
+        ! Allocate scattering order data
         allocate(t % scatt_order(3))
         t % scatt_order = 0
         
-        ! set macro_bins
+        ! Set macro_bins
         t % score_bins(1)  = SCORE_FLUX
         t % score_bins(2)  = SCORE_TOTAL
         t % score_bins(3)  = SCORE_SCATTER_N
@@ -394,16 +404,16 @@ contains
 
       else if (i == 2) then
 
-        ! set label
+        ! Set label
         t % label = "CMFD neutron production"
 
-        ! set tally estimator to analog
+        ! Set tally estimator to analog
         t % estimator = ESTIMATOR_ANALOG
 
-        ! set tally type to volume
+        ! Set tally type to volume
         t % type = TALLY_VOLUME
 
-        ! read and set outgoing energy mesh filter
+        ! Read and set outgoing energy mesh filter
         if (associated(mesh_ % energy)) then
           n_filters = n_filters + 1
           filters(n_filters) % type = FILTER_ENERGYOUT
@@ -414,34 +424,34 @@ contains
           t % find_filter(FILTER_ENERGYOUT) = n_filters
         end if
 
-        ! allocate and set filters
+        ! Allocate and set filters
         t % n_filters = n_filters
         allocate(t % filters(n_filters))
         t % filters = filters(1:n_filters)
 
-        ! deallocate filters bins array
+        ! Deallocate filters bins array
         if (associated(mesh_ % energy)) &
              deallocate(filters(n_filters) % real_bins)
 
-        ! allocate macro reactions
+        ! Allocate macro reactions
         allocate(t % score_bins(2))
         t % n_score_bins = 2
         t % n_user_score_bins = 2
 
-        ! allocate scattering order data
+        ! Allocate scattering order data
         allocate(t % scatt_order(2))
         t % scatt_order = 0
 
-        ! set macro_bins
+        ! Set macro_bins
         t % score_bins(1) = SCORE_NU_SCATTER
         t % score_bins(2) = SCORE_NU_FISSION
 
       else if (i == 3) then
 
-        ! set label
+        ! Set label
         t % label = "CMFD surface currents"
 
-        ! set tally estimator to analog
+        ! Set tally estimator to analog
         t % estimator = ESTIMATOR_ANALOG
 
         ! Add extra filter for surface
@@ -458,40 +468,41 @@ contains
         end if
         t % find_filter(FILTER_SURFACE) = n_filters
 
-        ! allocate and set filters
+        ! Allocate and set filters
         t % n_filters = n_filters
         allocate(t % filters(n_filters))
         t % filters = filters(1:n_filters)
 
-        ! deallocate filters bins array
+        ! Deallocate filters bins array
         deallocate(filters(n_filters) % int_bins)
 
-        ! allocate macro reactions
+        ! Allocate macro reactions
         allocate(t % score_bins(1))
         t % n_score_bins = 1
         t % n_user_score_bins = 1
 
-        ! allocate scattering order data
+        ! Allocate scattering order data
         allocate(t % scatt_order(1))
         t % scatt_order = 0
 
-        ! set macro bins
+        ! Set macro bins
         t % score_bins(1) = SCORE_CURRENT
         t % type = TALLY_SURFACE_CURRENT
 
-        ! we need to increase the dimension by one since we also need
+        ! We need to increase the dimension by one since we also need
         ! currents coming into and out of the boundary mesh cells.
         i_filter_mesh = t % find_filter(FILTER_MESH)
         t % filters(i_filter_mesh) % n_bins = product(m % dimension + 1)
 
       end if
 
-      ! deallocate filter bins
+      ! Deallocate filter bins
       deallocate(filters(1) % int_bins)
       if (associated(mesh_ % energy)) deallocate(filters(2) % real_bins)
 
     end do
 
+    ! Put cmfd tallies into active tally array and turn tallies on
     call setup_active_cmfdtallies()
     tallies_on = .true.
 

From 971946fc4c8f5c54d06927306f8baf4716c560b9 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 9 Jul 2013 09:57:28 -0400
Subject: [PATCH 13/69] no longer need cmfd message passing routines

---
 src/cmfd_message_passing.F90 | 98 ------------------------------------
 1 file changed, 98 deletions(-)
 delete mode 100644 src/cmfd_message_passing.F90

diff --git a/src/cmfd_message_passing.F90 b/src/cmfd_message_passing.F90
deleted file mode 100644
index f5269bc2a7..0000000000
--- a/src/cmfd_message_passing.F90
+++ /dev/null
@@ -1,98 +0,0 @@
-module cmfd_message_passing
-
-# ifdef PETSC
-
-! This module contains MPI routines related to the CMFD calculation
-
-  implicit none
-  private
-  public :: petsc_init_mpi, cmfd_bcast 
-
-# include <finclude/petsc.h90>
-
-contains
-
-!===============================================================================
-! PETSC_INIT_MPI
-!===============================================================================
-
-  subroutine petsc_init_mpi()
-
-    use global,       only: n_procs_cmfd, time_cmfd, rank, mpi_err
-
-    integer               :: new_comm   ! new communicator
-    integer               :: color      ! communicator color
-
-    ! assign color
-    if (rank < n_procs_cmfd) then
-      color = 1
-    else
-      color = 2
-    end if
-
-    ! split up procs
-    call MPI_COMM_SPLIT(MPI_COMM_WORLD,color,0,new_comm,mpi_err)
-
-    ! assign to PETSc
-    PETSC_COMM_WORLD = new_comm
-
-    ! initialize petsc on all procs
-    call PetscInitialize(PETSC_NULL_CHARACTER,mpi_err)
-
-    ! initialize timer
-    call time_cmfd % reset()
-
-  end subroutine petsc_init_mpi
-
-!===============================================================================
-! CMFD_BCAST
-!===============================================================================
-
-  subroutine cmfd_bcast()
-
-    use global,       only: cmfd_coremap, cmfd, mpi_err
-    use cmfd_header,  only: allocate_cmfd
-
-    integer :: nx  ! number of mesh cells in x direction
-    integer :: ny  ! number of mesh cells in y direction
-    integer :: nz  ! number of mesh cells in z direction
-    integer :: ng  ! number of energy groups
-
-    ! extract spatial and energy indices from object
-    nx = cmfd % indices(1)
-    ny = cmfd % indices(2)
-    nz = cmfd % indices(3)
-    ng = cmfd % indices(4)
-
-    ! initialize data
-    if (.not. allocated(cmfd%flux)) call allocate_cmfd(cmfd)
-
-    ! sync up procs
-    call MPI_Barrier(PETSC_COMM_WORLD, mpi_err)
-
-    ! broadcast all data
-    call MPI_BCAST(cmfd%flux, ng*nx*ny*nz, MPI_REAL8, 0, PETSC_COMM_WORLD, mpi_err)
-    call MPI_BCAST(cmfd%totalxs, ng*nx*ny*nz, MPI_REAL8, 0, PETSC_COMM_WORLD, mpi_err)
-    call MPI_BCAST(cmfd%p1scattxs, ng*nx*ny*nz, MPI_REAL8, 0, PETSC_COMM_WORLD, mpi_err)
-    call MPI_BCAST(cmfd%scattxs, ng*ng*nx*ny*nz, MPI_REAL8, 0, PETSC_COMM_WORLD, mpi_err)
-    call MPI_BCAST(cmfd%nfissxs, ng*ng*nx*ny*nz, MPI_REAL8, 0, PETSC_COMM_WORLD, mpi_err)
-    call MPI_BCAST(cmfd%diffcof, ng*nx*ny*nz, MPI_REAL8, 0, PETSC_COMM_WORLD, mpi_err)
-    call MPI_BCAST(cmfd%dtilde, 6*ng*nx*ny*nz, MPI_REAL8, 0, PETSC_COMM_WORLD, mpi_err)
-    call MPI_BCAST(cmfd%dhat, 6*ng*nx*ny*nz, MPI_REAL8, 0, PETSC_COMM_WORLD, mpi_err)
-    call MPI_BCAST(cmfd%hxyz, 3*nx*ny*nz, MPI_REAL8, 0, PETSC_COMM_WORLD, mpi_err)
-    call MPI_BCAST(cmfd%current, 12*ng*nx*ny*nz, MPI_REAL8, 0, PETSC_COMM_WORLD, mpi_err)
-
-    ! broadcast coremap info
-    if (cmfd_coremap) then
-      call MPI_BCAST(cmfd%coremap, nx*ny*nz, MPI_INTEGER8, 0, PETSC_COMM_WORLD, mpi_err)
-      call MPI_BCAST(cmfd%mat_dim, 1, MPI_INTEGER8, 0, PETSC_COMM_WORLD, mpi_err)
-      if (.not. allocated(cmfd % indexmap)) &
-           allocate(cmfd % indexmap(cmfd % mat_dim,3))
-      call MPI_BCAST(cmfd%indexmap, cmfd%mat_dim*3, MPI_INTEGER8, 0, PETSC_COMM_WORLD, mpi_err)
-    end if
-
-  end subroutine cmfd_bcast
-
-# endif
-
-end module cmfd_message_passing

From 171f92711e85e825fe0cca4b3dbc87fef8cf4912 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 9 Jul 2013 10:00:46 -0400
Subject: [PATCH 14/69] cleaned up comments in cmfd output

---
 src/cmfd_output.F90 | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/src/cmfd_output.F90 b/src/cmfd_output.F90
index ebfec812b0..2936dda35a 100644
--- a/src/cmfd_output.F90
+++ b/src/cmfd_output.F90
@@ -1,8 +1,6 @@
 module cmfd_output
 
-#ifdef PETSC
-
-! This modules cleans up cmfd objects and echos the results
+! This modules cleans up cmfd objects and outputs neutron balance info
 
   implicit none
   private
@@ -11,28 +9,30 @@ module cmfd_output
 contains
 
 !===============================================================================
-! FINALIZE_CMFD
+! FINALIZE_CMFD finalizes PETSc and frees memory associated with CMFD
 !===============================================================================
 
   subroutine finalize_cmfd() 
 
-    use global,      only: cmfd, cmfd_write_balance, cmfd_write_hdf5, &
+    use global,      only: cmfd, cmfd_write_balance, &
                            master, mpi_err
     use cmfd_header, only: deallocate_cmfd
 
-    ! finalize petsc
+    ! Finalize petsc
+#ifdef PETSC
     call PetscFinalize(mpi_err)
+# endif
 
-    ! write out final neutron balance
+    ! Write out final neutron balance
     if (master .and. cmfd_write_balance) call write_neutron_balance()
 
-    ! deallocate cmfd object
+    ! Deallocate cmfd object
     call deallocate_cmfd(cmfd)
 
   end subroutine finalize_cmfd 
 
 !===============================================================================
-! WRITE_NEUTRON_BALANCE 
+! WRITE_NEUTRON_BALANCE writes an ASCII neutron balance file out
 !===============================================================================
 
   subroutine write_neutron_balance()
@@ -45,17 +45,15 @@ contains
 
     filename = trim(path_output) // 'neutron_balance.out'
 
-    ! open file for output
+    ! Open file for output
     open(UNIT=CMFD_BALANCE, FILE=filename, ACTION='write')
 
-    ! write out the tally
+    ! Write out the tally
     call neutron_balance(CMFD_BALANCE) 
 
-    ! close file
+    ! Close file
     close(CMFD_BALANCE)
 
   end subroutine write_neutron_balance
 
-#endif
-
 end module cmfd_output

From 3b23097ff0eb5b5989b570703383c50262a6d200 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 9 Jul 2013 10:01:32 -0400
Subject: [PATCH 15/69] removed some useless cmfd vars and removed use mpi from
 global

---
 src/global.F90           | 59 ++++++++++++----------------------------
 src/templates/cmfd_t.xml | 40 ++++++++++++---------------
 2 files changed, 36 insertions(+), 63 deletions(-)

diff --git a/src/global.F90 b/src/global.F90
index ee9f752295..bfffb2f37b 100644
--- a/src/global.F90
+++ b/src/global.F90
@@ -16,10 +16,6 @@ module global
   use tally_header,     only: TallyObject, TallyMap, TallyResult
   use timer_header,     only: Timer
 
-#ifdef MPI
-  use mpi
-#endif
-
 #ifdef HDF5
   use hdf5_interface,  only: HID_T
 #endif
@@ -300,78 +296,59 @@ module global
   type(Timer) :: time_cmfd   ! timer for whole cmfd calculation
   type(Timer) :: time_solver ! timer for solver 
 
-  ! Flag for CMFD only
-  logical :: cmfd_only = .false.
-
-  ! Flag for coremap accelerator
+  ! Flag for active core map
   logical :: cmfd_coremap = .false.
 
-  ! number of processors for cmfd
-  integer :: n_procs_cmfd
-
-  ! reset dhats to zero
+  ! Flag to reset dhats to zero
   logical :: dhat_reset = .false.
 
-  ! activate neutronic feedback
+  ! Flag to activate neutronic feedback via source weights
   logical :: cmfd_feedback = .false.
 
-  ! activate auto-balance of tallies (2grp only)
-! logical :: cmfd_balance = .false.
+  ! User-defined tally information
+  integer :: n_cmfd_meshes  = 1 ! # of structured meshes
+  integer :: n_cmfd_tallies = 3 ! # of user-defined tallies
 
-  ! calculate effective downscatter
-! logical :: cmfd_downscatter = .false.
-
-  ! user-defined tally information
-  integer :: n_cmfd_meshes              = 1 ! # of structured meshes
-  integer :: n_cmfd_tallies             = 3 ! # of user-defined tallies
-
-  ! overwrite with 2grp xs
-  logical :: cmfd_run_2grp = .false.
-
-  ! hold cmfd weight adjustment factors
+  ! Flag to hold cmfd weight adjustment factors
   logical :: cmfd_hold_weights = .false.
 
-  ! eigenvalue solver type
+  ! Eigenvalue solver type
   character(len=10) :: cmfd_solver_type = 'power'
 
-  ! adjoint method type
+  ! Adjoint method type
   character(len=10) :: cmfd_adjoint_type = 'physical'
 
-  ! number of incomplete ilu factorization levels
+  ! Number of incomplete ilu factorization levels
   integer :: cmfd_ilu_levels = 1
 
-  ! batch to begin cmfd
+  ! Batch to begin cmfd
   integer :: cmfd_begin = 1
 
-  ! when and how long to flush cmfd tallies during inactive batches
+  ! When and how long to flush cmfd tallies during inactive batches
   integer :: cmfd_inact_flush(2) = (/9999,1/)
 
-  ! batch to last flush before active batches
+  ! Batch to last flush before active batches
   integer :: cmfd_act_flush = 0
 
-  ! compute effective downscatter cross section
+  ! Compute effective downscatter cross section
   logical :: cmfd_downscatter = .false.
 
-  ! convergence monitoring
+  ! Convergence monitoring
   logical :: cmfd_snes_monitor  = .false.
   logical :: cmfd_ksp_monitor   = .false.
   logical :: cmfd_power_monitor = .false.
 
-  ! cmfd output
+  ! Cmfd output
   logical :: cmfd_write_balance  = .false.
   logical :: cmfd_write_matrices = .false.
-  logical :: cmfd_write_hdf5     = .false.
 
-  ! run an adjoint calculation (last batch only)
+  ! Run an adjoint calculation (last batch only)
   logical :: cmfd_run_adjoint = .false.
 
-  ! cmfd run logicals
+  ! Cmfd run logicals
   logical :: cmfd_on             = .false.
   logical :: cmfd_tally_on       = .true. 
 
-  ! tolerance on keff to run cmfd
-  real(8) :: cmfd_keff_tol = 0.005_8
-
   ! Information about state points to be written
   integer :: n_state_points = 0
   type(SetInt) :: statepoint_batch
diff --git a/src/templates/cmfd_t.xml b/src/templates/cmfd_t.xml
index c9ae3c1fdc..631f2dd44d 100644
--- a/src/templates/cmfd_t.xml
+++ b/src/templates/cmfd_t.xml
@@ -13,27 +13,23 @@
     <component name="energy"      type="double-array"  />
   </typedef>
 
-  <variable name="mesh_"           tag="mesh"           type="mesh_xml"                                     />
-  <variable name="norm_"           tag="norm"           type="double"        default="1.0"                  />
-  <variable name="feedback_"       tag="feedback"       type="word"          length="5"                     />
-  <variable name="n_cmfd_procs_"   tag="n_procs"        type="integer"       default="1"                    />
-  <variable name="reset_"          tag="reset"          type="word"          length="5"                     />
-  <variable name="balance_"        tag="balance"        type="word"          length="5"                     />
-  <variable name="downscatter_"    tag="downscatter"    type="word"          length="5"                     />
-  <variable name="run_2grp_"       tag="run_2grp"       type="word"          length="5"                     />
-  <variable name="solver_"         tag="solver"         type="word"          default="'power'" length="250" />
-  <variable name="snes_monitor_"   tag="snes_monitor"   type="word"          length="5"                     />
-  <variable name="ksp_monitor_"    tag="ksp_monitor"    type="word"          length="5"                     />
-  <variable name="power_monitor_"  tag="power_monitor"  type="word"          length="5"                     />
-  <variable name="write_balance_"  tag="write_balance"  type="word"          length="5"                     />
-  <variable name="write_matrices_" tag="write_matrices" type="word"          length="5"                     />
-  <variable name="run_adjoint_"    tag="run_adjoint"    type="word"          length="5"                     />
-  <variable name="write_hdf5_"     tag="write_hdf5"     type="word"          length="5"                     />
-  <variable name="begin_"          tag="begin"          type="integer"       default="1"                    />
-  <variable name="inactive_"       tag="inactive"       type="word"          length="5"                     />
-  <variable name="active_flush_"   tag="active_flush"   type="integer"       default="0"                    />
-  <variable name="keff_tol_"       tag="keff_tol"       type="double"        default="0.005"                />
-  <variable name="inactive_flush_" tag="inactive_flush" type="integer"       default="9999"                 />
-  <variable name="num_flushes_"    tag="num_flushes"    type="integer"       default="1"                    />
+  <variable name="mesh_"           tag="mesh"           type="mesh_xml"                                    />
+  <variable name="norm_"           tag="norm"           type="double"        default="1.0"                 />
+  <variable name="feedback_"       tag="feedback"       type="word"          length="5"                    />
+  <variable name="reset_"          tag="reset"          type="word"          length="5"                    />
+  <variable name="balance_"        tag="balance"        type="word"          length="5"                    />
+  <variable name="downscatter_"    tag="downscatter"    type="word"          length="5"                    />
+  <variable name="solver_"         tag="solver"         type="word"          default="'power'" length="25" />
+  <variable name="snes_monitor_"   tag="snes_monitor"   type="word"          length="5"                    />
+  <variable name="ksp_monitor_"    tag="ksp_monitor"    type="word"          length="5"                    />
+  <variable name="power_monitor_"  tag="power_monitor"  type="word"          length="5"                    />
+  <variable name="write_balance_"  tag="write_balance"  type="word"          length="5"                    />
+  <variable name="write_matrices_" tag="write_matrices" type="word"          length="5"                    />
+  <variable name="run_adjoint_"    tag="run_adjoint"    type="word"          length="5"                    />
+  <variable name="begin_"          tag="begin"          type="integer"       default="1"                   />
+  <variable name="inactive_"       tag="inactive"       type="word"          length="5"                    />
+  <variable name="active_flush_"   tag="active_flush"   type="integer"       default="0"                   />
+  <variable name="inactive_flush_" tag="inactive_flush" type="integer"       default="9999"                />
+  <variable name="num_flushes_"    tag="num_flushes"    type="integer"       default="1"                   />
 
 </template>

From cd859cdaa31b73ffb570a4619bee716f13588d50 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 9 Jul 2013 10:44:32 -0400
Subject: [PATCH 16/69] added cmfd timers and output them at end of run

---
 src/cmfd_execute.F90      |  4 ----
 src/cmfd_input.F90        |  3 ++-
 src/cmfd_power_solver.F90 | 10 +++++++++-
 src/global.F90            |  5 +++--
 src/output.F90            |  6 +++++-
 5 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/src/cmfd_execute.F90 b/src/cmfd_execute.F90
index 8f94bbe18a..b2762ee236 100644
--- a/src/cmfd_execute.F90
+++ b/src/cmfd_execute.F90
@@ -37,7 +37,6 @@ contains
       call process_cmfd_options()
 
       ! Call solver
-      call time_solver % start()
       if (trim(cmfd_solver_type) == 'power') then
         call cmfd_power_execute()
       elseif (trim(cmfd_solver_type) == 'jfnk') then
@@ -46,10 +45,8 @@ contains
         message = 'solver type became invalid after input processing'
         call fatal_error() 
       end if
-      call time_solver % stop()
 
       ! check to perform adjoint on last batch
-      call time_solver % start()
       if (current_batch == n_batches .and. cmfd_run_adjoint) then
         if (trim(cmfd_solver_type) == 'power') then
           call cmfd_power_execute(adjoint = .true.)
@@ -57,7 +54,6 @@ contains
           call cmfd_snes_execute(adjoint = .true.)
         end if
       end if
-      call time_solver % stop()
 
     end if
 
diff --git a/src/cmfd_input.F90 b/src/cmfd_input.F90
index d251ec379a..e98eaac762 100644
--- a/src/cmfd_input.F90
+++ b/src/cmfd_input.F90
@@ -46,7 +46,8 @@ contains
 
     ! initialize timers
     call time_cmfd % reset()
-    call time_solver % reset()
+    call time_cmfdbuild % reset()
+    call time_cmfdsolve % reset()
 
   end subroutine configure_cmfd
 
diff --git a/src/cmfd_power_solver.F90 b/src/cmfd_power_solver.F90
index 0a258aac13..92bd77bcc7 100644
--- a/src/cmfd_power_solver.F90
+++ b/src/cmfd_power_solver.F90
@@ -34,7 +34,7 @@ contains
 
   subroutine cmfd_power_execute(k_tol, s_tol, adjoint)
 
-    use global,  only: cmfd_adjoint_type
+    use global,  only: cmfd_adjoint_type, time_cmfdbuild, time_cmfdsolve
 
     real(8), optional :: k_tol    ! tolerance on keff
     real(8), optional :: s_tol    ! tolerance on source
@@ -53,6 +53,9 @@ contains
     if (adjoint_calc .and. trim(cmfd_adjoint_type) == 'physical') &
         physical_adjoint = .true.
 
+    ! start timer for build
+    call time_cmfdbuild % start()
+
     ! initialize solver
     call gmres % create()
 
@@ -69,8 +72,13 @@ contains
     ! precondition matrix
     call gmres % precondition(loss % petsc_mat)
 
+    ! stop timer for build
+    call time_cmfdbuild % stop()
+
     ! begin power iteration 
+    call time_cmfdsolve % start()
     call execute_power_iter()
+    call time_cmfdsolve % stop()
 
     ! extract results
     call extract_results()
diff --git a/src/global.F90 b/src/global.F90
index bfffb2f37b..14b2c066e1 100644
--- a/src/global.F90
+++ b/src/global.F90
@@ -293,8 +293,9 @@ module global
   logical :: cmfd_run = .false.
  
   ! Timing objects
-  type(Timer) :: time_cmfd   ! timer for whole cmfd calculation
-  type(Timer) :: time_solver ! timer for solver 
+  type(Timer) :: time_cmfd      ! timer for whole cmfd calculation
+  type(Timer) :: time_cmfdbuild ! timer for matrix build
+  type(Timer) :: time_cmfdsolve ! timer for solver 
 
   ! Flag for active core map
   logical :: cmfd_coremap = .false.
diff --git a/src/output.F90 b/src/output.F90
index f19148fb02..728f845a70 100644
--- a/src/output.F90
+++ b/src/output.F90
@@ -1396,13 +1396,17 @@ contains
     write(ou,100) "Total time in simulation", time_inactive % elapsed + &
          time_active % elapsed
     write(ou,100) "  Time in transport only", time_transport % elapsed
-    if(cmfd_run) write(ou,100) "Total CMFD time", time_cmfd % elapsed
     write(ou,100) "  Time in inactive batches", time_inactive % elapsed
     write(ou,100) "  Time in active batches", time_active % elapsed
     write(ou,100) "  Time synchronizing fission bank", time_bank % elapsed
     write(ou,100) "    Sampling source sites", time_bank_sample % elapsed
     write(ou,100) "    SEND/RECV source sites", time_bank_sendrecv % elapsed
     write(ou,100) "  Time accumulating tallies", time_tallies % elapsed
+    if (cmfd_run) write(ou,100) "  Total CMFD time", time_cmfd % elapsed
+    if (cmfd_run) write(ou,100) "    Time building matrices", &
+                  time_cmfdbuild % elapsed
+    if (cmfd_run) write(ou,100) "    Time solving matrices", &
+                  time_cmfdsolve % elapsed
     write(ou,100) "Total time for finalization", time_finalize % elapsed
     write(ou,100) "Total time elapsed", time_total % elapsed
 

From 00101e39d95f56feb5f078fadc9b44fb1b085b7c Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 9 Jul 2013 10:48:07 -0400
Subject: [PATCH 17/69] changed CMFD timer output to match convention

---
 src/output.F90 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/output.F90 b/src/output.F90
index 728f845a70..c6810dc083 100644
--- a/src/output.F90
+++ b/src/output.F90
@@ -1402,10 +1402,10 @@ contains
     write(ou,100) "    Sampling source sites", time_bank_sample % elapsed
     write(ou,100) "    SEND/RECV source sites", time_bank_sendrecv % elapsed
     write(ou,100) "  Time accumulating tallies", time_tallies % elapsed
-    if (cmfd_run) write(ou,100) "  Total CMFD time", time_cmfd % elapsed
-    if (cmfd_run) write(ou,100) "    Time building matrices", &
+    if (cmfd_run) write(ou,100) "  Time in CMFD", time_cmfd % elapsed
+    if (cmfd_run) write(ou,100) "    Building matrices", &
                   time_cmfdbuild % elapsed
-    if (cmfd_run) write(ou,100) "    Time solving matrices", &
+    if (cmfd_run) write(ou,100) "    Solving matrices", &
                   time_cmfdsolve % elapsed
     write(ou,100) "Total time for finalization", time_finalize % elapsed
     write(ou,100) "Total time elapsed", time_total % elapsed

From 11dad806ffc6d6903cc43976568932c8fd3539b0 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 9 Jul 2013 11:04:36 -0400
Subject: [PATCH 18/69] fixed comments in these modules

---
 src/cmfd_header.F90        |  56 +++++++--------
 src/cmfd_loss_operator.F90 | 136 ++++++++++++++++++-------------------
 src/cmfd_power_solver.F90  |  52 +++++++-------
 src/cmfd_prod_operator.F90 |  10 +--
 4 files changed, 127 insertions(+), 127 deletions(-)

diff --git a/src/cmfd_header.F90 b/src/cmfd_header.F90
index c4dad04f2a..027172ee27 100644
--- a/src/cmfd_header.F90
+++ b/src/cmfd_header.F90
@@ -8,67 +8,67 @@ module cmfd_header
 
   type, public :: cmfd_type
 
-    ! indices for problem
+    ! Indices for problem
     integer :: indices(4)
 
-    ! albedo boundary condition
+    ! Albedo boundary condition
     real(8) :: albedo(6)
 
-    ! core overlay map
+    ! Core overlay map
     integer, allocatable :: coremap(:,:,:)
     integer, allocatable :: indexmap(:,:)
     integer :: mat_dim = CMFD_NOACCEL 
 
-    ! energy grid
+    ! Energy grid
     real(8), allocatable :: egrid(:)
 
-    ! cross sections
+    ! Cross sections
     real(8), allocatable :: totalxs(:,:,:,:)
     real(8), allocatable :: p1scattxs(:,:,:,:)
     real(8), allocatable :: scattxs(:,:,:,:,:)
     real(8), allocatable :: nfissxs(:,:,:,:,:)
 
-    ! diffusion coefficient
+    ! Diffusion coefficient
     real(8), allocatable :: diffcof(:,:,:,:)
 
-    ! current
+    ! Current
     real(8), allocatable :: current(:,:,:,:,:)
 
-    ! flux
+    ! Flux
     real(8), allocatable :: flux(:,:,:,:)
 
-    ! coupling coefficients and equivalence parameters
+    ! Coupling coefficients and equivalence parameters
     real(8), allocatable :: dtilde(:,:,:,:,:)
     real(8), allocatable :: dhat(:,:,:,:,:)
 
-    ! dimensions of mesh cells ([hu,hv,hw],xloc,yloc,zloc)
+    ! Dimensions of mesh cells ([hu,hv,hw],xloc,yloc,zloc)
     real(8), allocatable :: hxyz(:,:,:,:)
 
-    ! source distributions
+    ! Source distributions
     real(8), allocatable :: cmfd_src(:,:,:,:)
     real(8), allocatable :: openmc_src(:,:,:,:)
 
-    ! source sites in each mesh box
+    ! Source sites in each mesh box
     real(8), allocatable :: sourcecounts(:,:,:,:)
 
-    ! weight adjustment factors 
+    ! Weight adjustment factors 
     real(8), allocatable :: weightfactors(:,:,:,:)
 
-    ! eigenvector/eigenvalue from cmfd run
+    ! Eigenvector/eigenvalue from cmfd run
     real(8), allocatable :: phi(:)
     real(8) :: keff = ZERO
 
-    ! eigenvector/eigenvalue from adjoint run
+    ! Eigenvector/eigenvalue from adjoint run
     real(8), allocatable :: adj_phi(:)
     real(8) :: adj_keff = ZERO
 
-    ! residual for neutron balance
+    ! Residual for neutron balance
     real(8), allocatable :: resnb(:,:,:,:)
 
-    ! openmc source normalization factor
+    ! Openmc source normalization factor
     real(8) :: norm = ONE
 
-    ! Shannon entropy from cmfd fission source
+    ! "Shannon entropy" from cmfd fission source
     real(8) :: entropy 
 
   end type cmfd_type
@@ -76,7 +76,7 @@ module cmfd_header
 contains
 
 !==============================================================================
-! ALLOCATE_CMFD
+! ALLOCATE_CMFD allocates all data in of cmfd type
 !==============================================================================
 
   subroutine allocate_cmfd(this)
@@ -88,13 +88,13 @@ contains
     integer :: nz  ! number of mesh cells in z direction
     integer :: ng  ! number of energy groups
 
-   ! extract spatial and energy indices from object
+   ! Extract spatial and energy indices from object
     nx = this % indices(1)
     ny = this % indices(2)
     nz = this % indices(3)
     ng = this % indices(4)
 
-    ! allocate flux, cross sections and diffusion coefficient
+    ! Allocate flux, cross sections and diffusion coefficient
     if (.not. allocated(this % flux))       allocate(this % flux(ng,nx,ny,nz))
     if (.not. allocated(this % totalxs))    allocate(this % totalxs(ng,nx,ny,nz))
     if (.not. allocated(this % p1scattxs))  allocate(this % p1scattxs(ng,nx,ny,nz))
@@ -102,25 +102,25 @@ contains
     if (.not. allocated(this % nfissxs))    allocate(this % nfissxs(ng,ng,nx,ny,nz))
     if (.not. allocated(this % diffcof))    allocate(this % diffcof(ng,nx,ny,nz))
 
-    ! allocate dtilde and dhat 
+    ! Allocate dtilde and dhat 
     if (.not. allocated(this % dtilde))     allocate(this % dtilde(6,ng,nx,ny,nz))
     if (.not. allocated(this % dhat))       allocate(this % dhat(6,ng,nx,ny,nz))
 
-    ! allocate dimensions for each box (here for general case)
+    ! Allocate dimensions for each box (here for general case)
     if (.not. allocated(this % hxyz))       allocate(this % hxyz(3,nx,ny,nz))
 
-    ! allocate surface currents
+    ! Allocate surface currents
     if (.not. allocated(this % current))    allocate(this % current(12,ng,nx,ny,nz))
 
-    ! allocate source distributions
+    ! Allocate source distributions
     if (.not. allocated(this % cmfd_src)) allocate(this % cmfd_src(ng,nx,ny,nz))
     if (.not. allocated(this % openmc_src)) allocate(this % openmc_src(ng,nx,ny,nz))
 
-    ! allocate source weight modification vars
+    ! Allocate source weight modification vars
     if (.not. allocated(this % sourcecounts)) allocate(this % sourcecounts(ng,nx,ny,nz))
     if (.not. allocated(this % weightfactors)) allocate(this % weightfactors(ng,nx,ny,nz))
 
-    ! set everthing to 0 except weight multiply factors if feedback isnt on
+    ! Set everthing to 0 except weight multiply factors if feedback isnt on
     this % flux          = ZERO
     this % totalxs       = ZERO
     this % p1scattxs     = ZERO
@@ -139,7 +139,7 @@ contains
   end subroutine allocate_cmfd
 
 !===============================================================================
-! DEALLOCATE_CMFD 
+! DEALLOCATE_CMFD frees all memory of cmfd type 
 !===============================================================================
 
   subroutine deallocate_cmfd(this)
diff --git a/src/cmfd_loss_operator.F90 b/src/cmfd_loss_operator.F90
index 67551f78b9..22fa8dfa14 100644
--- a/src/cmfd_loss_operator.F90
+++ b/src/cmfd_loss_operator.F90
@@ -11,7 +11,7 @@ module cmfd_loss_operator
 contains
 
 !===============================================================================
-! INIT_LOSS_MATRIX
+! INIT_LOSS_MATRIX preallocates loss matrix and initializes it
 !===============================================================================
 
   subroutine init_loss_matrix(loss_matrix)
@@ -31,20 +31,20 @@ contains
     integer :: nz_s ! number of non-zero side cells
     integer :: nz_i ! number of non-zero interior cells
 
-    ! get maximum number of cells in each direction
+    ! Get maximum number of cells in each direction
     nx = cmfd%indices(1)
     ny = cmfd%indices(2)
     nz = cmfd%indices(3)
     ng = cmfd%indices(4)
 
-    ! calculate dimensions of matrix
+    ! Calculate dimensions of matrix
     if (cmfd_coremap) then
       n = cmfd % mat_dim * ng
     else
       n = nx*ny*nz*ng
     end if
 
-    ! calculate number of nonzeros, if core map -> need to determine manually
+    ! Calculate number of nonzeros, if core map -> need to determine manually
     if (cmfd_coremap) then
       nnz = preallocate_loss_matrix(nx, ny, nz, ng, n)
     else ! structured Cartesian grid
@@ -57,13 +57,13 @@ contains
       nnz  = nz_c + nz_s + nz_i
     end if
 
-    ! configure loss matrix
+    ! Configure loss matrix
     call loss_matrix % create(n, nnz)
 
   end subroutine init_loss_matrix
 
 !===============================================================================
-! PREALLOCATE_LOSS_MATRIX
+! PREALLOCATE_LOSS_MATRIX manually preallocates the loss matrix
 !===============================================================================
 
   function preallocate_loss_matrix(nx, ny, nz, ng, n) result(nnz)
@@ -91,64 +91,64 @@ contains
     integer :: neig_mat_idx  ! matrix index of neighbor cell
     integer :: scatt_mat_idx ! matrix index for h-->g scattering terms
 
-    ! reset to 0
+    ! Reset number of nonzeros to 0
     nnz = 0
 
-    ! create single vector of these indices for boundary calculation
+    ! Create single vector of these indices for boundary calculation
     nxyz(1,:) = (/1,nx/)
     nxyz(2,:) = (/1,ny/)
     nxyz(3,:) = (/1,nz/)
 
-    ! begin loop around local rows
+    ! Begin loop around local rows
     ROWS: do irow = 1, n
 
-      ! set a nonzero for diagonal
+      ! Set a nonzero for diagonal
       nnz = nnz + 1
 
-      ! get location indices
+      ! Get location indices
       call matrix_to_indices(irow, g, i, j, k, ng, nx, ny, nz)
 
-      ! create boundary vector
+      ! Create boundary vector
       bound = (/i,i,j,j,k,k/)
 
-      ! begin loop over leakages
+      ! Begin loop over leakages
       LEAK: do l = 1,6
 
-        ! define (x,y,z) and (-,+) indices
+        ! Define (x,y,z) and (-,+) indices
         xyz_idx = int(ceiling(real(l)/real(2)))  ! x=1, y=2, z=3
         dir_idx = 2 - mod(l,2) ! -=1, +=2
 
-        ! calculate spatial indices of neighbor
+        ! Calculate spatial indices of neighbor
         neig_idx = (/i,j,k/)                ! begin with i,j,k
         shift_idx = -2*mod(l,2) +1          ! shift neig by -1 or +1
         neig_idx(xyz_idx) = shift_idx + neig_idx(xyz_idx)
 
-        ! check for global boundary
+        ! Check for global boundary
         if (bound(l) /= nxyz(xyz_idx,dir_idx)) then
 
-          ! check for coremap 
+          ! Check for coremap 
           if (cmfd_coremap) then
 
-            ! check for neighbor that is non-acceleartred
+            ! Check for neighbor that is non-acceleartred
             if (cmfd % coremap(neig_idx(1),neig_idx(2),neig_idx(3)) /= &
                  CMFD_NOACCEL) then
 
-              ! get neighbor matrix index
+              ! Get neighbor matrix index
               call indices_to_matrix(g,neig_idx(1), neig_idx(2), & 
                    neig_idx(3), neig_mat_idx, ng, nx, ny)
 
-              ! record nonzero
+              ! Record nonzero
               nnz = nnz + 1 
 
             end if
 
           else
 
-            ! get neighbor matrix index
+            ! Get neighbor matrix index
             call indices_to_matrix(g, neig_idx(1), neig_idx(2), neig_idx(3), &
                  neig_mat_idx, ng, nx, ny)
 
-            ! record nonzero
+            ! Record nonzero
             nnz = nnz + 1
 
           end if
@@ -157,16 +157,16 @@ contains
 
       end do LEAK
 
-      ! begin loop over off diagonal in-scattering
+      ! Begin loop over off diagonal in-scattering
       SCATTR: do h = 1, ng
 
-        ! cycle though if h=g
+        ! Cycle though if h=g, it was already banked in removal xs
         if (h == g) cycle
 
-        ! get neighbor matrix index
+        ! Get neighbor matrix index
         call indices_to_matrix(h, i, j, k, scatt_mat_idx, ng, nx, ny)
 
-        ! record nonzero
+        ! Record nonzero
         nnz = nnz + 1
 
       end do SCATTR
@@ -215,140 +215,140 @@ contains
     real(8) :: jnet                 ! net leakage from jo
     real(8) :: val                  ! temporary variable before saving to 
 
-    ! check for adjoint
+    ! Check for adjoint
     adjoint_calc = .false.
     if (present(adjoint)) adjoint_calc = adjoint 
 
-    ! get maximum number of cells in each direction
+    ! Get maximum number of cells in each direction
     nx = cmfd%indices(1)
     ny = cmfd%indices(2)
     nz = cmfd%indices(3)
     ng = cmfd%indices(4)
 
-    ! create single vector of these indices for boundary calculation
+    ! Create single vector of these indices for boundary calculation
     nxyz(1,:) = (/1,nx/)
     nxyz(2,:) = (/1,ny/)
     nxyz(3,:) = (/1,nz/)
 
-    ! begin iteration loops
+    ! Begin iteration loops
     ROWS: do irow = 1, loss_matrix % n
 
-      ! set up a new row in matrix
+      ! Set up a new row in matrix
       call loss_matrix % new_row()
 
-      ! get indices for that row
+      ! Get indices for that row
       call matrix_to_indices(irow, g, i, j, k, ng, nx, ny, nz)
 
-      ! retrieve cell data
+      ! Retrieve cell data
       totxs = cmfd%totalxs(g,i,j,k)
       scattxsgg = cmfd%scattxs(g,g,i,j,k)
       dtilde = cmfd%dtilde(:,g,i,j,k)
       hxyz = cmfd%hxyz(:,i,j,k)
 
-      ! check and get dhat
+      ! Check and get dhat
       if (allocated(cmfd%dhat)) then
         dhat = cmfd%dhat(:,g,i,j,k)
       else
         dhat = ZERO
       end if
 
-      ! create boundary vector 
+      ! Create boundary vector 
       bound = (/i,i,j,j,k,k/)
 
-      ! begin loop over leakages
+      ! Begin loop over leakages
       ! 1=-x, 2=+x, 3=-y, 4=+y, 5=-z, 6=+z 
       LEAK: do l = 1,6
 
-        ! define (x,y,z) and (-,+) indices
+        ! Define (x,y,z) and (-,+) indices
         xyz_idx = int(ceiling(real(l)/real(2)))  ! x=1, y=2, z=3
         dir_idx = 2 - mod(l,2) ! -=1, +=2
 
-        ! calculate spatial indices of neighbor
+        ! Calculate spatial indices of neighbor
         neig_idx = (/i,j,k/)                ! begin with i,j,k
         shift_idx = -2*mod(l,2) +1          ! shift neig by -1 or +1
         neig_idx(xyz_idx) = shift_idx + neig_idx(xyz_idx)
 
-        ! check for global boundary
+        ! Check for global boundary
         if (bound(l) /= nxyz(xyz_idx,dir_idx)) then
 
-          ! check for core map
+          ! Check for core map
           if (cmfd_coremap) then
 
-            ! check that neighbor is not reflector
+            ! Check that neighbor is not reflector
             if (cmfd % coremap(neig_idx(1),neig_idx(2),neig_idx(3)) /= &
                  CMFD_NOACCEL) then
 
-              ! compute leakage coefficient for neighbor
+              ! Compute leakage coefficient for neighbor
               jn = -dtilde(l) + shift_idx*dhat(l)
 
-              ! get neighbor matrix index
+              ! Get neighbor matrix index
               call indices_to_matrix(g, neig_idx(1), neig_idx(2), neig_idx(3), &
                    neig_mat_idx, ng, nx, ny)
 
-              ! compute value and record to bank
+              ! Compute value and record to bank
               val = jn/hxyz(xyz_idx)
 
-              ! record value in matrix
+              ! Record value in matrix
               call loss_matrix % add_value(neig_mat_idx, val)
 
             end if
 
           else
 
-            ! compute leakage coefficient for neighbor
+            ! Compute leakage coefficient for neighbor
             jn = -dtilde(l) + shift_idx*dhat(l)
 
-            ! get neighbor matrix index
+            ! Get neighbor matrix index
             call indices_to_matrix(g, neig_idx(1), neig_idx(2), neig_idx(3), &
                  neig_mat_idx, ng, nx, ny)
 
-            ! compute value and record to bank
+            ! Compute value and record to bank
             val = jn/hxyz(xyz_idx)
 
-            ! record value in matrix
+            ! Record value in matrix
             call loss_matrix % add_value(neig_mat_idx, val)
 
           end if
 
         end if
 
-        ! compute leakage coefficient for target
+        ! Compute leakage coefficient for target
         jo(l) = shift_idx*dtilde(l) + dhat(l)
 
       end do LEAK
 
-      ! calate net leakage coefficient for target
+      ! Calculate net leakage coefficient for target
       jnet = (jo(2) - jo(1))/hxyz(1) + (jo(4) - jo(3))/hxyz(2) + &
            (jo(6) - jo(5))/hxyz(3)
 
-      ! calculate loss of neutrons
+      ! Calculate loss of neutrons
       val = jnet + totxs - scattxsgg
 
-      ! record diagonal term
+      ! Record diagonal term
       call loss_matrix % add_value(irow, val)
 
-      ! begin loop over off diagonal in-scattering
+      ! Begin loop over off diagonal in-scattering
       SCATTR: do h = 1, ng
 
-        ! cycle though if h=g
+        ! Cycle though if h=g, value already banked in removal xs
         if (h == g) cycle
 
-        ! get neighbor matrix index
+        ! Get neighbor matrix index
         call indices_to_matrix(h, i, j, k, scatt_mat_idx, ng, nx, ny)
 
-        ! check for adjoint
+        ! Check for adjoint
         if (adjoint_calc) then
-          ! get scattering macro xs, transposed!
+          ! Get scattering macro xs, transposed!
           scattxshg = cmfd%scattxs(g, h, i, j, k)
         else
-          ! get scattering macro xs
+          ! Get scattering macro xs
           scattxshg = cmfd%scattxs(h, g, i, j, k)
         end if
 
-        ! negate the scattering xs 
+        ! Negate the scattering xs 
         val = -scattxshg
 
-        ! record value in matrix
+        ! Record value in matrix
         call loss_matrix % add_value(scatt_mat_idx, val)
 
       end do SCATTR
@@ -375,15 +375,15 @@ contains
     integer :: ny     ! maximum number of y cells
     integer :: ng     ! maximum number of groups
 
-    ! check if coremap is used
+    ! Check if coremap is used
     if (cmfd_coremap) then
 
-      ! get idx from core map
+      ! Get idx from core map
       matidx = ng*(cmfd % coremap(i,j,k)) - (ng - g)
 
     else
 
-      ! compute index
+      ! Compute index
       matidx = g + ng*(i - 1) + ng*nx*(j - 1) + ng*nx*ny*(k - 1)
 
     end if
@@ -391,7 +391,7 @@ contains
   end subroutine indices_to_matrix
 
 !===============================================================================
-! MATRIX_TO_INDICES
+! MATRIX_TO_INDICES converts a matrix index to spatial and group indicies
 !===============================================================================
 
   subroutine matrix_to_indices(irow, g, i, j, k, ng, nx, ny, nz)
@@ -406,10 +406,10 @@ contains
     integer :: nz    ! maximum number of z cells
     integer :: ng    ! maximum number of groups
 
-    ! check for core map
+    ! Check for core map
     if (cmfd_coremap) then
 
-      ! get indices from indexmap
+      ! Get indices from indexmap
       g = mod(irow, ng) + 1
       i = cmfd % indexmap((irow-1)/ng+1,1)
       j = cmfd % indexmap((irow-1)/ng+1,2)
@@ -417,7 +417,7 @@ contains
 
     else
 
-      ! compute indices
+      ! Compute indices
       g = mod(irow-1, ng) + 1
       i = mod(irow-1, ng*nx)/ng + 1
       j = mod(irow-1, ng*nx*ny)/(ng*nx)+ 1
diff --git a/src/cmfd_power_solver.F90 b/src/cmfd_power_solver.F90
index 92bd77bcc7..6de504f5a3 100644
--- a/src/cmfd_power_solver.F90
+++ b/src/cmfd_power_solver.F90
@@ -29,7 +29,7 @@ module cmfd_power_solver
 contains
 
 !===============================================================================
-! CMFD_POWER_EXECUTE
+! CMFD_POWER_EXECUTE sets up and runs power iteration solver for CMFD
 !===============================================================================
 
   subroutine cmfd_power_execute(k_tol, s_tol, adjoint)
@@ -42,48 +42,48 @@ contains
 
     logical :: physical_adjoint = .false.
 
-    ! set tolerances if present
+    ! Set tolerances if present
     if (present(k_tol)) ktol = k_tol
     if (present(s_tol)) stol = s_tol
 
-    ! check for adjoint execution
+    ! Check for adjoint execution
     if (present(adjoint)) adjoint_calc = adjoint
 
-    ! check for physical adjoint
+    ! Check for physical adjoint
     if (adjoint_calc .and. trim(cmfd_adjoint_type) == 'physical') &
         physical_adjoint = .true.
 
-    ! start timer for build
+    ! Start timer for build
     call time_cmfdbuild % start()
 
-    ! initialize solver
+    ! Initialize solver
     call gmres % create()
 
-    ! initialize matrices and vectors
+    ! Initialize matrices and vectors
     call init_data(physical_adjoint)
 
-    ! check for adjoint calculation
+    ! Check for adjoint calculation
     if (adjoint_calc .and. trim(cmfd_adjoint_type) == 'math') &
         call compute_adjoint()
 
-    ! set up krylov info
+    ! Set up krylov info
     call gmres % set_oper(loss % petsc_mat, loss % petsc_mat)
 
-    ! precondition matrix
+    ! Precondition matrix
     call gmres % precondition(loss % petsc_mat)
 
-    ! stop timer for build
+    ! Stop timer for build
     call time_cmfdbuild % stop()
 
-    ! begin power iteration 
+    ! Begin power iteration 
     call time_cmfdsolve % start()
     call execute_power_iter()
     call time_cmfdsolve % stop()
 
-    ! extract results
+    ! Extract results
     call extract_results()
 
-    ! deallocate petsc objects
+    ! Deallocate petsc objects
     call finalize()
 
   end subroutine cmfd_power_execute
@@ -101,35 +101,35 @@ contains
     integer :: n      ! problem size
     real(8) :: guess  ! initial guess
 
-    ! set up matrices
+    ! Set up matrices
     call init_loss_matrix(loss)
     call init_prod_matrix(prod)
 
-    ! get problem size
+    ! Get problem size
     n = loss % n
 
-    ! set up flux vectors
+    ! Set up flux vectors
     call phi_n % create(n)
     call phi_o % create(n)
 
-    ! set up source vectors
+    ! Set up source vectors
     call S_n % create(n)
     call S_o % create(n)
 
-    ! set initial guess
+    ! Set initial guess
     guess = ONE
     phi_n % val = guess
     phi_o % val = guess
     k_n = guess
     k_o = guess
 
-    ! set up loss matrix
+    ! Set up loss matrix
     call build_loss_matrix(loss, adjoint=adjoint) 
 
-    ! set up production matrix
+    ! Set up production matrix
     call build_prod_matrix(prod, adjoint=adjoint)
 
-    ! setup petsc for everything
+    ! Setup petsc for everything
     call loss % assemble()
     call prod % assemble()
     call loss % setup_petsc()
@@ -142,7 +142,7 @@ contains
   end subroutine init_data
 
 !===============================================================================
-! COMPUTE_ADJOINT 
+! COMPUTE_ADJOINT computes a mathematical adjoint of CMFD problem 
 !===============================================================================
 
   subroutine compute_adjoint()
@@ -209,7 +209,7 @@ contains
   end subroutine execute_power_iter 
 
 !===============================================================================
-! CONVERGENCE
+! CONVERGENCE checks the convergence of the CMFD problem
 !===============================================================================
 
   subroutine convergence(iter)
@@ -244,7 +244,7 @@ contains
   end subroutine convergence
 
 !===============================================================================
-! EXTRACT_RESULTS 
+! EXTRACT_RESULTS takes results and puts them in CMFD global data object
 !===============================================================================
 
   subroutine extract_results()
@@ -298,7 +298,7 @@ contains
   end subroutine extract_results
 
 !===============================================================================
-! FINALIZE 
+! FINALIZE frees all memory associated with power iteration
 !===============================================================================
 
   subroutine finalize()
diff --git a/src/cmfd_prod_operator.F90 b/src/cmfd_prod_operator.F90
index 9e017013f8..4c6a28fb88 100644
--- a/src/cmfd_prod_operator.F90
+++ b/src/cmfd_prod_operator.F90
@@ -11,7 +11,7 @@ module cmfd_prod_operator
 contains
 
 !==============================================================================
-! INIT_PROD_MATRIX
+! INIT_PROD_MATRIX preallocates prod matrix and initializes it
 !==============================================================================
 
   subroutine init_prod_matrix(prod_matrix)
@@ -25,13 +25,13 @@ contains
     integer :: n    ! total length of matrix
     integer :: nnz  ! number of nonzeros in matrix
 
-    ! get maximum number of cells in each direction
+    ! Get maximum number of cells in each direction
     nx = cmfd%indices(1)
     ny = cmfd%indices(2)
     nz = cmfd%indices(3)
     ng = cmfd%indices(4)
 
-    ! calculate dimensions and number of nonzeros in matrix
+    ! Calculate dimensions and number of nonzeros in matrix
     if (cmfd_coremap) then
       n = cmfd % mat_dim * ng
     else
@@ -39,7 +39,7 @@ contains
     end if
     nnz = n * ng
 
-    ! configure prod matrix
+    ! Configure prod matrix
     call prod_matrix % create(n, nnz)
 
   end subroutine init_prod_matrix
@@ -160,7 +160,7 @@ contains
   end subroutine indices_to_matrix
 
 !===============================================================================
-! MATRIX_TO_INDICES
+! MATRIX_TO_INDICES converts matrix index to spatial and group indices
 !===============================================================================
 
   subroutine matrix_to_indices(irow, g, i, j, k, ng, nx, ny, nz)

From decf2e8b8aec912f975f8fde28b5fc26b708a87e Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 9 Jul 2013 11:54:08 -0400
Subject: [PATCH 19/69] added compiler flags so that code compiles without
 petsc

---
 src/DEPENDENCIES          |  9 +++++++--
 src/cmfd_execute.F90      |  4 ----
 src/cmfd_input.F90        |  2 +-
 src/cmfd_power_solver.F90 | 10 +++++-----
 src/matrix_header.F90     | 25 +++++++++++++++----------
 src/petsc_solver.F90      | 37 +++++++++++++++++++++++++++----------
 src/vector_header.F90     |  9 ++++-----
 7 files changed, 59 insertions(+), 37 deletions(-)

diff --git a/src/DEPENDENCIES b/src/DEPENDENCIES
index 433ed14828..0de6323f20 100644
--- a/src/DEPENDENCIES
+++ b/src/DEPENDENCIES
@@ -11,11 +11,14 @@ cmfd_output.o: cmfd_header.o
 cmfd_output.o: constants.o
 cmfd_output.o: global.o
 
-list_header.o: constants.o
-
 ace_header.o: constants.o
 ace_header.o: endf_header.o
 
+list_header.o: constants.o
+
+petsc_solver.o: matrix_header.o
+petsc_solver.o: vector_header.o
+
 cmfd_loss_operator.o: constants.o
 cmfd_loss_operator.o: global.o
 cmfd_loss_operator.o: matrix_header.o
@@ -118,6 +121,8 @@ cmfd_input.o: tally_header.o
 cmfd_input.o: tally_initialize.o
 cmfd_input.o: templates/cmfd_t.o
 
+matrix_header.o: vector_header.o
+
 input_xml.o: cmfd_input.o
 input_xml.o: constants.o
 input_xml.o: dict_header.o
diff --git a/src/cmfd_execute.F90 b/src/cmfd_execute.F90
index b2762ee236..c08fa4fd1b 100644
--- a/src/cmfd_execute.F90
+++ b/src/cmfd_execute.F90
@@ -105,8 +105,6 @@ contains
 
   end subroutine cmfd_init_batch
 
-# ifdef PETSC
-
 !==============================================================================
 ! PROCESS_CMFD_OPTIONS processes user options that interface with PETSc 
 !==============================================================================
@@ -404,8 +402,6 @@ contains
 
   end function get_matrix_idx
 
-# endif
-
 !===============================================================================
 ! CMFD_TALLY_RESET
 !===============================================================================
diff --git a/src/cmfd_input.F90 b/src/cmfd_input.F90
index e98eaac762..7b5e9c2d62 100644
--- a/src/cmfd_input.F90
+++ b/src/cmfd_input.F90
@@ -32,7 +32,7 @@ contains
     end if
 
     ! Split up procs
-# ifdef MPI
+# ifdef PETSC 
     call MPI_COMM_SPLIT(MPI_COMM_WORLD,color,0,new_comm,mpi_err)
 # endif
 
diff --git a/src/cmfd_power_solver.F90 b/src/cmfd_power_solver.F90
index 6de504f5a3..60ba09f2c6 100644
--- a/src/cmfd_power_solver.F90
+++ b/src/cmfd_power_solver.F90
@@ -67,10 +67,10 @@ contains
         call compute_adjoint()
 
     ! Set up krylov info
-    call gmres % set_oper(loss % petsc_mat, loss % petsc_mat)
+    call gmres % set_oper(loss, loss)
 
     ! Precondition matrix
-    call gmres % precondition(loss % petsc_mat)
+    call gmres % precondition(loss)
 
     ! Stop timer for build
     call time_cmfdbuild % stop()
@@ -177,16 +177,16 @@ contains
     do i = 1, 10000
 
       ! compute source vector
-      call prod % vector_multiply(phi_o % petsc_vec, S_o % petsc_vec)
+      call prod % vector_multiply(phi_o, S_o)
 
       ! normalize source vector
       S_o % val = S_o % val / k_o
 
       ! compute new flux vector
-      call gmres % solve(S_o % petsc_vec, phi_n % petsc_vec)
+      call gmres % solve(S_o, phi_n)
 
       ! compute new source vector
-      call prod % vector_multiply(phi_n % petsc_vec, S_n % petsc_vec)
+      call prod % vector_multiply(phi_n, S_n)
 
       ! compute new k-eigenvalue
       k_n = sum(S_n % val) / sum(S_o % val)
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index f402f32428..3abe276d71 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -24,12 +24,10 @@ module matrix_header
      procedure :: add_value    => matrix_add_value
      procedure :: new_row      => matrix_new_row
      procedure :: assemble     => matrix_assemble
-#   ifdef PETSC
      procedure :: setup_petsc        => matrix_setup_petsc
      procedure :: write_petsc_binary => matrix_write_petsc_binary
      procedure :: transpose          => matrix_transpose
      procedure :: vector_multiply    => matrix_vector_multiply
-#   endif
   end type matrix
 
 contains
@@ -102,8 +100,6 @@ contains
 
   end subroutine matrix_new_row
 
-# ifdef PETSC
-
 !===============================================================================
 ! MATRIX_ASSEMBLE
 !===============================================================================
@@ -224,8 +220,10 @@ contains
     self % col = self % col - 1
 
     ! link to petsc
+#ifdef PETSC
     call MatCreateSeqAIJWithArrays(PETSC_COMM_WORLD, self % n, self % n, &
             self % row, self % col, self % val, self % petsc_mat, petsc_err)
+#endif
 
   end subroutine matrix_setup_petsc
 
@@ -239,12 +237,14 @@ contains
     class(Matrix) :: self
 
     integer :: petsc_err
+#ifdef PETSC
     PetscViewer :: viewer
 
     call PetscViewerBinaryOpen(PETSC_COMM_WORLD, trim(filename), &
          FILE_MODE_WRITE, viewer, petsc_err)
     call MatView(self % petsc_mat, viewer, petsc_err)
     call PetscViewerDestroy(viewer, petsc_err)
+#endif
 
   end subroutine matrix_write_petsc_binary
 
@@ -258,8 +258,10 @@ contains
 
     integer :: petsc_err
 
+#ifdef PETSC
     call MatTranspose(self % petsc_mat, MAT_REUSE_MATRIX, self % petsc_mat, &
          petsc_err)
+#endif
 
   end subroutine matrix_transpose
 
@@ -269,16 +271,19 @@ contains
 
   subroutine matrix_vector_multiply(self, vec_in, vec_out)
 
+    use vector_header,  only: Vector
+
     class(Matrix) :: self
-    Vec           :: vec_in
-    Vec           :: vec_out
+    type(Vector)  :: vec_in
+    type(Vector)  :: vec_out
 
     integer :: petsc_err
-    call self % write_petsc_binary('prod_mat.bin')
-    call MatMult(self % petsc_mat, vec_in, vec_out, petsc_err)
+
+#ifdef PETSC
+    call MatMult(self % petsc_mat, vec_in % petsc_vec, vec_out % petsc_vec, &
+                 petsc_err)
+#endif
 
   end subroutine matrix_vector_multiply
 
-# endif
-
 end module matrix_header
diff --git a/src/petsc_solver.F90 b/src/petsc_solver.F90
index 8f42f53202..446e12732a 100644
--- a/src/petsc_solver.F90
+++ b/src/petsc_solver.F90
@@ -1,13 +1,20 @@
 module petsc_solver
 
+  use matrix_header,  only: Matrix
+  use vector_header,  only: Vector
+
   implicit none
   private
 
+#ifdef PETSC
 #  include <finclude/petsc.h90>
+#endif
 
   type, public :: Petsc_gmres 
+#ifdef PETSC
     KSP :: ksp
     PC  :: pc
+#endif
    contains
      procedure :: create       => gmres_create
      procedure :: precondition => gmres_precondition
@@ -32,6 +39,7 @@ contains
     real(8) :: rtol = 1.0e-10_8
     real(8) :: atol = 1.0e-10_8
 
+#ifdef PETSC
     call KSPCreate(PETSC_COMM_WORLD, self % ksp, petsc_err)
     call KSPSetTolerances(self % ksp, rtol, atol, &
          PETSC_DEFAULT_DOUBLE_PRECISION, PETSC_DEFAULT_INTEGER, petsc_err)
@@ -39,6 +47,7 @@ contains
     call KSPSetInitialGuessNonzero(self % ksp, PETSC_TRUE, petsc_err)
     call KSPGetPC(self % ksp, self % pc, petsc_err)
     call PCFactorSetLevels(self % pc, ilu_levels, petsc_err)
+#endif
 
   end subroutine gmres_create
 
@@ -46,13 +55,15 @@ contains
 ! GMRES_PRECONDITION
 !===============================================================================
 
-  subroutine gmres_precondition(self, matrix)
+  subroutine gmres_precondition(self, mat)
 
     class(Petsc_gmres) :: self
-    Mat          :: matrix
+    type(Matrix)       :: mat
 
+#ifdef PETSC
     call KSPSetUp(self % ksp, petsc_err) 
-    call PCFactorGetMatrix(self % pc, matrix, petsc_err) 
+    call PCFactorGetMatrix(self % pc, mat % petsc_mat, petsc_err) 
+#endif
 
   end subroutine gmres_precondition
 
@@ -60,14 +71,16 @@ contains
 ! GMRES_SET_OPER
 !===============================================================================
 
-  subroutine gmres_set_oper(self, prec_matrix, matrix)
+  subroutine gmres_set_oper(self, prec_mat, mat)
 
     class(Petsc_gmres) :: self
-    Mat                :: prec_matrix
-    Mat                :: matrix
+    type(Matrix)       :: prec_mat
+    type(Matrix)       :: mat
 
-    call KSPSetOperators(self % ksp, matrix, prec_matrix, &
+#ifdef PETSC
+    call KSPSetOperators(self % ksp, mat % petsc_mat, prec_mat % petsc_mat, &
          SAME_NONZERO_PATTERN, petsc_err)
+#endif
 
   end subroutine gmres_set_oper
 
@@ -79,7 +92,9 @@ contains
 
    class(Petsc_gmres) :: self
 
+#ifdef PETSC
     call KSPDestroy(self % ksp, petsc_err)
+#endif
 
   end subroutine gmres_destroy
 
@@ -90,12 +105,14 @@ contains
   subroutine gmres_solve(self, b, x)
 
     class(Petsc_gmres) :: self
-    Vec :: b
-    Vec :: x
+    type(Vector)       :: b
+    type(Vector)       :: x
 
     integer :: petsc_err
 
-    call KSPSolve(self % ksp, b, x, petsc_err)
+#ifdef PETSC
+    call KSPSolve(self % ksp, b % petsc_vec, x % petsc_vec, petsc_err)
+#endif
 
   end subroutine gmres_solve
 
diff --git a/src/vector_header.F90 b/src/vector_header.F90
index 53d3bb6fe5..ec9427e916 100644
--- a/src/vector_header.F90
+++ b/src/vector_header.F90
@@ -19,10 +19,8 @@ module vector_header
      procedure :: create       => vector_create
      procedure :: destroy      => vector_destroy
      procedure :: add_value    => vector_add_value
-#   ifdef PETSC
      procedure :: setup_petsc  => vector_setup_petsc
      procedure :: write_petsc_binary => vector_write_petsc_binary
-#   endif
   end type Vector
 
 contains
@@ -73,8 +71,6 @@ contains
 
   end subroutine vector_add_value
 
-# ifdef PETSC
-
 !===============================================================================
 ! VECTOR_SETUP_PETSC
 !===============================================================================
@@ -86,8 +82,10 @@ contains
     integer :: petsc_err
 
     ! link to petsc
+#ifdef PETSC
     call VecCreateSeqWithArray(PETSC_COMM_WORLD, 1, self % n, self % val, &
          self % petsc_vec, petsc_err) 
+#endif
 
   end subroutine vector_setup_petsc
 
@@ -101,14 +99,15 @@ contains
     class(Vector) :: self
 
     integer :: petsc_err
+#ifdef PETSC
     PetscViewer :: viewer
 
     call PetscViewerBinaryOpen(PETSC_COMM_WORLD, trim(filename), &
          FILE_MODE_WRITE, viewer, petsc_err)
     call VecView(self % petsc_vec, viewer, petsc_err)
     call PetscViewerDestroy(viewer, petsc_err)
+#endif
 
   end subroutine vector_write_petsc_binary
-# endif
 
 end module vector_header

From 22c373dd03b956264c8559204bcd8b7f34004878 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Wed, 10 Jul 2013 11:45:46 -0400
Subject: [PATCH 20/69] restructued jacobian and jfnk routines to fit with new
 convention

---
 src/DEPENDENCIES               |  19 +-
 src/OBJECTS                    |   3 +-
 src/cmfd_execute.F90           |   6 +-
 src/cmfd_jacobian_operator.F90 | 357 -------------------------
 src/cmfd_jfnk_solver.F90       | 353 ++++++++++++++++++++++++
 src/cmfd_snes_solver.F90       | 473 ---------------------------------
 src/matrix_header.F90          |  38 ++-
 src/petsc_solver.F90           | 146 +++++++++-
 src/vector_header.F90          |   6 +-
 9 files changed, 544 insertions(+), 857 deletions(-)
 delete mode 100644 src/cmfd_jacobian_operator.F90
 create mode 100644 src/cmfd_jfnk_solver.F90
 delete mode 100644 src/cmfd_snes_solver.F90

diff --git a/src/DEPENDENCIES b/src/DEPENDENCIES
index 0de6323f20..bb1f7a7cd6 100644
--- a/src/DEPENDENCIES
+++ b/src/DEPENDENCIES
@@ -104,13 +104,6 @@ fission.o: global.o
 fission.o: interpolation.o
 fission.o: search.o
 
-cmfd_jacobian_operator.o: constants.o
-cmfd_jacobian_operator.o: global.o
-
-cmfd_snes_solver.o: constants.o
-cmfd_snes_solver.o: global.o
-cmfd_snes_solver.o: string.o
-
 cmfd_input.o: error.o
 cmfd_input.o: global.o
 cmfd_input.o: mesh_header.o
@@ -121,6 +114,7 @@ cmfd_input.o: tally_header.o
 cmfd_input.o: tally_initialize.o
 cmfd_input.o: templates/cmfd_t.o
 
+matrix_header.o: constants.o
 matrix_header.o: vector_header.o
 
 input_xml.o: cmfd_input.o
@@ -144,6 +138,15 @@ input_xml.o: templates/plots_t.o
 input_xml.o: templates/settings_t.o
 input_xml.o: templates/tallies_t.o
 
+cmfd_jfnk_solver.o: cmfd_loss_operator.o
+cmfd_jfnk_solver.o: cmfd_power_solver.o
+cmfd_jfnk_solver.o: cmfd_prod_operator.o
+cmfd_jfnk_solver.o: constants.o
+cmfd_jfnk_solver.o: global.o
+cmfd_jfnk_solver.o: matrix_header.o
+cmfd_jfnk_solver.o: petsc_solver.o
+cmfd_jfnk_solver.o: vector_header.o
+
 main.o: constants.o
 main.o: eigenvalue.o
 main.o: finalize.o
@@ -317,8 +320,8 @@ plot_header.o: constants.o
 cmfd_header.o: constants.o
 
 cmfd_execute.o: cmfd_data.o
+cmfd_execute.o: cmfd_jfnk_solver.o
 cmfd_execute.o: cmfd_power_solver.o
-cmfd_execute.o: cmfd_snes_solver.o
 cmfd_execute.o: constants.o
 cmfd_execute.o: error.o
 cmfd_execute.o: global.o
diff --git a/src/OBJECTS b/src/OBJECTS
index fe309726d7..5075a5ee53 100644
--- a/src/OBJECTS
+++ b/src/OBJECTS
@@ -6,12 +6,11 @@ cmfd_data.o \
 cmfd_execute.o \
 cmfd_header.o \
 cmfd_input.o \
-cmfd_jacobian_operator.o \
+cmfd_jfnk_solver.o \
 cmfd_loss_operator.o \
 cmfd_output.o \
 cmfd_power_solver.o \
 cmfd_prod_operator.o \
-cmfd_snes_solver.o \
 constants.o \
 cross_section.o \
 dict_header.o \
diff --git a/src/cmfd_execute.F90 b/src/cmfd_execute.F90
index c08fa4fd1b..60452c8850 100644
--- a/src/cmfd_execute.F90
+++ b/src/cmfd_execute.F90
@@ -21,7 +21,7 @@ contains
 
     use cmfd_data,              only: set_up_cmfd
     use cmfd_power_solver,      only: cmfd_power_execute
-    use cmfd_snes_solver,       only: cmfd_snes_execute
+    use cmfd_jfnk_solver,       only: cmfd_jfnk_execute
     use error,                  only: warning, fatal_error 
 
     ! CMFD single processor on master
@@ -40,7 +40,7 @@ contains
       if (trim(cmfd_solver_type) == 'power') then
         call cmfd_power_execute()
       elseif (trim(cmfd_solver_type) == 'jfnk') then
-        call cmfd_snes_execute()
+        call cmfd_jfnk_execute()
       else
         message = 'solver type became invalid after input processing'
         call fatal_error() 
@@ -51,7 +51,7 @@ contains
         if (trim(cmfd_solver_type) == 'power') then
           call cmfd_power_execute(adjoint = .true.)
         elseif (trim(cmfd_solver_type) == 'jfnk') then
-          call cmfd_snes_execute(adjoint = .true.)
+          call cmfd_jfnk_execute(adjoint = .true.)
         end if
       end if
 
diff --git a/src/cmfd_jacobian_operator.F90 b/src/cmfd_jacobian_operator.F90
deleted file mode 100644
index b5532b66a7..0000000000
--- a/src/cmfd_jacobian_operator.F90
+++ /dev/null
@@ -1,357 +0,0 @@
-module cmfd_jacobian_operator
-
-# ifdef PETSC
-
-  use cmfd_loss_operator, only: loss_operator,init_M_operator, &
-                                build_loss_matrix,destroy_M_operator
-  use cmfd_prod_operator, only: prod_operator,init_F_operator, &
-                                build_prod_matrix,destroy_F_operator
-  implicit none
-  private
-  public :: init_J_operator, build_jacobian_matrix, destroy_J_operator
-
-# include <finclude/petsc.h90>
-
-  integer  :: nx   ! maximum number of x cells
-  integer  :: ny   ! maximum number of y cells
-  integer  :: nz   ! maximum number of z cells
-  integer  :: ng   ! maximum number of groups
-  integer  :: ierr ! petsc error code
-    
-  type, public :: jacobian_operator
-    Mat      :: J    ! petsc matrix for neutronic prod operator
-    integer  :: n    ! dimensions of matrix
-    integer  :: nnz  ! max number of nonzeros
-    integer  :: localn ! local size on proc
-    integer, allocatable :: d_nnz(:) ! vector of diagonal preallocation
-    integer, allocatable :: o_nnz(:) ! vector of off-diagonal preallocation
-  end type jacobian_operator
-
-  type, public :: operators 
-    type(loss_operator) :: loss
-    type(prod_operator) :: prod
-  end type operators
-
-contains
-
-!==============================================================================
-! INIT_J_OPERATOR
-!==============================================================================
-
-  subroutine init_J_operator(this,ctx)
-
-    type(jacobian_operator) :: this
-    type(operators)         :: ctx
-
-    ! get indices
-    call get_J_indices(this)
-
-    ! get preallocation
-    call preallocate_jacobian_matrix(this,ctx)
-
-    ! set up M operator
-    call MatCreateAIJ(PETSC_COMM_WORLD, this%localn, this%localn, PETSC_DECIDE,&
-         PETSC_DECIDE, PETSC_NULL_INTEGER, this%d_nnz, PETSC_NULL_INTEGER, &
-         this%o_nnz, this%J,ierr)
-    call MatSetOption(this%J, MAT_NEW_NONZERO_LOCATIONS, PETSC_TRUE, ierr)
-    call MatSetOption(this%J, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE, ierr)
-
-  end subroutine init_J_operator
-
-!==============================================================================
-! GET_J_INDICES
-!==============================================================================
-
-  subroutine get_J_indices(this)
-
-    use global,  only: cmfd, cmfd_coremap
-
-    type(jacobian_operator) :: this
-
-    ! get maximum number of cells in each direction
-    nx = cmfd%indices(1)
-    ny = cmfd%indices(2)
-    nz = cmfd%indices(3)
-    ng = cmfd%indices(4)
-
-    ! get number of nonzeros
-    this%nnz = 7 + ng - 1
-
-    ! calculate dimensions of matrix
-    this%n = nx*ny*nz*ng
-
-    ! calculate dimensions of matrix
-    if (cmfd_coremap) then
-      this%n = cmfd % mat_dim * ng
-    else
-      this%n = nx*ny*nz*ng
-    end if
-
-    ! add 1 for eigenvalue row
-!   this%n = this%n + 1
-
-  end subroutine get_J_indices
-
-!===============================================================================
-! PREALLOCATE_JACOBIAN_MATRIX
-!===============================================================================
-
-  subroutine preallocate_jacobian_matrix(this,ctx)
-
-    use global,      only: cmfd, n_procs_cmfd, rank
-
-    type(jacobian_operator) :: this
-    type(operators)         :: ctx
-
-    integer :: n             ! the extent of the matrix
-    integer :: row_start     ! index of local starting row
-    integer :: row_end       ! index of local final row
-
-    ! get local problem size
-    n = this%n
-
-    ! determine local size, divide evenly between all other procs
-    this%localn = n/(n_procs_cmfd)
-
-    ! add 1 more if less proc id is less than mod
-    if (rank < mod(n,n_procs_cmfd)) this%localn = this%localn + 1
-
-    ! add another 1 on last proc
-    if (rank == n_procs_cmfd - 1) this%localn = this%localn + 1
-
-    ! determine local starting row
-    row_start = 0
-    if (rank < mod(n,n_procs_cmfd)) then
-      row_start = rank*(n/n_procs_cmfd+1)
-    else
-      row_start = min(mod(n,n_procs_cmfd)*(n/n_procs_cmfd+1) + (rank - &
-           mod(n,n_procs_cmfd))*(n/n_procs_cmfd),n)
-    end if
-
-    ! determine local final row
-    row_end = row_start + this%localn - 1
-
-    ! allocate counters
-    if (.not. allocated(this%d_nnz)) allocate(this%d_nnz(row_start:row_end))
-    if (.not. allocated(this%o_nnz)) allocate(this%o_nnz(row_start:row_end))
-    this % d_nnz = 0
-    this % o_nnz = 0
-
-    ! start with pattern from loss matrix
-    if (rank == n_procs_cmfd - 1) then
-      this % d_nnz(row_start:row_end-1) = ctx%loss%d_nnz
-      this % o_nnz(row_start:row_end-1) = ctx%loss%o_nnz
-    else
-      this % d_nnz = ctx%loss%d_nnz
-      this % o_nnz = ctx%loss%o_nnz
-    end if
-
-    ! append -F*phi term for last processor will take care of 1 for lambda
-    if (rank == n_procs_cmfd - 1) then
-      this%d_nnz = this%d_nnz + 1
-    else
-      this%o_nnz = this%o_nnz + 1
-    end if
-
-    ! do last row which has all filled (already did lower left corner above)
-    if (rank == n_procs_cmfd - 1) then
-      this % d_nnz(row_end) = this % d_nnz(row_end) + (row_end - row_start)
-      this % o_nnz(row_end) = this % o_nnz(row_end) + (this%n  - (row_end - &
-           row_start))
-    end if
-
-  end subroutine preallocate_jacobian_matrix
-
-!===============================================================================
-! BUILD_JACOBIAN_MATRIX creates the matrix representing loss of neutrons
-!===============================================================================
-
-  subroutine build_jacobian_matrix(snes,x,jac,jac_prec,flag,ctx,ierr)
-
-    use constants,    only: ZERO, ONE
-    use global,       only: n_procs_cmfd, cmfd_write_matrices, rank
-
-    SNES            :: snes      ! the snes context
-    Vec             :: x         ! the solution vector
-    Mat             :: jac       ! the jacobian matrix
-    Mat             :: jac_prec  ! the jacobian preconditioner
-    MatStructure    :: flag      ! not used
-    type(operators) :: ctx       ! not used
-    integer         :: ierr      ! petsc error flag
-
-    Vec                  :: phi      ! flux vector
-    Vec                  :: source   ! source vector
-    integer              :: n        ! problem size
-    integer              :: k        ! implied do loop counter
-    integer              :: ncols    ! number of nonzeros in cols
-    integer              :: irow     ! row counter
-    integer              :: row_start! starting local row on process
-    integer              :: row_end  ! ending local row on process
-    integer, allocatable :: dims(:)  ! vec of starting and ending rows
-    integer, allocatable :: dims1(:) ! vec of sizes on each proc
-    integer, allocatable :: cols(:)  ! vector of column numbers
-    real(8)              :: lambda   ! eigenvalue
-    real(8), pointer     :: xptr(:)  ! pointer to solution vector
-    real(8), pointer     :: sptr(:)  ! pointer to source vector
-    real(8), allocatable :: vals(:)  ! vector of row values
-    real(8), allocatable :: phi_tmp(:) ! temp buffer for flux
-
-    ! create operators
-    call build_loss_matrix(ctx%loss)
-    call build_prod_matrix(ctx%prod)
-
-    ! get problem size
-    n = ctx%loss%n
-
-    ! get local size on each processor
-    call MatGetOwnershipRange(jac_prec, row_start, row_end, ierr)
-
-    ! allocate cols and  initialize to zero
-    if (.not. allocated(cols)) allocate(cols(&
-         maxval(ctx%loss%d_nnz + ctx%loss%o_nnz)))
-    if (.not. allocated(vals)) allocate(vals(&
-         maxval(ctx%loss%d_nnz + ctx%loss%o_nnz)))
-    cols = 0
-    vals = ZERO
-
-    ! get pointers to residual vector 
-    call VecGetArrayF90(x, xptr, ierr)
-
-    ! create petsc vector for flux
-    call VecCreateMPI(PETSC_COMM_WORLD, ctx%loss%localn, PETSC_DECIDE, phi, ierr)
-
-    ! extract flux and eigenvalue
-    call VecPlaceArray(phi, xptr, ierr)
-    if (rank == n_procs_cmfd - 1) lambda = xptr(size(xptr))
-    call MPI_BCAST(lambda, 1, MPI_REAL8, n_procs_cmfd-1, PETSC_COMM_WORLD, ierr)
-
-    ! compute math (M-lambda*F) M is overwritten here
-    call MatAXPY(ctx%loss%M, -lambda, ctx%prod%F, &
-         DIFFERENT_NONZERO_PATTERN, ierr)
-
-    ! create tmp petsc vector for source
-    call VecCreateMPI(PETSC_COMM_WORLD, ctx%loss%localn, PETSC_DECIDE, &
-         source, ierr)
-
-    ! perform math (-F*phi --> source)
-    call MatMult(ctx%prod%F, phi, source, ierr)
-    call VecScale(source, -ONE, ierr)
-
-    ! get pointer to source
-    call VecGetArrayF90(source, sptr, ierr)
-
-    ! begin loop to insert things into matrix
-    do irow = row_start, row_end - 1
-
-      ! don't do last row
-      if (irow == n) cycle 
-
-      ! get row of matrix
-      call MatGetRow(ctx%loss%M, irow, ncols, cols, vals, ierr)
-
-      ! set that row to Jacobian matrix
-      call MatSetValues(jac_prec, 1, (/irow/), ncols, cols(1:ncols), vals, &
-           INSERT_VALUES, ierr)
-
-      ! restore the row
-      call MatRestoreRow(ctx%loss%M, irow, ncols, cols, vals, ierr)
-
-      ! insert source value
-      call MatSetValue(jac_prec, irow, n, sptr(irow-row_start+1), &
-           INSERT_VALUES, ierr)
-
-    end do
-
-    ! allocate space for flux vector buffer
-    if (rank == n_procs_cmfd - 1) then
-      if (.not. allocated(phi_tmp)) allocate(phi_tmp(0:n-1))
-    end if
-
-    ! get size on each proc
-    if (.not. allocated(dims)) allocate(dims(0:n_procs_cmfd))
-    if (.not. allocated(dims1)) allocate(dims1(0:n_procs_cmfd-1))
-    call VecGetOwnershipRanges(phi, dims, ierr)
-    do k = 0, n_procs_cmfd-1
-      dims1(k) = dims(k+1) - dims(k)
-    end do
-
-    ! gather data on all procs (will truncate xptr if needed for last proc)
-    call MPI_GATHERV(xptr, dims1(rank), MPI_REAL8, phi_tmp, dims1, &
-         dims(0:n_procs_cmfd-1), MPI_REAL8, n_procs_cmfd-1, &
-         PETSC_COMM_WORLD, ierr)
-
-    ! set values in last row of matrix
-    if (rank == n_procs_cmfd - 1) then
-      phi_tmp = -phi_tmp ! negate the transpose
-      call MatSetValues(jac_prec, 1, (/n/), n, (/(k,k=0,n-1)/), phi_tmp, &
-           INSERT_VALUES, ierr)
-      call MatSetValue(jac_prec, n, n, ONE, INSERT_VALUES, ierr)
-    end if
-
-    ! assemble matrix
-    call MatAssemblyBegin(jac_prec, MAT_FINAL_ASSEMBLY, ierr)
-    call MatAssemblyEnd(jac_prec, MAT_FINAL_ASSEMBLY, ierr)
-    call MatAssemblyBegin(jac, MAT_FINAL_ASSEMBLY, ierr)
-    call MatAssemblyEnd(jac, MAT_FINAL_ASSEMBLY, ierr)
-
-    ! reset all vectors
-    call VecResetArray(phi,ierr)
-
-    ! restore all vectors
-    call VecRestoreArrayF90(x, xptr, ierr)
-    call VecRestoreArrayF90(source, sptr, ierr)
-
-    ! destroy all temporary objects
-    call VecDestroy(phi, ierr)
-    call VecDestroy(source, ierr)
-
-    ! deallocate all temporary space
-    if (allocated(cols)) deallocate(cols)
-    if (allocated(vals)) deallocate(vals)
-    if (allocated(phi_tmp)) deallocate(phi_tmp)
-    if (allocated(dims)) deallocate(dims)
-    if (allocated(dims1)) deallocate(dims1)
-
-    ! print jacobian out
-    if (cmfd_write_matrices) call print_J_operator(jac_prec)
-
-  end subroutine build_jacobian_matrix 
-
-!===============================================================================
-! PRINT_J_OPERATOR
-!===============================================================================
-
-  subroutine print_J_operator(jac)
-
-    Mat :: jac
- 
-    PetscViewer :: viewer
-
-    ! write out matrix in binary file (debugging)
-    call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'jacobian.bin', &
-         FILE_MODE_WRITE, viewer, ierr)
-    call MatView(jac, viewer, ierr)
-    call PetscViewerDestroy(viewer, ierr)
-
-  end subroutine print_J_operator
-
-!==============================================================================
-! DESTROY_J_OPERATOR
-!==============================================================================
-
-  subroutine destroy_J_operator(this)
-
-    type(jacobian_operator) :: this
-
-    ! deallocate matrix
-    call MatDestroy(this%J,ierr)
-
-    ! deallocate other parameters
-    if (allocated(this%d_nnz)) deallocate(this%d_nnz)
-    if (allocated(this%o_nnz)) deallocate(this%o_nnz)
-
-  end subroutine destroy_J_operator
-
-# endif
-
-end module cmfd_jacobian_operator
diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
new file mode 100644
index 0000000000..acba55b84b
--- /dev/null
+++ b/src/cmfd_jfnk_solver.F90
@@ -0,0 +1,353 @@
+module cmfd_jfnk_solver
+
+  use cmfd_loss_operator,  only: init_loss_matrix, build_loss_matrix
+  use cmfd_power_solver,   only: cmfd_power_execute
+  use cmfd_prod_operator,  only: init_prod_matrix, build_prod_matrix
+  use matrix_header,       only: Matrix
+  use petsc_solver,        only: Petsc_jfnk, Jfnk_ctx
+  use vector_header,       only: Vector
+
+  implicit none
+  private
+  public :: cmfd_jfnk_execute
+
+  logical          :: adjoint_calc = .false. ! adjoint calculation
+  type(Jfnk_ctx)   :: jfnk_data
+  type(Matrix)     :: jac_prec
+  type(Matrix)     :: loss
+  type(Matrix)     :: prod
+  type(Petsc_jfnk) :: jfnk 
+  type(Vector)     :: resvec
+  type(Vector)     :: xvec
+
+contains
+
+!===============================================================================
+! CMFD_JFNK_EXECUTE
+!===============================================================================
+
+  subroutine cmfd_jfnk_execute(adjoint)
+
+    logical, optional :: adjoint ! adjoint calculation
+
+    ! check for adjoint
+    if (present(adjoint)) adjoint_calc = adjoint
+
+    ! seed with power iteration
+    call cmfd_power_execute(k_tol=1.E-3_8, s_tol=1.E-3_8, adjoint=adjoint_calc)
+
+    ! initialize data
+    call init_data()
+
+    ! initialize solver
+    call jfnk % create()
+
+    ! solve the system
+!   call SNESSolve(snes, PETSC_NULL_DOUBLE, xvec, ierr)
+
+    ! extracts results to cmfd object
+    call extract_results()
+
+    ! deallocate all slepc data
+    call finalize()
+
+  end subroutine cmfd_jfnk_execute
+
+!===============================================================================
+! INIT_DATA allocates matrices vectors for CMFD solution
+!===============================================================================
+
+  subroutine init_data()
+
+    use constants, only: ONE
+    use global,    only: cmfd
+
+    integer              :: n         ! problem size
+
+    ! Set up all matrices
+    call init_loss_matrix(loss)
+    call init_prod_matrix(prod)
+    call init_jacobian_matrix()
+
+    ! Set up for use with petsc
+    call jac_prec % setup_petsc()
+
+    ! Get problem size
+    n = jac_prec % n
+
+    ! Create problem vectors
+    call resvec % create(n)
+    call xvec % create(n)
+
+    ! set flux in guess
+    if (adjoint_calc) then
+      xvec % val(1:n) = cmfd % adj_phi
+    else
+      xvec % val(1:n) = cmfd % phi
+    end if
+
+    ! set keff in guess
+    if (adjoint_calc) then
+      xvec % val(n + 1) = ONE/cmfd % adj_keff
+    else
+      xvec % val(n + 1) = ONE/cmfd % keff
+    end if
+
+    ! set solver names
+
+  end subroutine init_data
+
+!==============================================================================
+! INIT_JACOBIAN_MATRIX preallocates jacobian matrix and initializes it
+!==============================================================================
+
+  subroutine init_jacobian_matrix()
+
+    integer :: nnz
+    integer :: n
+
+    ! Get length of matrix and number of nonzeros total in loss matrix
+    nnz = loss % nnz
+    n   = loss % n
+
+    ! There is one more nonzero for each row and and additional row of nonzeros
+    nnz = nnz + 2*n + 1
+
+    ! We need 1 more row for the jacobian
+    n = n + 1
+
+    ! Configure Jacobian matrix
+    call jac_prec % create(n, nnz)
+
+  end subroutine init_jacobian_matrix
+
+!===============================================================================
+! BUILD_JACOBIAN_MATRIX creates the Jacobian of nonlinear eigenvalue problem
+!===============================================================================
+
+  subroutine build_jacobian_matrix(x)
+
+    use constants,  only: ONE
+
+    type(Vector) :: x
+
+    integer      :: i      ! loop counter for jacobian rows
+    integer      :: jjac   ! loop counter for jacobian cols
+    integer      :: jloss  ! loop counter for loss matrix cols
+    integer      :: jprod  ! loop counter for prod matrix cols
+    integer      :: n      ! problem size
+    real(8)      :: lambda ! eigenvalue
+    real(8)      :: val    ! temporary real scalar
+    type(Vector) :: flux   ! flux vector
+    type(Vector) :: fsrc   ! fission source vector
+
+    ! get the problem size
+    n = loss % n
+
+    ! get flux and eigenvalue
+    flux % val => x % val(1:n)
+    lambda = x % val(n + 1)
+
+    ! create fission source vector
+    call fsrc % create(n)
+    call prod % vector_multiply(flux, fsrc)
+
+    ! reset counters in Jacobian matrix
+    jac_prec % n_kount  = 1
+    jac_prec % nz_kount = 1
+
+    ! begin loop around rows of Jacobian matrix
+    ROWS: do i = 1, n
+
+      ! add a new row in Jacobian
+      call jac_prec % new_row()
+
+      ! begin loop around columns of loss matrix
+      COLS_LOSS: do jloss = loss % row(i), loss % row(i + 1) - 1
+
+        ! start with the value in the loss matrix
+        val = loss % val(jloss)
+
+        ! loop around columns of prod matrix
+        COLS_PROD: do jprod = prod % row(i), prod % row(i + 1) - 1
+
+          ! see if columns agree with loss matrix
+          if (prod % col(jprod) == loss % col(jloss)) then
+            val = val - lambda*prod % val(jprod)
+            exit
+          end if
+
+        end do COLS_PROD 
+
+        ! record value in jacobian
+        call jac_prec % add_value(jloss, val)
+
+      end do COLS_LOSS
+
+      ! add fission source value
+      val = fsrc % val(n)
+      call jac_prec % add_value(n + 1, val)
+
+    end do ROWS
+
+    ! need to add negative transpose of flux vector on last row
+    call jac_prec % new_row()
+    do jjac = 1, n
+      val = -flux % val(jjac)
+      call jac_prec % add_value(jjac, val)
+    end do
+
+    ! add unity on bottom right corner of matrix
+    call jac_prec % add_value(n + 1, ONE)
+
+    ! free all allocated memory
+    flux % val => null()
+    call fsrc % destroy()
+
+  end subroutine build_jacobian_matrix
+
+!===============================================================================
+! COMPUTE_NONLINEAR_RESIDUAL
+!===============================================================================
+
+  subroutine compute_nonlinear_residual(x)
+
+    use global,       only: cmfd_write_matrices, cmfd_adjoint_type
+
+    type(Vector)      :: x 
+
+    character(len=25) :: filename
+    integer           :: n
+    logical           :: physical_adjoint = .false.
+    real(8)           :: lambda
+    type(Vector)      :: res_loss
+    type(Vector)      :: res_prod
+    type(Vector)      :: flux
+
+    ! Check for physical adjoint
+    if (adjoint_calc .and. trim(cmfd_adjoint_type) == 'physical') &
+         physical_adjoint = .true.
+
+    ! Create operators
+    call build_loss_matrix(loss, adjoint = physical_adjoint)
+    call build_prod_matrix(prod, adjoint = physical_adjoint)
+
+    ! Check for adjoint
+    if (adjoint_calc .and. trim(cmfd_adjoint_type) == 'math') &
+         call compute_adjoint()
+
+    ! Get problem size
+    n = loss % n
+
+    ! Set up temporary vectors
+    call res_loss % create(n)
+    call res_prod % create(n)
+
+    ! Extract flux
+    flux % n = n
+    flux % val => x % val(1:n)
+
+    ! Extract eigenvalue
+    lambda = x % val(n + 1)
+
+    ! Calculate M*flux then F*flux
+    call loss % vector_multiply(flux, res_loss)
+    call prod % vector_multiply(flux, res_prod)
+
+    ! Put flux component values in residual vector
+    resvec % val(1:n) = res_loss % val - lambda*res_prod % val
+
+    ! Put eigenvalue component in residual vector
+    resvec % val(n+1) = 0.5_8 - 0.5_8*sum(flux % val * flux % val)
+
+    ! write out data in binary files (debugging)
+    if (cmfd_write_matrices) then
+
+      ! write out residual vector 
+      if (adjoint_calc) then
+        filename = 'adj_res.bin'
+      else
+        filename = 'res.bin'
+      end if
+      call resvec % write_petsc_binary(filename)
+
+      ! write out solution vector
+      if (adjoint_calc) then
+        filename = 'adj_x.bin'
+      else
+        filename = 'x.bin'
+      end if
+      call x % write_petsc_binary(filename)
+
+    end if
+
+  end subroutine compute_nonlinear_residual
+
+!===============================================================================
+! COMPUTE_ADJOINT
+!===============================================================================
+
+  subroutine compute_adjoint()
+
+    use global,  only: cmfd_write_matrices
+
+    ! transpose matrices
+    call loss % transpose()
+    call prod % transpose()
+
+    ! write out matrix in binary file (debugging)
+    if (cmfd_write_matrices) then
+      call loss % write_petsc_binary('adj_lossmat.bin')
+      call loss % write_petsc_binary('adj_prodmat.bin')
+    end if
+
+  end subroutine compute_adjoint
+
+!===============================================================================
+! EXTRACT_RESULTS 
+!===============================================================================
+
+  subroutine extract_results()
+
+    use constants, only: ZERO, ONE
+    use global,    only: cmfd
+
+    integer :: n ! problem size
+
+    ! get problem size
+    n = jfnk_data % loss % n
+
+    ! also allocate in cmfd object
+    if (adjoint_calc) then
+      if (.not. allocated(cmfd % adj_phi)) allocate(cmfd % adj_phi(n))
+    else
+      if (.not. allocated(cmfd % phi)) allocate(cmfd % phi(n))
+    end if
+
+    ! Get flux and eigenvalue
+    if (adjoint_calc) then
+      cmfd % adj_phi  = xvec % val(1:n)
+      cmfd % adj_keff = ONE / xvec % val(n+1)
+    else
+      cmfd % phi  = xvec % val(1:n)
+      cmfd % keff = xvec % val(n+1)
+    end if
+
+  end subroutine extract_results
+
+!===============================================================================
+! FINALIZE frees all memory from a JFNK calculation
+!===============================================================================
+
+  subroutine finalize()
+
+    call jfnk_data % loss % destroy()
+    call jfnk_data % prod % destroy()
+    call jac_prec % destroy()
+    call xvec % destroy()
+    call resvec % destroy()
+    call jfnk % destroy() 
+
+  end subroutine finalize
+
+end module cmfd_jfnk_solver
diff --git a/src/cmfd_snes_solver.F90 b/src/cmfd_snes_solver.F90
deleted file mode 100644
index 6293720207..0000000000
--- a/src/cmfd_snes_solver.F90
+++ /dev/null
@@ -1,473 +0,0 @@
-module cmfd_snes_solver
-
-# ifdef PETSC
-
-  use cmfd_loss_operator,     only: loss_operator,init_M_operator,           &
-                                    build_loss_matrix,destroy_M_operator
-  use cmfd_prod_operator,     only: prod_operator,init_F_operator,           &
-                                    build_prod_matrix,destroy_F_operator
-  use cmfd_jacobian_operator, only: jacobian_operator,init_J_operator,       &
-                                    build_jacobian_matrix,destroy_J_operator,&
-                                    operators
-  use cmfd_power_solver,      only: cmfd_power_execute
-
-  implicit none
-  private
-  public :: cmfd_snes_execute
-
-# include <finclude/petsc.h90>
-
-  type(jacobian_operator) :: jac_prec
-  type(operators) :: ctx
-
-  Mat         :: jac                    ! jacobian matrix
-  Vec         :: resvec                 ! residual vector
-  Vec         :: xvec                   ! results
-  KSP         :: ksp                    ! linear solver context
-  PC          :: pc                     ! preconditioner
-  SNES        :: snes                   ! nonlinear solver context
-  integer     :: ierr                   ! error flag
-
-  logical     :: adjoint_calc = .false. ! adjoint calculation
-
-contains
-
-!===============================================================================
-! CMFD_SNES_EXECUTE
-!===============================================================================
-
-  subroutine cmfd_snes_execute(adjoint)
-
-    logical, optional :: adjoint ! adjoint calculation
-
-    ! check for adjoint
-    if (present(adjoint)) adjoint_calc = adjoint
-
-    ! seed with power iteration
-    call cmfd_power_execute(k_tol=1.E-3_8, s_tol=1.E-3_8, adjoint=adjoint_calc)
-
-    ! initialize data
-    call init_data()
-
-    ! initialize solver
-    call init_solver()
-
-    ! precondition
-    call precondition_matrix()
-
-    ! solve the system
-    call SNESSolve(snes, PETSC_NULL_DOUBLE, xvec, ierr)
-
-    ! extracts results to cmfd object
-    call extract_results()
-
-    ! deallocate all slepc data
-    call finalize()
-
-  end subroutine cmfd_snes_execute
-
-!===============================================================================
-! INIT_DATA allocates matrices vectors for CMFD solution
-!===============================================================================
-
-  subroutine init_data()
-
-    use constants, only: ONE
-    use global,    only: cmfd, n_procs_cmfd, rank
-
-    integer              :: k         ! implied do counter
-    integer              :: n         ! problem size
-    integer              :: row_start ! local row start
-    integer              :: row_end   ! local row end
-    real(8), pointer     :: xptr(:)   ! solution pointer
-
-    ! set up operator matrices
-    call init_M_operator(ctx%loss)
-    call init_F_operator(ctx%prod)
-    call init_J_operator(jac_prec, ctx)
-
-    ! get problem size
-    n = jac_prec%n
-
-    ! create PETSc vectors
-    call VecCreateMPI(PETSC_COMM_WORLD, jac_prec%localn, PETSC_DECIDE, &
-         resvec, ierr)
-    call VecCreateMPI(PETSC_COMM_WORLD, jac_prec%localn, PETSC_DECIDE, &
-         xvec, ierr)
-
-    ! get the local dimensions for each process
-    call VecGetOwnershipRange(xvec, row_start, row_end, ierr)
-
-    if (rank == n_procs_cmfd - 1) row_end = n
-
-    ! set flux in guess
-    if (adjoint_calc) then
-      call VecSetValues(xvec, row_end-row_start, (/(k,k=row_start,row_end-1)/), &
-           cmfd%adj_phi(row_start+1:row_end), INSERT_VALUES, ierr)
-    else
-      call VecSetValues(xvec, row_end-row_start, (/(k,k=row_start,row_end-1)/), &
-           cmfd%phi(row_start+1:row_end), INSERT_VALUES, ierr)
-    end if
-    call VecAssemblyBegin(xvec, ierr)
-    call VecAssemblyEnd(xvec, ierr)
-
-    ! set keff in guess
-    if (rank == n_procs_cmfd - 1) then
-      call VecGetArrayF90(xvec, xptr, ierr)
-      if (adjoint_calc) then
-        xptr(size(xptr)) = ONE/cmfd%adj_keff
-      else
-        xptr(size(xptr)) = ONE/cmfd%keff
-      end if
-      call VecRestoreArrayF90(xvec, xptr, ierr)
-    end if
-
-    ! nullify pointers
-    if (associated(xptr)) nullify(xptr)
-
-  end subroutine init_data
-
-!===============================================================================
-! INIT_SOLVER
-!===============================================================================
-
-  subroutine init_solver()
-
-    use global,  only: cmfd_snes_monitor, n_procs_cmfd
-
-    ! turn on mf_operator option
-    call PetscOptionsSetValue("-snes_mf_operator", "TRUE", ierr)
-
-    ! create SNES context
-    call SNESCreate(PETSC_COMM_WORLD, snes, ierr)
-
-    ! set the residual function
-    call SNESSetFunction(snes, resvec, compute_nonlinear_residual, &
-         PETSC_NULL_DOUBLE, ierr)
-
-    ! set GMRES solver
-    call SNESGetKSP(snes, ksp, ierr)
-    call KSPSetType(ksp, KSPGMRES, ierr)
-
-    ! create matrix free jacobian
-    call MatCreateSNESMF(snes, jac, ierr)
-
-    ! set matrix free finite difference
-    call SNESSetJacobian(snes, jac, jac_prec, build_jacobian_matrix, ctx, ierr)
-
-    ! set lags
-    call SNESSetLagJacobian(snes, -2, ierr)
-    call SNESSetLagPreconditioner(snes, -1, ierr)
-
-    ! set convergence
-!   call SNESSetTolerances(snes, 1.0e-8_8, 1.0e-8_8, 1.0e-8_8, 50, 10000, ierr)
-
-    ! set SNES options
-    call SNESSetFromOptions(snes, ierr)
-
-    ! turn off line searching
-!   call SNESLineSearchSet(snes, SNESLineSearchNo, PETSC_NULL_OBJECT, ierr)
-
-  end subroutine init_solver
-
-!===============================================================================
-! PRECONDITION_MATRIX 
-!===============================================================================
-
-  subroutine precondition_matrix()
-
-    use global,                 only: cmfd_snes_monitor, cmfd_solver_type, &
-                                      n_procs_cmfd, cmfd_ilu_levels, master
-    use string,                 only: to_str
-    use, intrinsic :: ISO_FORTRAN_ENV
-
-    character(LEN=20) :: snestype, ksptype, pctype
-
-    ! set up preconditioner
-    call KSPGetPC(ksp, pc, ierr)
-    if (n_procs_cmfd == 1) then
-      call PCSetType(pc, PCILU, ierr)
-      call PCFactorSetLevels(pc, cmfd_ilu_levels, ierr)
-    else
-      call PetscOptionsSetValue("-pc_type", "bjacobi", ierr)
-      call PetscOptionsSetValue("-sub_pc_type", "ilu", ierr)
-      call PetscOptionsSetValue("-sub_pc_factor_levels", &
-           trim(to_str(cmfd_ilu_levels)), ierr)
-    end if
-
-    ! get options
-    call PCSetFromOptions(pc, ierr)
-    call KSPSetFromOptions(ksp, ierr)
-
-    ! finalize ksp setup
-!   call KSPSetUp(ksp, ierr)
-
-    ! get all types and print
-    call SNESGetType(snes, snestype, ierr)
-    call KSPGetType(ksp, ksptype, ierr)
-    call PCGetType(pc, pctype, ierr)
-
-    ! display solver info to user
-    if (cmfd_snes_monitor .and. master) then
-      write(OUTPUT_UNIT,'(A)') ''
-      write(OUTPUT_UNIT,'(A)') '########################################################'
-      write(OUTPUT_UNIT,'(A)') '################ JFNK Nonlinear Solver  ################'
-      write(OUTPUT_UNIT,'(A)') '########################################################'
-      write(OUTPUT_UNIT,'(A)') ''
-      write(OUTPUT_UNIT,101) 'NONLINEAR SOLVER: ', snestype
-      write(OUTPUT_UNIT,101) 'LINEAR SOLVER:    ', ksptype
-      write(OUTPUT_UNIT,101) 'PRECONDITIONER:   ', pctype
-      write(OUTPUT_UNIT,100) 'ILU levels: ', cmfd_ilu_levels
-      write(OUTPUT_UNIT,'(A)') ''
-      write(OUTPUT_UNIT,'(A)') '---------------------------------------------'
-      write(OUTPUT_UNIT,'(A)') ''
-    end if
-
- 100 FORMAT(A,1X,I0)
- 101 FORMAT(A,1X,A)
- 
-  end subroutine precondition_matrix
-
-!===============================================================================
-! COMPUTE_NONLINEAR_RESIDUAL
-!===============================================================================
-
-  subroutine compute_nonlinear_residual(snes, x, res, ierr)
-
-    use global,       only: n_procs_cmfd, cmfd_write_matrices, &
-                            cmfd_adjoint_type, rank
-
-    SNES        :: snes          ! nonlinear solver context
-    Vec         :: x             ! independent vector
-    Vec         :: res           ! residual vector
-    integer     :: ierr          ! error flag
-
-    Vec         :: phi           ! flux vector
-    Vec         :: rphi          ! flux part of residual
-    Vec         :: phiM          ! M part of residual flux calc
-    integer     :: n             ! problem size
-    real(8)     :: lambda        ! eigenvalue
-    real(8)     :: reslamb       ! residual for lambda
-
-    real(8), pointer :: xptr(:)  ! pointer to solution vector
-    real(8), pointer :: rptr(:)  ! pointer to residual vector
-    PetscViewer :: viewer
-
-    logical :: physical_adjoint = .false.
-
-    ! check for physical adjoint
-    if (adjoint_calc .and. trim(cmfd_adjoint_type) == 'physical') &
-         physical_adjoint = .true.
-
-    ! create operators
-    call build_loss_matrix(ctx%loss, adjoint = physical_adjoint)
-    call build_prod_matrix(ctx%prod, adjoint = physical_adjoint)
-
-    ! check for adjoint
-    if (adjoint_calc .and. trim(cmfd_adjoint_type) == 'math') &
-         call compute_adjoint()
-
-    ! get problem size
-    n = ctx%loss%n 
-
-    ! get pointers to vectors
-    call VecGetArrayF90(x, xptr, ierr)
-    call VecGetArrayF90(res, rptr, ierr)
-
-    ! create petsc vector for flux
-    call VecCreateMPI(PETSC_COMM_WORLD, ctx%loss%localn, &
-         PETSC_DECIDE, phi, ierr)
-    call VecCreateMPI(PETSC_COMM_WORLD, ctx%loss%localn, &
-         PETSC_DECIDE, rphi, ierr)
-
-    ! extract flux and place in petsc vector 
-    call VecPlaceArray(phi, xptr, ierr)
-    call VecPlaceArray(rphi, rptr, ierr)
-
-    ! extract eigenvalue and broadcast (going to want to make this more general in future)
-    if (rank == n_procs_cmfd - 1) lambda = xptr(size(xptr)) 
-    call MPI_BCAST(lambda, 1, MPI_REAL8, n_procs_cmfd-1, PETSC_COMM_WORLD, ierr)
-
-    ! create new petsc vectors to perform math
-    call VecCreateMPI(PETSC_COMM_WORLD, ctx%loss%localn, PETSC_DECIDE, &
-         phiM, ierr)
-
-    ! calculate flux part of residual vector
-    call MatMult(ctx%loss%M, phi, phiM, ierr)
-    call MatMult(ctx%prod%F, phi, rphi, ierr)
-    call VecAYPX(rphi, -lambda, phiM, ierr)
-
-    ! set eigenvalue part of residual vector
-    call VecDot(phi, phi, reslamb, ierr)
-
-    ! map to ptr
-    if (rank == n_procs_cmfd - 1) rptr(size(rptr)) = 0.5_8 - 0.5_8*reslamb
-
-    ! reset arrays that are not used
-    call VecResetArray(phi, ierr)
-    call VecResetArray(rphi, ierr)
-
-    ! restore arrays for residual and solution
-    call VecRestoreArrayF90(x, xptr, ierr)
-    call VecRestoreArrayF90(res, rptr, ierr)
-
-    ! destroy all temp vectors
-    call VecDestroy(phi, ierr)
-    call VecDestroy(phiM, ierr)
-    call VecDestroy(rphi, ierr)
-
-    ! nullify all pointers
-    if (associated(xptr)) nullify(xptr)
-    if (associated(rptr)) nullify(rptr)
-
-    ! write out matrix in binary file (debugging)
-    if (cmfd_write_matrices) then
-      if (adjoint_calc) then
-        call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'adj_res.bin', &
-             FILE_MODE_WRITE, viewer, ierr)
-      else
-        call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'res.bin', &
-             FILE_MODE_WRITE, viewer, ierr)
-      end if
-      call VecView(res, viewer, ierr)
-      call PetscViewerDestroy(viewer, ierr)
-
-      if (adjoint_calc) then
-        call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'adj_x.bin', &
-             FILE_MODE_WRITE, viewer, ierr)
-      else
-        call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'x.bin', &
-             FILE_MODE_WRITE, viewer, ierr)
-      end if
-       call VecView(x, viewer, ierr)
-       call PetscViewerDestroy(viewer, ierr)
-
-    end if
-
-  end subroutine compute_nonlinear_residual
-
-!===============================================================================
-! COMPUTE_ADJOINT
-!===============================================================================
-
-  subroutine compute_adjoint()
-
-    use global,  only: cmfd_write_matrices
-
-    PetscViewer :: viewer
-
-    ! transpose matrices
-    call MatTranspose(ctx%loss%M, MAT_REUSE_MATRIX, ctx%loss%M, ierr)
-    call MatTranspose(ctx%prod%F, MAT_REUSE_MATRIX, ctx%prod%F, ierr)
-
-    ! write out matrix in binary file (debugging)
-    if (cmfd_write_matrices) then
-      call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'adj_lossmat.bin', &
-           FILE_MODE_WRITE, viewer, ierr)
-      call MatView(ctx%loss%M, viewer, ierr)
-      call PetscViewerDestroy(viewer, ierr)
-
-      call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'adj_prodmat.bin', &
-           FILE_MODE_WRITE, viewer, ierr)
-      call MatView(ctx%prod%F, viewer, ierr)
-      call PetscViewerDestroy(viewer, ierr)
-    end if
-
-  end subroutine compute_adjoint
-
-!===============================================================================
-! EXTRACT_RESULTS 
-!===============================================================================
-
-  subroutine extract_results()
-
-    use constants, only: ZERO, ONE
-    use global,    only: cmfd, n_procs_cmfd, rank
-
-    integer              :: n         ! problem size
-    integer              :: row_start ! local row start
-    integer              :: row_end   ! local row end
-    real(8)              :: keff      ! keff of problem
-    real(8),allocatable  :: mybuf(:)  ! temp buffer
-    PetscScalar, pointer :: xptr(:)   ! pointer to eigenvector info
-
-    ! get problem size
-    n = ctx%loss%n
-
-    ! also allocate in cmfd object
-    if (adjoint_calc) then
-      if (.not. allocated(cmfd%adj_phi)) allocate(cmfd%adj_phi(n))
-    else
-      if (.not. allocated(cmfd%phi)) allocate(cmfd%phi(n))
-    end if
-    if (.not. allocated(mybuf)) allocate(mybuf(n))
-
-    ! get ownership range
-    call VecGetOwnershipRange(xvec, row_start, row_end, ierr)
-
-    ! resize the last proc
-    if (rank == n_procs_cmfd - 1) row_end = row_end - 1
-
-    ! convert petsc phi_object to cmfd_obj
-    call VecGetArrayF90(xvec, xptr, ierr)
-    if (adjoint_calc) then
-      cmfd%adj_phi(row_start+1:row_end) = xptr(1:row_end - row_start)
-    else
-      cmfd%phi(row_start+1:row_end) = xptr(1:row_end - row_start) 
-    end if
-
-    ! reduce result to all 
-    mybuf = ZERO
-    if (adjoint_calc) then
-      call MPI_ALLREDUCE(cmfd%adj_phi, mybuf, n, MPI_REAL8, MPI_SUM, &
-           PETSC_COMM_WORLD, ierr)
-    else
-      call MPI_ALLREDUCE(cmfd%phi, mybuf, n, MPI_REAL8, MPI_SUM, &
-           PETSC_COMM_WORLD, ierr)
-    end if
-
-    ! move buffer to object and deallocate
-    cmfd%phi = mybuf
-    if(allocated(mybuf)) deallocate(mybuf)
-
-    ! save eigenvalue
-    if(rank == n_procs_cmfd - 1) keff = ONE / xptr(size(xptr))
-    if (adjoint_calc) then
-      cmfd%adj_keff = keff
-      call MPI_BCAST(cmfd%adj_keff, 1, MPI_REAL8, n_procs_cmfd-1, &
-           PETSC_COMM_WORLD, ierr)
-    else
-      cmfd%keff = keff
-      call MPI_BCAST(cmfd%keff, 1, MPI_REAL8, n_procs_cmfd-1, &
-           PETSC_COMM_WORLD, ierr)
-    end if
-    call VecRestoreArrayF90(xvec, xptr, ierr)
-
-    ! nullify pointers and deallocate local variables
-    if (associated(xptr)) nullify(xptr)
-    if (allocated(mybuf)) deallocate(mybuf)
-
-  end subroutine extract_results
-
-!===============================================================================
-! FINALIZE 
-!===============================================================================
-
-  subroutine finalize()
-
-    ! finalize data objects
-    call destroy_M_operator(ctx%loss)
-    call destroy_F_operator(ctx%prod)
-    call destroy_J_operator(jac_prec)
-    call VecDestroy(xvec, ierr)
-    call VecDestroy(resvec, ierr)
-    call MatDestroy(jac, ierr)
-
-    ! finalize solver objects
-    call SNESDestroy(snes, ierr)
-
-  end subroutine finalize
-
-# endif
-
-end module cmfd_snes_solver
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index 3abe276d71..dd62c42a6d 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -18,6 +18,7 @@ module matrix_header
 #  ifdef PETSC
     Mat :: petsc_mat
 #  endif
+    logical :: petsc_active = .false.
    contains
      procedure :: create       => matrix_create
      procedure :: destroy      => matrix_destroy
@@ -223,6 +224,7 @@ contains
 #ifdef PETSC
     call MatCreateSeqAIJWithArrays(PETSC_COMM_WORLD, self % n, self % n, &
             self % row, self % col, self % val, self % petsc_mat, petsc_err)
+    self % petsc_active = .true.
 #endif
 
   end subroutine matrix_setup_petsc
@@ -271,18 +273,44 @@ contains
 
   subroutine matrix_vector_multiply(self, vec_in, vec_out)
 
+    use constants,      only: ZERO
     use vector_header,  only: Vector
 
     class(Matrix) :: self
     type(Vector)  :: vec_in
     type(Vector)  :: vec_out
 
-    integer :: petsc_err
+    integer :: i
+    integer :: j
+    integer :: shift
 
-#ifdef PETSC
-    call MatMult(self % petsc_mat, vec_in % petsc_vec, vec_out % petsc_vec, &
-                 petsc_err)
-#endif
+!    integer :: petsc_err
+
+!#ifdef PETSC
+!     call MatMult(self % petsc_mat, vec_in % petsc_vec, vec_out % petsc_vec, &
+!                 petsc_err)
+!#endif
+
+    ! set shift by default 0 and change if petsc is active
+    ! this is because PETSc needs vectors to remain referenced to 0
+    shift = 0
+    if (self % petsc_active) shift = 1
+
+    ! begin loop around rows
+    ROWS: do i = 1, vec_in % n
+
+      ! initialize target location in vector
+      vec_out % val(i) = ZERO
+
+      ! begin loop around columns (need to shift if petsc is active)
+      COLS: do j = self % row(i) + shift, self % row(i + 1) - 1 + shift
+
+        vec_out % val(i) = vec_out % val(i) + self % val(j) * &
+                           vec_in % val(self % col(j) + shift)
+
+      end do COLS
+
+    end do ROWS
 
   end subroutine matrix_vector_multiply
 
diff --git a/src/petsc_solver.F90 b/src/petsc_solver.F90
index 446e12732a..7f4c2e78ad 100644
--- a/src/petsc_solver.F90
+++ b/src/petsc_solver.F90
@@ -10,6 +10,7 @@ module petsc_solver
 #  include <finclude/petsc.h90>
 #endif
 
+  ! Petsc GMRES solver context
   type, public :: Petsc_gmres 
 #ifdef PETSC
     KSP :: ksp
@@ -23,8 +24,40 @@ module petsc_solver
      procedure :: solve        => gmres_solve
   end type Petsc_gmres
 
+  ! Derived type to contain list of data needed to be passed to procedures
+  type, public :: Jfnk_ctx
+    type(Matrix) :: loss
+    type(Matrix) :: prod
+    procedure (res_interface), pointer, nopass :: res_proc 
+    procedure (jac_interface), pointer, nopass :: jac_proc
+  end type Jfnk_ctx
+
+  ! Petsc SNES JFNK solver context
+  type, public :: Petsc_jfnk
+#ifdef PETSC
+    KSP  :: ksp
+    PC   :: pc
+    SNES :: snes
+    Mat  :: jac_mf
+#endif
+   contains
+     procedure :: create  => jfnk_create
+     procedure :: destroy => jfnk_destroy
+  end type Petsc_jfnk
+
   integer :: Petsc_err
 
+  ! Abstract interface stating how jacobian and residual routines look
+  abstract interface
+    subroutine res_interface(x)
+      real(8) :: x
+    end subroutine res_interface
+
+    subroutine jac_interface(x)
+      real(8) :: x
+    end subroutine jac_interface
+  end interface
+
 contains
 
 !===============================================================================
@@ -55,14 +88,14 @@ contains
 ! GMRES_PRECONDITION
 !===============================================================================
 
-  subroutine gmres_precondition(self, mat)
+  subroutine gmres_precondition(self, mat_in)
 
     class(Petsc_gmres) :: self
-    type(Matrix)       :: mat
+    type(Matrix)       :: mat_in
 
 #ifdef PETSC
     call KSPSetUp(self % ksp, petsc_err) 
-    call PCFactorGetMatrix(self % pc, mat % petsc_mat, petsc_err) 
+    call PCFactorGetMatrix(self % pc, mat_in % petsc_mat, petsc_err) 
 #endif
 
   end subroutine gmres_precondition
@@ -71,14 +104,14 @@ contains
 ! GMRES_SET_OPER
 !===============================================================================
 
-  subroutine gmres_set_oper(self, prec_mat, mat)
+  subroutine gmres_set_oper(self, prec_mat, mat_in)
 
     class(Petsc_gmres) :: self
     type(Matrix)       :: prec_mat
-    type(Matrix)       :: mat
+    type(Matrix)       :: mat_in
 
 #ifdef PETSC
-    call KSPSetOperators(self % ksp, mat % petsc_mat, prec_mat % petsc_mat, &
+    call KSPSetOperators(self % ksp, mat_in % petsc_mat, prec_mat % petsc_mat, &
          SAME_NONZERO_PATTERN, petsc_err)
 #endif
 
@@ -116,4 +149,105 @@ contains
 
   end subroutine gmres_solve
 
+!===============================================================================
+! JFNK_CREATE
+!===============================================================================
+
+  subroutine jfnk_create(self)
+
+    class(Petsc_jfnk) :: self
+
+#ifdef PETSC
+    ! Turn on mf_operator option for matrix free jacobian
+    call PetscOptionsSetValue("-snes_mf_operator", "TRUE", petsc_err)
+
+    ! Create the SNES context
+    call SNESCreate(PETSC_COMM_WORLD, self % snes, petsc_err)
+
+    ! Set up the GMRES solver
+    call SNESGetKSP(self % snes, self % ksp, petsc_err)
+    call KSPSetType(self % ksp, KSPGMRES, petsc_err)    
+
+    ! Create the matrix free jacobian
+    call MatCreateSNESMF(self % snes, self % jac_mf, petsc_err)
+
+    ! Set up Jacobian Lags
+    call SNESSetLagJacobian(self % snes, -2, petsc_err)
+    call SNESSetLagPreconditioner(self % snes, -1, petsc_err)
+
+    ! Set up preconditioner
+    call KSPGetPC(self % ksp, self % pc, petsc_err)
+    call PCSetType(self % pc, PCILU, petsc_err)
+    call PCFactorSetLevels(self % pc, 5, petsc_err)
+#endif
+
+  end subroutine jfnk_create
+
+!===============================================================================
+! JFNK_DESTROY
+!===============================================================================
+
+  subroutine jfnk_destroy(self)
+
+    class(Petsc_jfnk) :: self
+
+#ifdef PETSC
+    call MatDestroy(self % jac_mf, petsc_err)
+    call SNESDestroy(self % snes, petsc_err)
+#endif
+
+  end subroutine jfnk_destroy
+
+!===============================================================================
+! JFNK_SET_FUNCTIONS
+!===============================================================================
+
+  subroutine jfnk_set_functions(self, ctx, res, jac_prec)
+
+    class(Petsc_jfnk) :: self
+    type(Jfnk_ctx)    :: ctx
+    type(Vector)      :: res
+    type(Matrix)      :: jac_prec
+
+    ! Set residual procedure
+    call SNESSetFunction(self % snes, res % petsc_vec, jfnk_compute_residual, &
+         ctx, petsc_err)
+
+    ! Set Jacobian procedure
+    call SNESSetJacobian(self % snes, self % jac_mf, jac_prec % petsc_mat, &
+         jfnk_compute_jacobian, ctx, petsc_err)
+
+  end subroutine jfnk_set_functions
+
+!===============================================================================
+! JFNK_COMPUTE_RESIDUAL
+!===============================================================================
+
+  subroutine jfnk_compute_residual(snes, x, res, ctx, petsc_err)
+
+    SNES           :: snes
+    Vec            :: x
+    Vec            :: res
+    integer        :: petsc_err
+    type(Jfnk_ctx) :: ctx
+
+  end subroutine jfnk_compute_residual
+
+!===============================================================================
+! JFNK_COMPUTE_JACOBIAN
+!===============================================================================
+
+  subroutine jfnk_compute_jacobian(snes, x, jac_mf, jac_prec, flag, ctx, &
+             petsc_err)
+
+    SNES           :: snes
+    Vec            :: x
+    Mat            :: jac_mf
+    Mat            :: jac_prec
+    MatStructure   :: flag 
+    type(Jfnk_ctx) :: ctx
+    integer        :: petsc_err
+
+  end subroutine jfnk_compute_jacobian
+
 end module petsc_solver
diff --git a/src/vector_header.F90 b/src/vector_header.F90
index ec9427e916..1a867ef5c6 100644
--- a/src/vector_header.F90
+++ b/src/vector_header.F90
@@ -11,7 +11,7 @@ module vector_header
 
   type, public :: Vector 
     integer :: n        ! number of rows/cols in matrix
-    real(8), allocatable :: val(:) ! matrix value vector
+    real(8), pointer :: val(:) ! matrix value vector
 #  ifdef PETSC
     Vec :: petsc_vec
 #  endif
@@ -35,7 +35,7 @@ contains
     class(Vector) :: self
 
     ! preallocate vector
-    if (.not.allocated(self % val)) allocate(self % val(n))
+    if (.not.associated(self % val)) allocate(self % val(n))
 
     ! set n
     self % n = n
@@ -53,7 +53,7 @@ contains
 
     class(Vector) :: self
 
-    if (allocated(self % val)) deallocate(self % val)
+    if (associated(self % val)) deallocate(self % val)
 
   end subroutine vector_destroy
 

From 657d8cd532c16801b9798addc643ad6a97b1bb28 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Thu, 11 Jul 2013 12:03:21 -0400
Subject: [PATCH 21/69] Fixed bugs in JFNK but currently does Full Newton
 Iterations

---
 src/cmfd_jfnk_solver.F90 | 148 +++++++++++++++++++++++++++------------
 src/matrix_header.F90    |   7 +-
 src/petsc_solver.F90     | 136 ++++++++++++++++++++++++++---------
 3 files changed, 212 insertions(+), 79 deletions(-)

diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
index acba55b84b..7b6ce7dbb0 100644
--- a/src/cmfd_jfnk_solver.F90
+++ b/src/cmfd_jfnk_solver.F90
@@ -11,14 +11,14 @@ module cmfd_jfnk_solver
   private
   public :: cmfd_jfnk_execute
 
-  logical          :: adjoint_calc = .false. ! adjoint calculation
+  logical          :: adjoint_calc
   type(Jfnk_ctx)   :: jfnk_data
-  type(Matrix)     :: jac_prec
+  type(Matrix), target     :: jac_prec
   type(Matrix)     :: loss
   type(Matrix)     :: prod
   type(Petsc_jfnk) :: jfnk 
-  type(Vector)     :: resvec
-  type(Vector)     :: xvec
+  type(Vector), target     :: resvec
+  type(Vector), target     :: xvec
 
 contains
 
@@ -31,6 +31,7 @@ contains
     logical, optional :: adjoint ! adjoint calculation
 
     ! check for adjoint
+    adjoint_calc = .false.
     if (present(adjoint)) adjoint_calc = adjoint
 
     ! seed with power iteration
@@ -42,8 +43,13 @@ contains
     ! initialize solver
     call jfnk % create()
 
+    ! set up residual and jacobian routines
+    jfnk_data % res_proc_ptr => compute_nonlinear_residual
+    jfnk_data % jac_proc_ptr => build_jacobian_matrix
+    call jfnk % set_functions(jfnk_data, resvec, jac_prec)
+
     ! solve the system
-!   call SNESSolve(snes, PETSC_NULL_DOUBLE, xvec, ierr)
+    call jfnk % solve(xvec)
 
     ! extracts results to cmfd object
     call extract_results()
@@ -60,24 +66,46 @@ contains
   subroutine init_data()
 
     use constants, only: ONE
-    use global,    only: cmfd
-
-    integer              :: n         ! problem size
+    use global,    only: cmfd, cmfd_adjoint_type
 
+    logical :: physical_adjoint
+    integer :: n
+    type(Vector), pointer :: x
+    type(Vector), pointer :: res
+    type(Matrix), pointer :: jac
+    x => xvec
+    res => resvec
+    jac => jac_prec
     ! Set up all matrices
     call init_loss_matrix(loss)
     call init_prod_matrix(prod)
     call init_jacobian_matrix()
 
-    ! Set up for use with petsc
-    call jac_prec % setup_petsc()
+    ! Check for physical adjoint
+    physical_adjoint = .false.
+    if (adjoint_calc .and. trim(cmfd_adjoint_type) == 'physical') &
+         physical_adjoint = .true.
+
+    ! Create matrix operators
+    call build_loss_matrix(loss, adjoint = physical_adjoint)
+    call build_prod_matrix(prod, adjoint = physical_adjoint)
+
+    ! Assembly matrices and use petsc
+    call loss % assemble()
+    call prod % assemble()
+    call loss % setup_petsc()
+    call prod % setup_petsc()
+
+    ! Check for mathematical adjoint
+    if (adjoint_calc .and. trim(cmfd_adjoint_type) == 'math') &
+         call compute_adjoint()
 
     ! Get problem size
-    n = jac_prec % n
+    n = loss % n
 
     ! Create problem vectors
-    call resvec % create(n)
-    call xvec % create(n)
+    call resvec % create(n + 1)
+    call xvec % create(n + 1)
 
     ! set flux in guess
     if (adjoint_calc) then
@@ -93,7 +121,21 @@ contains
       xvec % val(n + 1) = ONE/cmfd % keff
     end if
 
-    ! set solver names
+    ! Set up vectors for petsc
+    call resvec   % setup_petsc()
+    call xvec     % setup_petsc()
+
+    ! Build jacobian from initial guess
+    call build_jacobian_matrix(xvec)
+
+    ! Set up Jacobian for Petsc
+    call jac_prec % setup_petsc()
+
+    ! write all out
+    call loss % write_petsc_binary('loss.bin')
+    call prod % write_petsc_binary('prod.bin')
+    call jac_prec % write_petsc_binary('jac.bin')
+!   call xvec % write_petsc_binary('x.bin')
 
   end subroutine init_data
 
@@ -136,17 +178,22 @@ contains
     integer      :: jloss  ! loop counter for loss matrix cols
     integer      :: jprod  ! loop counter for prod matrix cols
     integer      :: n      ! problem size
+    integer      :: shift  ! integer to account for index shift for Petsc
     real(8)      :: lambda ! eigenvalue
     real(8)      :: val    ! temporary real scalar
     type(Vector) :: flux   ! flux vector
     type(Vector) :: fsrc   ! fission source vector
-
+print *, 'IN JACOBIAN'
     ! get the problem size
     n = loss % n
 
+    ! Check to use shift
+    shift = 0
+    if (loss % petsc_active) shift = 1
+
     ! get flux and eigenvalue
-    flux % val => x % val(1:n)
-    lambda = x % val(n + 1)
+    flux % val => xvec % val(1:n)
+    lambda = xvec % val(n + 1)
 
     ! create fission source vector
     call fsrc % create(n)
@@ -163,13 +210,15 @@ contains
       call jac_prec % new_row()
 
       ! begin loop around columns of loss matrix
-      COLS_LOSS: do jloss = loss % row(i), loss % row(i + 1) - 1
+      COLS_LOSS: do jloss = loss % row(i) + shift, &
+                            loss % row(i + 1) - 1 + shift
 
         ! start with the value in the loss matrix
         val = loss % val(jloss)
 
         ! loop around columns of prod matrix
-        COLS_PROD: do jprod = prod % row(i), prod % row(i + 1) - 1
+        COLS_PROD: do jprod = prod % row(i) + shift, &
+                              prod % row(i + 1) - 1 + shift
 
           ! see if columns agree with loss matrix
           if (prod % col(jprod) == loss % col(jloss)) then
@@ -180,12 +229,12 @@ contains
         end do COLS_PROD 
 
         ! record value in jacobian
-        call jac_prec % add_value(jloss, val)
+        call jac_prec % add_value(loss % col(jloss) + shift, val)
 
       end do COLS_LOSS
 
       ! add fission source value
-      val = fsrc % val(n)
+      val = -fsrc % val(i)
       call jac_prec % add_value(n + 1, val)
 
     end do ROWS
@@ -200,6 +249,15 @@ contains
     ! add unity on bottom right corner of matrix
     call jac_prec % add_value(n + 1, ONE)
 
+    ! CRS requires a final value in row
+    call jac_prec % new_row()
+
+    ! If the Jacobian is already associated with PETSc need to reshift row/col
+    if (jac_prec % petsc_active) then
+      jac_prec % row = jac_prec % row - 1
+      jac_prec % col = jac_prec % col - 1
+    end if
+
     ! free all allocated memory
     flux % val => null()
     call fsrc % destroy()
@@ -210,31 +268,27 @@ contains
 ! COMPUTE_NONLINEAR_RESIDUAL
 !===============================================================================
 
-  subroutine compute_nonlinear_residual(x)
+  subroutine compute_nonlinear_residual(x, res)
 
-    use global,       only: cmfd_write_matrices, cmfd_adjoint_type
+    use global,       only: cmfd_write_matrices
 
-    type(Vector)      :: x 
+    type(Vector) :: x
+    type(Vector) :: res
 
     character(len=25) :: filename
     integer           :: n
-    logical           :: physical_adjoint = .false.
     real(8)           :: lambda
     type(Vector)      :: res_loss
     type(Vector)      :: res_prod
     type(Vector)      :: flux
 
-    ! Check for physical adjoint
-    if (adjoint_calc .and. trim(cmfd_adjoint_type) == 'physical') &
-         physical_adjoint = .true.
+    type(Vector), pointer :: res_ptr
+    type(Vector), pointer :: x_ptr
 
-    ! Create operators
-    call build_loss_matrix(loss, adjoint = physical_adjoint)
-    call build_prod_matrix(prod, adjoint = physical_adjoint)
+    res_ptr => resvec
+    x_ptr => xvec
 
-    ! Check for adjoint
-    if (adjoint_calc .and. trim(cmfd_adjoint_type) == 'math') &
-         call compute_adjoint()
+print *, 'IN RESIDUAL'
 
     ! Get problem size
     n = loss % n
@@ -255,10 +309,10 @@ contains
     call prod % vector_multiply(flux, res_prod)
 
     ! Put flux component values in residual vector
-    resvec % val(1:n) = res_loss % val - lambda*res_prod % val
+    res % val(1:n) = res_loss % val - lambda*res_prod % val
 
     ! Put eigenvalue component in residual vector
-    resvec % val(n+1) = 0.5_8 - 0.5_8*sum(flux % val * flux % val)
+    res % val(n+1) = 0.5_8 - 0.5_8*sum(flux % val * flux % val)
 
     ! write out data in binary files (debugging)
     if (cmfd_write_matrices) then
@@ -269,7 +323,7 @@ contains
       else
         filename = 'res.bin'
       end if
-      call resvec % write_petsc_binary(filename)
+      call res % write_petsc_binary(filename)
 
       ! write out solution vector
       if (adjoint_calc) then
@@ -280,7 +334,8 @@ contains
       call x % write_petsc_binary(filename)
 
     end if
-
+!   print *, x % val
+!   print *, res % val
   end subroutine compute_nonlinear_residual
 
 !===============================================================================
@@ -313,9 +368,12 @@ contains
     use global,    only: cmfd
 
     integer :: n ! problem size
-
+call loss % write_petsc_binary('loss.mat')
+call prod % write_petsc_binary('prod.mat')
+call xvec % write_petsc_binary('x.bin')
+call resvec % write_petsc_binary('res.bin')
     ! get problem size
-    n = jfnk_data % loss % n
+    n = loss % n
 
     ! also allocate in cmfd object
     if (adjoint_calc) then
@@ -330,9 +388,9 @@ contains
       cmfd % adj_keff = ONE / xvec % val(n+1)
     else
       cmfd % phi  = xvec % val(1:n)
-      cmfd % keff = xvec % val(n+1)
+      cmfd % keff = ONE / xvec % val(n+1)
     end if
-
+stop
   end subroutine extract_results
 
 !===============================================================================
@@ -341,12 +399,14 @@ contains
 
   subroutine finalize()
 
-    call jfnk_data % loss % destroy()
-    call jfnk_data % prod % destroy()
+    call loss % destroy()
+    call prod % destroy()
     call jac_prec % destroy()
     call xvec % destroy()
     call resvec % destroy()
     call jfnk % destroy() 
+    nullify(jfnk_data % res_proc_ptr)
+    nullify(jfnk_data % jac_proc_ptr)
 
   end subroutine finalize
 
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index dd62c42a6d..f3ec97ecb5 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -18,7 +18,7 @@ module matrix_header
 #  ifdef PETSC
     Mat :: petsc_mat
 #  endif
-    logical :: petsc_active = .false.
+    logical :: petsc_active
    contains
      procedure :: create       => matrix_create
      procedure :: destroy      => matrix_destroy
@@ -56,6 +56,9 @@ contains
     self % n = n
     self % nnz = nnz
 
+    ! set petsc active by default to false
+    self % petsc_active = .false.
+
   end subroutine matrix_create
 
 !===============================================================================
@@ -297,7 +300,7 @@ contains
     if (self % petsc_active) shift = 1
 
     ! begin loop around rows
-    ROWS: do i = 1, vec_in % n
+    ROWS: do i = 1, self % n
 
       ! initialize target location in vector
       vec_out % val(i) = ZERO
diff --git a/src/petsc_solver.F90 b/src/petsc_solver.F90
index 7f4c2e78ad..f3de5e1f2c 100644
--- a/src/petsc_solver.F90
+++ b/src/petsc_solver.F90
@@ -26,10 +26,8 @@ module petsc_solver
 
   ! Derived type to contain list of data needed to be passed to procedures
   type, public :: Jfnk_ctx
-    type(Matrix) :: loss
-    type(Matrix) :: prod
-    procedure (res_interface), pointer, nopass :: res_proc 
-    procedure (jac_interface), pointer, nopass :: jac_proc
+    procedure (res_interface), pointer, nopass :: res_proc_ptr
+    procedure (jac_interface), pointer, nopass :: jac_proc_ptr
   end type Jfnk_ctx
 
   ! Petsc SNES JFNK solver context
@@ -39,22 +37,28 @@ module petsc_solver
     PC   :: pc
     SNES :: snes
     Mat  :: jac_mf
+    SNESLineSearch :: ls
 #endif
    contains
-     procedure :: create  => jfnk_create
-     procedure :: destroy => jfnk_destroy
+     procedure :: create        => jfnk_create
+     procedure :: destroy       => jfnk_destroy
+     procedure :: set_functions => jfnk_set_functions
+     procedure :: solve         => jfnk_solve
   end type Petsc_jfnk
 
-  integer :: Petsc_err
+  integer :: petsc_err
 
   ! Abstract interface stating how jacobian and residual routines look
   abstract interface
-    subroutine res_interface(x)
-      real(8) :: x
+    subroutine res_interface(x, r)
+      import :: Vector
+      type(Vector) :: x
+      type(Vector) :: r
     end subroutine res_interface
 
     subroutine jac_interface(x)
-      real(8) :: x
+      import :: Vector
+      type(Vector) :: x
     end subroutine jac_interface
   end interface
 
@@ -141,8 +145,6 @@ contains
     type(Vector)       :: b
     type(Vector)       :: x
 
-    integer :: petsc_err
-
 #ifdef PETSC
     call KSPSolve(self % ksp, b % petsc_vec, x % petsc_vec, petsc_err)
 #endif
@@ -159,26 +161,11 @@ contains
 
 #ifdef PETSC
     ! Turn on mf_operator option for matrix free jacobian
-    call PetscOptionsSetValue("-snes_mf_operator", "TRUE", petsc_err)
+!   call PetscOptionsSetValue("-snes_mf_operator", "TRUE", petsc_err)
 
     ! Create the SNES context
     call SNESCreate(PETSC_COMM_WORLD, self % snes, petsc_err)
 
-    ! Set up the GMRES solver
-    call SNESGetKSP(self % snes, self % ksp, petsc_err)
-    call KSPSetType(self % ksp, KSPGMRES, petsc_err)    
-
-    ! Create the matrix free jacobian
-    call MatCreateSNESMF(self % snes, self % jac_mf, petsc_err)
-
-    ! Set up Jacobian Lags
-    call SNESSetLagJacobian(self % snes, -2, petsc_err)
-    call SNESSetLagPreconditioner(self % snes, -1, petsc_err)
-
-    ! Set up preconditioner
-    call KSPGetPC(self % ksp, self % pc, petsc_err)
-    call PCSetType(self % pc, PCILU, petsc_err)
-    call PCFactorSetLevels(self % pc, 5, petsc_err)
 #endif
 
   end subroutine jfnk_create
@@ -213,24 +200,85 @@ contains
     call SNESSetFunction(self % snes, res % petsc_vec, jfnk_compute_residual, &
          ctx, petsc_err)
 
+    ! Set up the GMRES solver
+    call SNESGetKSP(self % snes, self % ksp, petsc_err)
+    call KSPSetType(self % ksp, KSPGMRES, petsc_err)    
+
+    ! Create the matrix free jacobian
+!   call MatCreateSNESMF(self % snes, self % jac_mf, petsc_err)
+
     ! Set Jacobian procedure
-    call SNESSetJacobian(self % snes, self % jac_mf, jac_prec % petsc_mat, &
+    call SNESSetJacobian(self % snes, jac_prec % petsc_mat, jac_prec % petsc_mat, &
          jfnk_compute_jacobian, ctx, petsc_err)
 
+    ! Set up Jacobian Lags
+!   call SNESSetLagJacobian(self % snes, -2, petsc_err)
+!   call SNESSetLagPreconditioner(self % snes, -1, petsc_err)
+
+    ! Apply options
+    call SNESGetLineSearch(self % snes, self % ls, petsc_err)
+    call SNESLineSearchSetType(self % ls, SNESLINESEARCHBASIC, petsc_err)
+    call SNESSetFromOptions(self % snes, petsc_err)
+    call SNESView(self % snes, PETSC_VIEWER_STDOUT_WORLD, petsc_err)
+
+    ! Set up preconditioner
+    call KSPGetPC(self % ksp, self % pc, petsc_err)
+    call PCSetType(self % pc, PCILU, petsc_err)
+    call PCFactorSetLevels(self % pc, 5, petsc_err)
+    call PCSetFromOptions(self % pc, petsc_err)
+    call KSPSetFromOptions(self % ksp, petsc_err)
+
   end subroutine jfnk_set_functions
 
+!===============================================================================
+! JFNK_SOLVE
+!===============================================================================
+
+  subroutine jfnk_solve(self, xvec)
+
+    class(Petsc_jfnk) :: self
+    type(Vector)      :: xvec
+
+#ifdef PETSC
+    call SNESSolve(self % snes, PETSC_NULL_DOUBLE, xvec % petsc_vec, petsc_err)
+#endif
+
+  end subroutine jfnk_solve
+
 !===============================================================================
 ! JFNK_COMPUTE_RESIDUAL
 !===============================================================================
 
-  subroutine jfnk_compute_residual(snes, x, res, ctx, petsc_err)
+  subroutine jfnk_compute_residual(snes, x, res, ctx, ierr)
 
     SNES           :: snes
     Vec            :: x
     Vec            :: res
-    integer        :: petsc_err
+    integer        :: ierr
     type(Jfnk_ctx) :: ctx
 
+    PetscViewer ::viewer
+
+    type(Vector) :: xvec
+    type(Vector) :: resvec
+
+    call VecGetArrayF90(x, xvec % val, ierr)
+    call VecGetArrayF90(res, resvec % val, ierr)
+
+    call ctx % res_proc_ptr(xvec, resvec)
+!   call VecView(x, PETSC_VIEWER_STDOUT_WORLD, ierr)
+!   call VecView(res, PETSC_VIEWER_STDOUT_WORLD, ierr)
+
+!   call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'res.bin', FILE_MODE_WRITE, viewer, ierr)
+!   call VecView(res, viewer, ierr)
+!   call PetscViewerDestroy(viewer, ierr)
+!   call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'x.bin', FILE_MODE_WRITE, viewer, ierr)
+!   call VecView(x, viewer, ierr)
+!   call PetscViewerDestroy(viewer, ierr)
+
+    call VecRestoreArrayF90(x, xvec % val, ierr)
+    call VecRestoreArrayF90(res, resvec % val, ierr)
+    read *
   end subroutine jfnk_compute_residual
 
 !===============================================================================
@@ -238,7 +286,7 @@ contains
 !===============================================================================
 
   subroutine jfnk_compute_jacobian(snes, x, jac_mf, jac_prec, flag, ctx, &
-             petsc_err)
+             ierr)
 
     SNES           :: snes
     Vec            :: x
@@ -246,7 +294,29 @@ contains
     Mat            :: jac_prec
     MatStructure   :: flag 
     type(Jfnk_ctx) :: ctx
-    integer        :: petsc_err
+    integer        :: ierr
+
+    type(Vector) :: xvec
+
+    PetscViewer :: viewer
+
+    call VecGetArrayF90(x, xvec % val, ierr)
+
+    call ctx % jac_proc_ptr(xvec)
+
+    call VecRestoreArrayF90(x, xvec % val, ierr)
+
+!   call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'jac.bin', FILE_MODE_WRITE, viewer, petsc_err)
+!   call MatView(jac_prec, viewer, petsc_err)
+!   call PetscViewerDestroy(viewer, petsc_err)
+
+!   call PetscViewerASCIIOpen(PETSC_COMM_WORLD, 'jac.out', viewer, petsc_err)
+!   call MatView(jac_prec, viewer, petsc_err)
+!   call PetscViewerDestroy(viewer, petsc_err)
+#ifdef PETSC
+!   call MatAssemblyBegin(jac_mf, MAT_FINAL_ASSEMBLY, petsc_err)
+!   call MatAssemblyEnd(jac_mf, MAT_FINAL_ASSEMBLY, petsc_err)
+#endif
 
   end subroutine jfnk_compute_jacobian
 

From 3a941886d96febfa1359783f88725b8a92005c87 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Thu, 11 Jul 2013 12:13:28 -0400
Subject: [PATCH 22/69] Put back in the Jacobian free mode in the JFNK solver

---
 src/cmfd_jfnk_solver.F90 |  6 ++---
 src/petsc_solver.F90     | 50 ++++++++++++++++++++--------------------
 2 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
index 7b6ce7dbb0..225e2b7211 100644
--- a/src/cmfd_jfnk_solver.F90
+++ b/src/cmfd_jfnk_solver.F90
@@ -183,7 +183,7 @@ contains
     real(8)      :: val    ! temporary real scalar
     type(Vector) :: flux   ! flux vector
     type(Vector) :: fsrc   ! fission source vector
-print *, 'IN JACOBIAN'
+!nprint *, 'IN JACOBIAN'
     ! get the problem size
     n = loss % n
 
@@ -288,7 +288,7 @@ print *, 'IN JACOBIAN'
     res_ptr => resvec
     x_ptr => xvec
 
-print *, 'IN RESIDUAL'
+!print *, 'IN RESIDUAL'
 
     ! Get problem size
     n = loss % n
@@ -390,7 +390,7 @@ call resvec % write_petsc_binary('res.bin')
       cmfd % phi  = xvec % val(1:n)
       cmfd % keff = ONE / xvec % val(n+1)
     end if
-stop
+
   end subroutine extract_results
 
 !===============================================================================
diff --git a/src/petsc_solver.F90 b/src/petsc_solver.F90
index f3de5e1f2c..3fd38a7f3d 100644
--- a/src/petsc_solver.F90
+++ b/src/petsc_solver.F90
@@ -161,11 +161,28 @@ contains
 
 #ifdef PETSC
     ! Turn on mf_operator option for matrix free jacobian
-!   call PetscOptionsSetValue("-snes_mf_operator", "TRUE", petsc_err)
+    call PetscOptionsSetValue("-snes_mf_operator", "TRUE", petsc_err)
 
     ! Create the SNES context
     call SNESCreate(PETSC_COMM_WORLD, self % snes, petsc_err)
 
+    ! Set up the GMRES solver
+    call SNESGetKSP(self % snes, self % ksp, petsc_err)
+    call KSPSetType(self % ksp, KSPGMRES, petsc_err)    
+
+    ! Apply options
+    call SNESGetLineSearch(self % snes, self % ls, petsc_err)
+    call SNESLineSearchSetType(self % ls, SNESLINESEARCHBASIC, petsc_err)
+    call SNESSetFromOptions(self % snes, petsc_err)
+!   call SNESView(self % snes, PETSC_VIEWER_STDOUT_WORLD, petsc_err)
+
+    ! Set up preconditioner
+    call KSPGetPC(self % ksp, self % pc, petsc_err)
+    call PCSetType(self % pc, PCILU, petsc_err)
+    call PCFactorSetLevels(self % pc, 5, petsc_err)
+    call PCSetFromOptions(self % pc, petsc_err)
+    call KSPSetFromOptions(self % ksp, petsc_err)
+
 #endif
 
   end subroutine jfnk_create
@@ -200,33 +217,16 @@ contains
     call SNESSetFunction(self % snes, res % petsc_vec, jfnk_compute_residual, &
          ctx, petsc_err)
 
-    ! Set up the GMRES solver
-    call SNESGetKSP(self % snes, self % ksp, petsc_err)
-    call KSPSetType(self % ksp, KSPGMRES, petsc_err)    
-
     ! Create the matrix free jacobian
-!   call MatCreateSNESMF(self % snes, self % jac_mf, petsc_err)
+    call MatCreateSNESMF(self % snes, self % jac_mf, petsc_err)
 
     ! Set Jacobian procedure
-    call SNESSetJacobian(self % snes, jac_prec % petsc_mat, jac_prec % petsc_mat, &
+    call SNESSetJacobian(self % snes, self % jac_mf, jac_prec % petsc_mat, &
          jfnk_compute_jacobian, ctx, petsc_err)
 
     ! Set up Jacobian Lags
-!   call SNESSetLagJacobian(self % snes, -2, petsc_err)
-!   call SNESSetLagPreconditioner(self % snes, -1, petsc_err)
-
-    ! Apply options
-    call SNESGetLineSearch(self % snes, self % ls, petsc_err)
-    call SNESLineSearchSetType(self % ls, SNESLINESEARCHBASIC, petsc_err)
-    call SNESSetFromOptions(self % snes, petsc_err)
-    call SNESView(self % snes, PETSC_VIEWER_STDOUT_WORLD, petsc_err)
-
-    ! Set up preconditioner
-    call KSPGetPC(self % ksp, self % pc, petsc_err)
-    call PCSetType(self % pc, PCILU, petsc_err)
-    call PCFactorSetLevels(self % pc, 5, petsc_err)
-    call PCSetFromOptions(self % pc, petsc_err)
-    call KSPSetFromOptions(self % ksp, petsc_err)
+    call SNESSetLagJacobian(self % snes, -2, petsc_err)
+    call SNESSetLagPreconditioner(self % snes, -1, petsc_err)
 
   end subroutine jfnk_set_functions
 
@@ -278,7 +278,7 @@ contains
 
     call VecRestoreArrayF90(x, xvec % val, ierr)
     call VecRestoreArrayF90(res, resvec % val, ierr)
-    read *
+   ! read *
   end subroutine jfnk_compute_residual
 
 !===============================================================================
@@ -314,8 +314,8 @@ contains
 !   call MatView(jac_prec, viewer, petsc_err)
 !   call PetscViewerDestroy(viewer, petsc_err)
 #ifdef PETSC
-!   call MatAssemblyBegin(jac_mf, MAT_FINAL_ASSEMBLY, petsc_err)
-!   call MatAssemblyEnd(jac_mf, MAT_FINAL_ASSEMBLY, petsc_err)
+    call MatAssemblyBegin(jac_mf, MAT_FINAL_ASSEMBLY, petsc_err)
+    call MatAssemblyEnd(jac_mf, MAT_FINAL_ASSEMBLY, petsc_err)
 #endif
 
   end subroutine jfnk_compute_jacobian

From f92dbfff7ddceb36b1dd8fa69837b870f5eb31a7 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Thu, 11 Jul 2013 12:24:26 -0400
Subject: [PATCH 23/69] cleaned up the JFNK routines

---
 src/cmfd_jfnk_solver.F90 | 36 +++++++++---------------------------
 src/petsc_solver.F90     | 38 +++++++++++++++-----------------------
 2 files changed, 24 insertions(+), 50 deletions(-)

diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
index 225e2b7211..ea0a7bb304 100644
--- a/src/cmfd_jfnk_solver.F90
+++ b/src/cmfd_jfnk_solver.F90
@@ -13,12 +13,12 @@ module cmfd_jfnk_solver
 
   logical          :: adjoint_calc
   type(Jfnk_ctx)   :: jfnk_data
-  type(Matrix), target     :: jac_prec
+  type(Matrix)     :: jac_prec
   type(Matrix)     :: loss
   type(Matrix)     :: prod
   type(Petsc_jfnk) :: jfnk 
-  type(Vector), target     :: resvec
-  type(Vector), target     :: xvec
+  type(Vector)     :: resvec
+  type(Vector)     :: xvec
 
 contains
 
@@ -70,12 +70,7 @@ contains
 
     logical :: physical_adjoint
     integer :: n
-    type(Vector), pointer :: x
-    type(Vector), pointer :: res
-    type(Matrix), pointer :: jac
-    x => xvec
-    res => resvec
-    jac => jac_prec
+
     ! Set up all matrices
     call init_loss_matrix(loss)
     call init_prod_matrix(prod)
@@ -135,7 +130,6 @@ contains
     call loss % write_petsc_binary('loss.bin')
     call prod % write_petsc_binary('prod.bin')
     call jac_prec % write_petsc_binary('jac.bin')
-!   call xvec % write_petsc_binary('x.bin')
 
   end subroutine init_data
 
@@ -183,7 +177,7 @@ contains
     real(8)      :: val    ! temporary real scalar
     type(Vector) :: flux   ! flux vector
     type(Vector) :: fsrc   ! fission source vector
-!nprint *, 'IN JACOBIAN'
+
     ! get the problem size
     n = loss % n
 
@@ -192,8 +186,8 @@ contains
     if (loss % petsc_active) shift = 1
 
     ! get flux and eigenvalue
-    flux % val => xvec % val(1:n)
-    lambda = xvec % val(n + 1)
+    flux % val => x % val(1:n)
+    lambda = x % val(n + 1)
 
     ! create fission source vector
     call fsrc % create(n)
@@ -282,14 +276,6 @@ contains
     type(Vector)      :: res_prod
     type(Vector)      :: flux
 
-    type(Vector), pointer :: res_ptr
-    type(Vector), pointer :: x_ptr
-
-    res_ptr => resvec
-    x_ptr => xvec
-
-!print *, 'IN RESIDUAL'
-
     ! Get problem size
     n = loss % n
 
@@ -334,8 +320,7 @@ contains
       call x % write_petsc_binary(filename)
 
     end if
-!   print *, x % val
-!   print *, res % val
+
   end subroutine compute_nonlinear_residual
 
 !===============================================================================
@@ -368,10 +353,7 @@ contains
     use global,    only: cmfd
 
     integer :: n ! problem size
-call loss % write_petsc_binary('loss.mat')
-call prod % write_petsc_binary('prod.mat')
-call xvec % write_petsc_binary('x.bin')
-call resvec % write_petsc_binary('res.bin')
+
     ! get problem size
     n = loss % n
 
diff --git a/src/petsc_solver.F90 b/src/petsc_solver.F90
index 3fd38a7f3d..97f1b05839 100644
--- a/src/petsc_solver.F90
+++ b/src/petsc_solver.F90
@@ -174,7 +174,6 @@ contains
     call SNESGetLineSearch(self % snes, self % ls, petsc_err)
     call SNESLineSearchSetType(self % ls, SNESLINESEARCHBASIC, petsc_err)
     call SNESSetFromOptions(self % snes, petsc_err)
-!   call SNESView(self % snes, PETSC_VIEWER_STDOUT_WORLD, petsc_err)
 
     ! Set up preconditioner
     call KSPGetPC(self % ksp, self % pc, petsc_err)
@@ -182,7 +181,6 @@ contains
     call PCFactorSetLevels(self % pc, 5, petsc_err)
     call PCSetFromOptions(self % pc, petsc_err)
     call KSPSetFromOptions(self % ksp, petsc_err)
-
 #endif
 
   end subroutine jfnk_create
@@ -257,28 +255,24 @@ contains
     integer        :: ierr
     type(Jfnk_ctx) :: ctx
 
-    PetscViewer ::viewer
-
     type(Vector) :: xvec
     type(Vector) :: resvec
 
+    ! Need to point an OpenMC vector to PETSc vector
+#ifdef PETSC
     call VecGetArrayF90(x, xvec % val, ierr)
     call VecGetArrayF90(res, resvec % val, ierr)
+#endif
 
+    ! Call user residual routine
     call ctx % res_proc_ptr(xvec, resvec)
-!   call VecView(x, PETSC_VIEWER_STDOUT_WORLD, ierr)
-!   call VecView(res, PETSC_VIEWER_STDOUT_WORLD, ierr)
-
-!   call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'res.bin', FILE_MODE_WRITE, viewer, ierr)
-!   call VecView(res, viewer, ierr)
-!   call PetscViewerDestroy(viewer, ierr)
-!   call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'x.bin', FILE_MODE_WRITE, viewer, ierr)
-!   call VecView(x, viewer, ierr)
-!   call PetscViewerDestroy(viewer, ierr)
 
+    ! Need to restore the PETSc vector
+#ifdef PETSC
     call VecRestoreArrayF90(x, xvec % val, ierr)
     call VecRestoreArrayF90(res, resvec % val, ierr)
-   ! read *
+#endif
+
   end subroutine jfnk_compute_residual
 
 !===============================================================================
@@ -298,21 +292,19 @@ contains
 
     type(Vector) :: xvec
 
-    PetscViewer :: viewer
-
+    ! Need to point OpenMC vector to PETSc Vector
+#ifdef PETSC
     call VecGetArrayF90(x, xvec % val, ierr)
+#endif
 
+    ! Evaluate user jacobian routine
     call ctx % jac_proc_ptr(xvec)
 
+    ! Restore the PETSc vector
+#ifdef PETSC
     call VecRestoreArrayF90(x, xvec % val, ierr)
+#endif
 
-!   call PetscViewerBinaryOpen(PETSC_COMM_WORLD, 'jac.bin', FILE_MODE_WRITE, viewer, petsc_err)
-!   call MatView(jac_prec, viewer, petsc_err)
-!   call PetscViewerDestroy(viewer, petsc_err)
-
-!   call PetscViewerASCIIOpen(PETSC_COMM_WORLD, 'jac.out', viewer, petsc_err)
-!   call MatView(jac_prec, viewer, petsc_err)
-!   call PetscViewerDestroy(viewer, petsc_err)
 #ifdef PETSC
     call MatAssemblyBegin(jac_mf, MAT_FINAL_ASSEMBLY, petsc_err)
     call MatAssemblyEnd(jac_mf, MAT_FINAL_ASSEMBLY, petsc_err)

From ef6425fa10e0a61bbc220fb4f16cc0188fdf8499 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Thu, 11 Jul 2013 15:32:38 -0400
Subject: [PATCH 24/69] fixed memory leaks in cmfd based routines using
 valgrind

---
 src/cmfd_power_solver.F90 |  2 +-
 src/matrix_header.F90     | 15 +++++++++------
 src/petsc_solver.F90      |  5 +++--
 src/vector_header.F90     | 16 +++++++++++++---
 4 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/src/cmfd_power_solver.F90 b/src/cmfd_power_solver.F90
index 60ba09f2c6..c63cff3205 100644
--- a/src/cmfd_power_solver.F90
+++ b/src/cmfd_power_solver.F90
@@ -70,7 +70,7 @@ contains
     call gmres % set_oper(loss, loss)
 
     ! Precondition matrix
-    call gmres % precondition(loss)
+!   call gmres % precondition(loss)
 
     ! Stop timer for build
     call time_cmfdbuild % stop()
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index f3ec97ecb5..b920e41bbd 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -31,6 +31,8 @@ module matrix_header
      procedure :: vector_multiply    => matrix_vector_multiply
   end type matrix
 
+  integer :: petsc_err
+
 contains
 
 !===============================================================================
@@ -69,6 +71,10 @@ contains
 
     class(Matrix) :: self
 
+#ifdef PETSC
+    if (self % petsc_active) call MatDestroy(self % petsc_mat, petsc_err)
+#endif
+
     if (allocated(self % row)) deallocate(self % row)
     if (allocated(self % col)) deallocate(self % col)
     if (allocated(self % val)) deallocate(self % val)
@@ -217,8 +223,6 @@ contains
 
     class(Matrix) :: self
 
-    integer :: petsc_err
-
     ! change indices to c notation
     self % row = self % row - 1
     self % col = self % col - 1
@@ -227,9 +231,11 @@ contains
 #ifdef PETSC
     call MatCreateSeqAIJWithArrays(PETSC_COMM_WORLD, self % n, self % n, &
             self % row, self % col, self % val, self % petsc_mat, petsc_err)
-    self % petsc_active = .true.
 #endif
 
+    ! Petsc is now active
+    self % petsc_active = .true.
+
   end subroutine matrix_setup_petsc
 
 !===============================================================================
@@ -241,7 +247,6 @@ contains
     character(*) :: filename
     class(Matrix) :: self
 
-    integer :: petsc_err
 #ifdef PETSC
     PetscViewer :: viewer
 
@@ -261,8 +266,6 @@ contains
 
     class(Matrix) :: self
 
-    integer :: petsc_err
-
 #ifdef PETSC
     call MatTranspose(self % petsc_mat, MAT_REUSE_MATRIX, self % petsc_mat, &
          petsc_err)
diff --git a/src/petsc_solver.F90 b/src/petsc_solver.F90
index 97f1b05839..49802a9dcd 100644
--- a/src/petsc_solver.F90
+++ b/src/petsc_solver.F90
@@ -84,6 +84,7 @@ contains
     call KSPSetInitialGuessNonzero(self % ksp, PETSC_TRUE, petsc_err)
     call KSPGetPC(self % ksp, self % pc, petsc_err)
     call PCFactorSetLevels(self % pc, ilu_levels, petsc_err)
+    call KSPSetFromOptions(self % ksp, petsc_err)
 #endif
 
   end subroutine gmres_create
@@ -98,8 +99,7 @@ contains
     type(Matrix)       :: mat_in
 
 #ifdef PETSC
-    call KSPSetUp(self % ksp, petsc_err) 
-    call PCFactorGetMatrix(self % pc, mat_in % petsc_mat, petsc_err) 
+!   call PCFactorGetMatrix(self % pc, mat_in % petsc_mat, petsc_err) 
 #endif
 
   end subroutine gmres_precondition
@@ -117,6 +117,7 @@ contains
 #ifdef PETSC
     call KSPSetOperators(self % ksp, mat_in % petsc_mat, prec_mat % petsc_mat, &
          SAME_NONZERO_PATTERN, petsc_err)
+    call KSPSetUp(self % ksp, petsc_err) 
 #endif
 
   end subroutine gmres_set_oper
diff --git a/src/vector_header.F90 b/src/vector_header.F90
index 1a867ef5c6..1c525fd2f8 100644
--- a/src/vector_header.F90
+++ b/src/vector_header.F90
@@ -15,6 +15,7 @@ module vector_header
 #  ifdef PETSC
     Vec :: petsc_vec
 #  endif
+    logical :: petsc_active
    contains
      procedure :: create       => vector_create
      procedure :: destroy      => vector_destroy
@@ -23,6 +24,8 @@ module vector_header
      procedure :: write_petsc_binary => vector_write_petsc_binary
   end type Vector
 
+  integer :: petsc_err
+
 contains
 
 !===============================================================================
@@ -43,6 +46,9 @@ contains
     ! initialize to zero
     self % val = ZERO
 
+    ! petsc is default not active
+    self % petsc_active = .false.
+
   end subroutine vector_create
 
 !===============================================================================
@@ -53,6 +59,10 @@ contains
 
     class(Vector) :: self
 
+#ifdef PETSC
+    if (self % petsc_active) call VecDestroy(self % petsc_vec, petsc_err)
+#endif
+
     if (associated(self % val)) deallocate(self % val)
 
   end subroutine vector_destroy
@@ -79,14 +89,15 @@ contains
 
     class(Vector) :: self
 
-    integer :: petsc_err
-
     ! link to petsc
 #ifdef PETSC
     call VecCreateSeqWithArray(PETSC_COMM_WORLD, 1, self % n, self % val, &
          self % petsc_vec, petsc_err) 
 #endif
 
+    ! set that petsc is now active
+    self % petsc_active = .true.
+
   end subroutine vector_setup_petsc
 
 !===============================================================================
@@ -98,7 +109,6 @@ contains
     character(*) :: filename
     class(Vector) :: self
 
-    integer :: petsc_err
 #ifdef PETSC
     PetscViewer :: viewer
 

From abd7e4a49ce4064eada501e495413f0df04360e7 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Thu, 11 Jul 2013 15:53:56 -0400
Subject: [PATCH 25/69] cleaned up and added comments to cmfd code

---
 src/cmfd_jfnk_solver.F90  | 93 ++++++++++++++++++---------------------
 src/cmfd_power_solver.F90 | 53 +++++++++++-----------
 src/matrix_header.F90     | 57 +++++++++++-------------
 src/petsc_solver.F90      | 49 +++++++++------------
 src/vector_header.F90     | 16 +++----
 5 files changed, 122 insertions(+), 146 deletions(-)

diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
index ea0a7bb304..54c26326b5 100644
--- a/src/cmfd_jfnk_solver.F90
+++ b/src/cmfd_jfnk_solver.F90
@@ -11,50 +11,50 @@ module cmfd_jfnk_solver
   private
   public :: cmfd_jfnk_execute
 
-  logical          :: adjoint_calc
-  type(Jfnk_ctx)   :: jfnk_data
-  type(Matrix)     :: jac_prec
-  type(Matrix)     :: loss
-  type(Matrix)     :: prod
-  type(Petsc_jfnk) :: jfnk 
-  type(Vector)     :: resvec
-  type(Vector)     :: xvec
+  logical          :: adjoint_calc ! true if an adjoint is to be calculated
+  type(Jfnk_ctx)   :: jfnk_data    ! object that holds pointers to user routines
+  type(Matrix)     :: jac_prec     ! Jacobian preconditioner object
+  type(Matrix)     :: loss         ! CMFD loss matrix
+  type(Matrix)     :: prod         ! CMFD production matrix
+  type(Petsc_jfnk) :: jfnk         ! JFNK solver object
+  type(Vector)     :: resvec       ! JFNK residual vector
+  type(Vector)     :: xvec         ! JFNK solution vector
 
 contains
 
 !===============================================================================
-! CMFD_JFNK_EXECUTE
+! CMFD_JFNK_EXECUTE main routine for JFNK solver
 !===============================================================================
 
   subroutine cmfd_jfnk_execute(adjoint)
 
     logical, optional :: adjoint ! adjoint calculation
 
-    ! check for adjoint
+    ! Check for adjoint
     adjoint_calc = .false.
     if (present(adjoint)) adjoint_calc = adjoint
 
-    ! seed with power iteration
+    ! Seed with power iteration to help converge to fundamental mode
     call cmfd_power_execute(k_tol=1.E-3_8, s_tol=1.E-3_8, adjoint=adjoint_calc)
 
-    ! initialize data
+    ! Initialize data
     call init_data()
 
-    ! initialize solver
+    ! Initialize solver
     call jfnk % create()
 
-    ! set up residual and jacobian routines
+    ! Set up residual and jacobian routines
     jfnk_data % res_proc_ptr => compute_nonlinear_residual
     jfnk_data % jac_proc_ptr => build_jacobian_matrix
     call jfnk % set_functions(jfnk_data, resvec, jac_prec)
 
-    ! solve the system
+    ! Solve the system
     call jfnk % solve(xvec)
 
-    ! extracts results to cmfd object
+    ! Extracts results to cmfd object
     call extract_results()
 
-    ! deallocate all slepc data
+    ! Deallocate all slepc data
     call finalize()
 
   end subroutine cmfd_jfnk_execute
@@ -102,14 +102,14 @@ contains
     call resvec % create(n + 1)
     call xvec % create(n + 1)
 
-    ! set flux in guess
+    ! Set flux in guess
     if (adjoint_calc) then
       xvec % val(1:n) = cmfd % adj_phi
     else
       xvec % val(1:n) = cmfd % phi
     end if
 
-    ! set keff in guess
+    ! Set keff in guess
     if (adjoint_calc) then
       xvec % val(n + 1) = ONE/cmfd % adj_keff
     else
@@ -126,11 +126,6 @@ contains
     ! Set up Jacobian for Petsc
     call jac_prec % setup_petsc()
 
-    ! write all out
-    call loss % write_petsc_binary('loss.bin')
-    call prod % write_petsc_binary('prod.bin')
-    call jac_prec % write_petsc_binary('jac.bin')
-
   end subroutine init_data
 
 !==============================================================================
@@ -178,43 +173,43 @@ contains
     type(Vector) :: flux   ! flux vector
     type(Vector) :: fsrc   ! fission source vector
 
-    ! get the problem size
+    ! Get the problem size
     n = loss % n
 
     ! Check to use shift
     shift = 0
     if (loss % petsc_active) shift = 1
 
-    ! get flux and eigenvalue
+    ! Get flux and eigenvalue
     flux % val => x % val(1:n)
     lambda = x % val(n + 1)
 
-    ! create fission source vector
+    ! Create fission source vector
     call fsrc % create(n)
     call prod % vector_multiply(flux, fsrc)
 
-    ! reset counters in Jacobian matrix
+    ! Reset counters in Jacobian matrix
     jac_prec % n_kount  = 1
     jac_prec % nz_kount = 1
 
-    ! begin loop around rows of Jacobian matrix
+    ! Begin loop around rows of Jacobian matrix
     ROWS: do i = 1, n
 
-      ! add a new row in Jacobian
+      ! Add a new row in Jacobian
       call jac_prec % new_row()
 
-      ! begin loop around columns of loss matrix
+      ! Begin loop around columns of loss matrix
       COLS_LOSS: do jloss = loss % row(i) + shift, &
                             loss % row(i + 1) - 1 + shift
 
-        ! start with the value in the loss matrix
+        ! Start with the value in the loss matrix
         val = loss % val(jloss)
 
-        ! loop around columns of prod matrix
+        ! Loop around columns of prod matrix
         COLS_PROD: do jprod = prod % row(i) + shift, &
                               prod % row(i + 1) - 1 + shift
 
-          ! see if columns agree with loss matrix
+          ! See if columns agree with loss matrix
           if (prod % col(jprod) == loss % col(jloss)) then
             val = val - lambda*prod % val(jprod)
             exit
@@ -222,25 +217,25 @@ contains
 
         end do COLS_PROD 
 
-        ! record value in jacobian
+        ! Record value in jacobian
         call jac_prec % add_value(loss % col(jloss) + shift, val)
 
       end do COLS_LOSS
 
-      ! add fission source value
+      ! Add fission source value
       val = -fsrc % val(i)
       call jac_prec % add_value(n + 1, val)
 
     end do ROWS
 
-    ! need to add negative transpose of flux vector on last row
+    ! Need to add negative transpose of flux vector on last row
     call jac_prec % new_row()
     do jjac = 1, n
       val = -flux % val(jjac)
       call jac_prec % add_value(jjac, val)
     end do
 
-    ! add unity on bottom right corner of matrix
+    ! Add unity on bottom right corner of matrix
     call jac_prec % add_value(n + 1, ONE)
 
     ! CRS requires a final value in row
@@ -252,14 +247,14 @@ contains
       jac_prec % col = jac_prec % col - 1
     end if
 
-    ! free all allocated memory
+    ! Free all allocated memory
     flux % val => null()
     call fsrc % destroy()
 
   end subroutine build_jacobian_matrix
 
 !===============================================================================
-! COMPUTE_NONLINEAR_RESIDUAL
+! COMPUTE_NONLINEAR_RESIDUAL computes the residual of the nonlinear equations
 !===============================================================================
 
   subroutine compute_nonlinear_residual(x, res)
@@ -300,10 +295,10 @@ contains
     ! Put eigenvalue component in residual vector
     res % val(n+1) = 0.5_8 - 0.5_8*sum(flux % val * flux % val)
 
-    ! write out data in binary files (debugging)
+    ! Write out data in binary files (debugging)
     if (cmfd_write_matrices) then
 
-      ! write out residual vector 
+      ! Write out residual vector 
       if (adjoint_calc) then
         filename = 'adj_res.bin'
       else
@@ -311,7 +306,7 @@ contains
       end if
       call res % write_petsc_binary(filename)
 
-      ! write out solution vector
+      ! Write out solution vector
       if (adjoint_calc) then
         filename = 'adj_x.bin'
       else
@@ -324,18 +319,18 @@ contains
   end subroutine compute_nonlinear_residual
 
 !===============================================================================
-! COMPUTE_ADJOINT
+! COMPUTE_ADJOINT calculates mathematical adjoint by taking transposes
 !===============================================================================
 
   subroutine compute_adjoint()
 
     use global,  only: cmfd_write_matrices
 
-    ! transpose matrices
+    ! Transpose matrices
     call loss % transpose()
     call prod % transpose()
 
-    ! write out matrix in binary file (debugging)
+    ! Write out matrix in binary file (debugging)
     if (cmfd_write_matrices) then
       call loss % write_petsc_binary('adj_lossmat.bin')
       call loss % write_petsc_binary('adj_prodmat.bin')
@@ -344,7 +339,7 @@ contains
   end subroutine compute_adjoint
 
 !===============================================================================
-! EXTRACT_RESULTS 
+! EXTRACT_RESULTS gets the results and puts them in global CMFD object
 !===============================================================================
 
   subroutine extract_results()
@@ -354,10 +349,10 @@ contains
 
     integer :: n ! problem size
 
-    ! get problem size
+    ! Get problem size
     n = loss % n
 
-    ! also allocate in cmfd object
+    ! Also allocate in cmfd object
     if (adjoint_calc) then
       if (.not. allocated(cmfd % adj_phi)) allocate(cmfd % adj_phi(n))
     else
diff --git a/src/cmfd_power_solver.F90 b/src/cmfd_power_solver.F90
index c63cff3205..6c6bcbd9d2 100644
--- a/src/cmfd_power_solver.F90
+++ b/src/cmfd_power_solver.F90
@@ -69,9 +69,6 @@ contains
     ! Set up krylov info
     call gmres % set_oper(loss, loss)
 
-    ! Precondition matrix
-!   call gmres % precondition(loss)
-
     ! Stop timer for build
     call time_cmfdbuild % stop()
 
@@ -149,11 +146,11 @@ contains
 
     use global,  only: cmfd_write_matrices
 
-    ! transpose matrices
+    ! Transpose matrices
     call loss % transpose()
     call prod % transpose()
 
-    ! write out matrix in binary file (debugging)
+    ! Write out matrix in binary file (debugging)
     if (cmfd_write_matrices) then
       call loss % write_petsc_binary('adj_lossmat.bin')
       call prod % write_petsc_binary('adj_prodmat.bin')
@@ -170,37 +167,37 @@ contains
 
     integer     :: i         ! iteration counter
 
-    ! reset convergence flag
+    ! Reset convergence flag
     iconv = .false.
 
-    ! begin power iteration
+    ! Begin power iteration
     do i = 1, 10000
 
-      ! compute source vector
+      ! Compute source vector
       call prod % vector_multiply(phi_o, S_o)
 
-      ! normalize source vector
+      ! Normalize source vector
       S_o % val = S_o % val / k_o
 
-      ! compute new flux vector
+      ! Compute new flux vector
       call gmres % solve(S_o, phi_n)
 
-      ! compute new source vector
+      ! Compute new source vector
       call prod % vector_multiply(phi_n, S_n)
 
-      ! compute new k-eigenvalue
+      ! Compute new k-eigenvalue
       k_n = sum(S_n % val) / sum(S_o % val)
 
-      ! renormalize the old source
+      ! Renormalize the old source
       S_o % val = S_o % val * k_o
 
-      ! check convergence
+      ! Check convergence
       call convergence(i)
 
-      ! to break or not to break
+      ! To break or not to break
       if (iconv) exit
 
-      ! record old values
+      ! Record old values
       phi_o % val = phi_n % val
       k_o = k_n
 
@@ -223,19 +220,19 @@ contains
     real(8)     :: kerr           ! error in keff
     real(8)     :: serr           ! error in source
 
-    ! reset convergence flag
+    ! Reset convergence flag
     iconv = .false.
 
-    ! calculate error in keff
+    ! Calculate error in keff
     kerr = abs(k_o - k_n)/k_n
 
-    ! calculate max error in source
+    ! Calculate max error in source
     serr = sqrt(ONE/dble(S_n % n) * sum(((S_n % val - S_o % val)/S_n % val)**2))
 
-    ! check for convergence
+    ! Check for convergence
     if(kerr < ktol .and. serr < stol) iconv = .true.
 
-    ! print out to user
+    ! Print out to user
     if (cmfd_power_monitor .and. master) then
       write(OUTPUT_UNIT,FMT='(I0,":",T10,"k-eff: ",F0.8,T30,"k-error: ",1PE12.5,T55, &
            "src-error: ",1PE12.5)') iter, k_n, kerr, serr
@@ -254,38 +251,38 @@ contains
     character(len=25)    :: filename  ! name of file to write data 
     integer              :: n         ! problem size
 
-    ! get problem size
+    ! Get problem size
     n = loss % n
 
-    ! also allocate in cmfd object
+    ! Also allocate in cmfd object
     if (adjoint_calc) then
       if (.not. allocated(cmfd%adj_phi)) allocate(cmfd%adj_phi(n))
     else
       if (.not. allocated(cmfd%phi)) allocate(cmfd%phi(n))
     end if
 
-    ! save values 
+    ! Save values 
     if (adjoint_calc) then
       cmfd % adj_phi = phi_n % val
     else
       cmfd % phi = phi_n % val 
     end if
 
-    ! save eigenvalue
+    ! Save eigenvalue
     if(adjoint_calc) then
       cmfd%adj_keff = k_n
     else
       cmfd%keff = k_n
     end if
 
-    ! normalize phi to 1
+    ! Normalize phi to 1
     if (adjoint_calc) then
       cmfd%adj_phi = cmfd%adj_phi/sqrt(sum(cmfd%adj_phi*cmfd%adj_phi))
     else
       cmfd%phi = cmfd%phi/sqrt(sum(cmfd%phi*cmfd%phi))
     end if
 
-    ! write out results
+    ! Write out results
     if (cmfd_write_matrices) then
       if (adjoint_calc) then
         filename = 'adj_fluxvec.bin'
@@ -303,7 +300,7 @@ contains
 
   subroutine finalize()
 
-    ! destroy all objects 
+    ! Destroy all objects 
     call gmres % destroy() 
     call loss  % destroy() 
     call prod  % destroy()
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index b920e41bbd..f62437c417 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -45,20 +45,20 @@ contains
     integer       :: nnz
     class(Matrix) :: self
 
-    ! preallocate vectors
+    ! Preallocate vectors
     if (.not.allocated(self % row)) allocate(self % row(n+1))
     if (.not.allocated(self % col)) allocate(self % col(nnz))
     if (.not.allocated(self % val)) allocate(self % val(nnz))
 
-    ! set counters to 1
+    ! Set counters to 1
     self % n_kount  = 1
     self % nz_kount = 1
 
-    ! set n and nnz
+    ! Set n and nnz
     self % n = n
     self % nnz = nnz
 
-    ! set petsc active by default to false
+    ! Set petsc active by default to false
     self % petsc_active = .false.
 
   end subroutine matrix_create
@@ -111,7 +111,7 @@ contains
   end subroutine matrix_new_row
 
 !===============================================================================
-! MATRIX_ASSEMBLE
+! MATRIX_ASSEMBLE main rountine to sort all the columns in CSR matrix
 !===============================================================================
 
   subroutine matrix_assemble(self)
@@ -122,14 +122,14 @@ contains
     integer :: first
     integer :: last
 
-    ! loop around row vector
+    ! Loop around row vector
     do i = 1, self % n
 
-      ! get bounds
+      ! Get bounds
       first = self % row(i)
       last  = self % row(i+1) - 1
 
-      ! sort a row
+      ! Sort a row
       call sort_csr(self % col, self % val, first, last)
 
     end do
@@ -137,7 +137,7 @@ contains
   end subroutine matrix_assemble
 
 !===============================================================================
-! SORT_CSR
+! SORT_CSR main routine that performs a sort on a CSR col/val subvector
 !===============================================================================
 
   recursive subroutine sort_csr(col, val, first, last)
@@ -158,7 +158,7 @@ contains
   end subroutine sort_csr
 
 !===============================================================================
-! SPLIT
+! SPLIT bisects the search space for the sorting routine
 !===============================================================================
 
   subroutine split(col, val, low, high, mid)
@@ -181,20 +181,20 @@ contains
     pivot = col(low)
     val0 = val(low)
 
-    ! repeat the following while left and right havent met
+    ! Repeat the following while left and right havent met
     do while (left < right)
 
-      ! scan right to left to find element < pivot
+      ! Scan right to left to find element < pivot
       do while (left < right .and. col(right) >= pivot)
         right = right - 1
       end do
 
-      ! scan left to right to find element > pivot
+      ! Scan left to right to find element > pivot
       do while (left < right .and. col(left) <= pivot)
         left = left + 1
       end do
 
-      ! if left and right havent met, exchange the items
+      ! If left and right havent met, exchange the items
       if (left < right) then
         iswap = col(left)
         col(left) = col(right)
@@ -206,7 +206,7 @@ contains
 
     end do
 
-    ! switch the element in split position with pivot
+    ! Switch the element in split position with pivot
     col(low) = col(right)
     col(right) = pivot
     mid = right
@@ -216,7 +216,7 @@ contains
   end subroutine split
 
 !===============================================================================
-! MATRIX_SETUP_PETSC
+! MATRIX_SETUP_PETSC configures the row/col vectors and links to a petsc object
 !===============================================================================
 
   subroutine matrix_setup_petsc(self)
@@ -227,7 +227,7 @@ contains
     self % row = self % row - 1
     self % col = self % col - 1
 
-    ! link to petsc
+    ! Link to petsc
 #ifdef PETSC
     call MatCreateSeqAIJWithArrays(PETSC_COMM_WORLD, self % n, self % n, &
             self % row, self % col, self % val, self % petsc_mat, petsc_err)
@@ -239,7 +239,7 @@ contains
   end subroutine matrix_setup_petsc
 
 !===============================================================================
-! MATRIX_WRITE_PETSC_BINARY
+! MATRIX_WRITE_PETSC_BINARY writes a petsc matrix binary file
 !===============================================================================
 
   subroutine matrix_write_petsc_binary(self, filename)
@@ -259,7 +259,7 @@ contains
   end subroutine matrix_write_petsc_binary
 
 !===============================================================================
-! MATRIX_TRANSPOSE
+! MATRIX_TRANSPOSE uses PETSc to transpose a matrix
 !===============================================================================
 
   subroutine matrix_transpose(self)
@@ -274,7 +274,7 @@ contains
   end subroutine matrix_transpose
 
 !===============================================================================
-! MATRIX_VECTOR_MULTIPLY
+! MATRIX_VECTOR_MULTIPLY allow a vector to multiply the matrix
 !===============================================================================
 
   subroutine matrix_vector_multiply(self, vec_in, vec_out)
@@ -290,25 +290,18 @@ contains
     integer :: j
     integer :: shift
 
-!    integer :: petsc_err
-
-!#ifdef PETSC
-!     call MatMult(self % petsc_mat, vec_in % petsc_vec, vec_out % petsc_vec, &
-!                 petsc_err)
-!#endif
-
-    ! set shift by default 0 and change if petsc is active
-    ! this is because PETSc needs vectors to remain referenced to 0
+    ! Set shift by default 0 and change if petsc is active
+    ! This is because PETSc needs vectors to remain referenced to 0
     shift = 0
     if (self % petsc_active) shift = 1
 
-    ! begin loop around rows
+    ! Begin loop around rows
     ROWS: do i = 1, self % n
 
-      ! initialize target location in vector
+      ! Initialize target location in vector
       vec_out % val(i) = ZERO
 
-      ! begin loop around columns (need to shift if petsc is active)
+      ! Begin loop around columns (need to shift if petsc is active)
       COLS: do j = self % row(i) + shift, self % row(i + 1) - 1 + shift
 
         vec_out % val(i) = vec_out % val(i) + self % val(j) * &
diff --git a/src/petsc_solver.F90 b/src/petsc_solver.F90
index 49802a9dcd..8318ca174b 100644
--- a/src/petsc_solver.F90
+++ b/src/petsc_solver.F90
@@ -10,7 +10,7 @@ module petsc_solver
 #  include <finclude/petsc.h90>
 #endif
 
-  ! Petsc GMRES solver context
+  ! PETSc GMRES solver type 
   type, public :: Petsc_gmres 
 #ifdef PETSC
     KSP :: ksp
@@ -18,7 +18,6 @@ module petsc_solver
 #endif
    contains
      procedure :: create       => gmres_create
-     procedure :: precondition => gmres_precondition
      procedure :: set_oper     => gmres_set_oper
      procedure :: destroy      => gmres_destroy
      procedure :: solve        => gmres_solve
@@ -30,7 +29,7 @@ module petsc_solver
     procedure (jac_interface), pointer, nopass :: jac_proc_ptr
   end type Jfnk_ctx
 
-  ! Petsc SNES JFNK solver context
+  ! Petsc SNES JFNK solver type 
   type, public :: Petsc_jfnk
 #ifdef PETSC
     KSP  :: ksp
@@ -50,10 +49,10 @@ module petsc_solver
 
   ! Abstract interface stating how jacobian and residual routines look
   abstract interface
-    subroutine res_interface(x, r)
+    subroutine res_interface(x, res)
       import :: Vector
       type(Vector) :: x
-      type(Vector) :: r
+      type(Vector) :: res
     end subroutine res_interface
 
     subroutine jac_interface(x)
@@ -65,7 +64,7 @@ module petsc_solver
 contains
 
 !===============================================================================
-! GMRES_CREATE
+! GMRES_CREATE sets up a PETSc GMRES solver
 !===============================================================================
 
   subroutine gmres_create(self)
@@ -90,22 +89,7 @@ contains
   end subroutine gmres_create
 
 !===============================================================================
-! GMRES_PRECONDITION
-!===============================================================================
-
-  subroutine gmres_precondition(self, mat_in)
-
-    class(Petsc_gmres) :: self
-    type(Matrix)       :: mat_in
-
-#ifdef PETSC
-!   call PCFactorGetMatrix(self % pc, mat_in % petsc_mat, petsc_err) 
-#endif
-
-  end subroutine gmres_precondition
-
-!===============================================================================
-! GMRES_SET_OPER
+! GMRES_SET_OPER sets the matrix opetors for the GMRES solver
 !===============================================================================
 
   subroutine gmres_set_oper(self, prec_mat, mat_in)
@@ -123,7 +107,7 @@ contains
   end subroutine gmres_set_oper
 
 !===============================================================================
-! GMRES_DESTROY
+! GMRES_DESTROY frees all memory associated with the GMRES solver
 !===============================================================================
 
   subroutine gmres_destroy(self)
@@ -137,7 +121,7 @@ contains
   end subroutine gmres_destroy
 
 !===============================================================================
-! GMRES_SOLVE
+! GMRES_SOLVE solves the linear system
 !===============================================================================
 
   subroutine gmres_solve(self, b, x)
@@ -153,7 +137,7 @@ contains
   end subroutine gmres_solve
 
 !===============================================================================
-! JFNK_CREATE
+! JFNK_CREATE sets up a JFNK solver using PETSc SNES
 !===============================================================================
 
   subroutine jfnk_create(self)
@@ -187,7 +171,7 @@ contains
   end subroutine jfnk_create
 
 !===============================================================================
-! JFNK_DESTROY
+! JFNK_DESTROY frees all memory associated with JFNK solver
 !===============================================================================
 
   subroutine jfnk_destroy(self)
@@ -202,7 +186,7 @@ contains
   end subroutine jfnk_destroy
 
 !===============================================================================
-! JFNK_SET_FUNCTIONS
+! JFNK_SET_FUNCTIONS sets user functions and matrix free objects
 !===============================================================================
 
   subroutine jfnk_set_functions(self, ctx, res, jac_prec)
@@ -230,7 +214,7 @@ contains
   end subroutine jfnk_set_functions
 
 !===============================================================================
-! JFNK_SOLVE
+! JFNK_SOLVE solves the nonlinear system
 !===============================================================================
 
   subroutine jfnk_solve(self, xvec)
@@ -245,7 +229,7 @@ contains
   end subroutine jfnk_solve
 
 !===============================================================================
-! JFNK_COMPUTE_RESIDUAL
+! JFNK_COMPUTE_RESIDUAL buffer routine to user specifed residual routine
 !===============================================================================
 
   subroutine jfnk_compute_residual(snes, x, res, ctx, ierr)
@@ -259,6 +243,10 @@ contains
     type(Vector) :: xvec
     type(Vector) :: resvec
 
+    ! We need to use the x and res that come from PETSc because the pointer
+    ! location changes and therefore we can not use module variables
+    ! for the residual and x vectors here
+
     ! Need to point an OpenMC vector to PETSc vector
 #ifdef PETSC
     call VecGetArrayF90(x, xvec % val, ierr)
@@ -293,6 +281,9 @@ contains
 
     type(Vector) :: xvec
 
+    ! Again, we use the vector that comes from Petsc to build the Jacobian
+    ! matrix
+
     ! Need to point OpenMC vector to PETSc Vector
 #ifdef PETSC
     call VecGetArrayF90(x, xvec % val, ierr)
diff --git a/src/vector_header.F90 b/src/vector_header.F90
index 1c525fd2f8..a201dc9c21 100644
--- a/src/vector_header.F90
+++ b/src/vector_header.F90
@@ -37,16 +37,16 @@ contains
     integer       :: n
     class(Vector) :: self
 
-    ! preallocate vector
+    ! Preallocate vector
     if (.not.associated(self % val)) allocate(self % val(n))
 
-    ! set n
+    ! Set n
     self % n = n
 
-    ! initialize to zero
+    ! Initialize to zero
     self % val = ZERO
 
-    ! petsc is default not active
+    ! Petsc is default not active
     self % petsc_active = .false.
 
   end subroutine vector_create
@@ -82,26 +82,26 @@ contains
   end subroutine vector_add_value
 
 !===============================================================================
-! VECTOR_SETUP_PETSC
+! VECTOR_SETUP_PETSC links the data to a PETSc vector
 !===============================================================================
 
   subroutine vector_setup_petsc(self)
 
     class(Vector) :: self
 
-    ! link to petsc
+    ! Link to PETSc 
 #ifdef PETSC
     call VecCreateSeqWithArray(PETSC_COMM_WORLD, 1, self % n, self % val, &
          self % petsc_vec, petsc_err) 
 #endif
 
-    ! set that petsc is now active
+    ! Set that PETSc is now active
     self % petsc_active = .true.
 
   end subroutine vector_setup_petsc
 
 !===============================================================================
-! VECTOR_WRITE_PETSC_BINARY
+! VECTOR_WRITE_PETSC_BINARY writes the PETSc vector to a binary file
 !===============================================================================
 
   subroutine vector_write_petsc_binary(self, filename)

From 5285c37b72900e95a2615958938b56b189dc85f7 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Thu, 11 Jul 2013 16:55:14 -0400
Subject: [PATCH 26/69] fixed some comments

---
 src/cmfd_input.F90        |  2 +-
 src/cmfd_jfnk_solver.F90  | 10 +++++-----
 src/cmfd_output.F90       |  2 +-
 src/cmfd_power_solver.F90 | 18 +++++++++---------
 src/matrix_header.F90     | 10 +++++-----
 5 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/cmfd_input.F90 b/src/cmfd_input.F90
index 7b5e9c2d62..b8e90cb262 100644
--- a/src/cmfd_input.F90
+++ b/src/cmfd_input.F90
@@ -40,7 +40,7 @@ contains
 # ifdef PETSC
     PETSC_COMM_WORLD = new_comm
 
-    ! Initialize petsc on all procs
+    ! Initialize PETSc on all procs
     call PetscInitialize(PETSC_NULL_CHARACTER,mpi_err)
 # endif
 
diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
index 54c26326b5..70593a97ef 100644
--- a/src/cmfd_jfnk_solver.F90
+++ b/src/cmfd_jfnk_solver.F90
@@ -85,7 +85,7 @@ contains
     call build_loss_matrix(loss, adjoint = physical_adjoint)
     call build_prod_matrix(prod, adjoint = physical_adjoint)
 
-    ! Assembly matrices and use petsc
+    ! Assemble matrices and use PETSc
     call loss % assemble()
     call prod % assemble()
     call loss % setup_petsc()
@@ -102,28 +102,28 @@ contains
     call resvec % create(n + 1)
     call xvec % create(n + 1)
 
-    ! Set flux in guess
+    ! Set flux in guess from rough power iteration
     if (adjoint_calc) then
       xvec % val(1:n) = cmfd % adj_phi
     else
       xvec % val(1:n) = cmfd % phi
     end if
 
-    ! Set keff in guess
+    ! Set keff in guess from rough power iteration
     if (adjoint_calc) then
       xvec % val(n + 1) = ONE/cmfd % adj_keff
     else
       xvec % val(n + 1) = ONE/cmfd % keff
     end if
 
-    ! Set up vectors for petsc
+    ! Set up vectors for PETSc 
     call resvec   % setup_petsc()
     call xvec     % setup_petsc()
 
     ! Build jacobian from initial guess
     call build_jacobian_matrix(xvec)
 
-    ! Set up Jacobian for Petsc
+    ! Set up Jacobian for PETSc
     call jac_prec % setup_petsc()
 
   end subroutine init_data
diff --git a/src/cmfd_output.F90 b/src/cmfd_output.F90
index 2936dda35a..eb2bf69f93 100644
--- a/src/cmfd_output.F90
+++ b/src/cmfd_output.F90
@@ -18,7 +18,7 @@ contains
                            master, mpi_err
     use cmfd_header, only: deallocate_cmfd
 
-    ! Finalize petsc
+    ! Finalize PETSc 
 #ifdef PETSC
     call PetscFinalize(mpi_err)
 # endif
diff --git a/src/cmfd_power_solver.F90 b/src/cmfd_power_solver.F90
index 6c6bcbd9d2..5c8333cbc1 100644
--- a/src/cmfd_power_solver.F90
+++ b/src/cmfd_power_solver.F90
@@ -62,7 +62,7 @@ contains
     ! Initialize matrices and vectors
     call init_data(physical_adjoint)
 
-    ! Check for adjoint calculation
+    ! Check for mathematical adjoint calculation
     if (adjoint_calc .and. trim(cmfd_adjoint_type) == 'math') &
         call compute_adjoint()
 
@@ -80,7 +80,7 @@ contains
     ! Extract results
     call extract_results()
 
-    ! Deallocate petsc objects
+    ! Deallocate data 
     call finalize()
 
   end subroutine cmfd_power_execute
@@ -120,10 +120,10 @@ contains
     k_n = guess
     k_o = guess
 
-    ! Set up loss matrix
+    ! Fill in loss matrix
     call build_loss_matrix(loss, adjoint=adjoint) 
 
-    ! Set up production matrix
+    ! Fill in production matrix
     call build_prod_matrix(prod, adjoint=adjoint)
 
     ! Setup petsc for everything
@@ -159,13 +159,13 @@ contains
   end subroutine compute_adjoint
 
 !===============================================================================
-! EXECUTE_POWER_ITER  in the main power iteration routine
+! EXECUTE_POWER_ITER  is the main power iteration routine
 !                     for the cmfd calculation
 !===============================================================================
 
   subroutine execute_power_iter()
 
-    integer     :: i         ! iteration counter
+    integer :: i ! iteration counter
 
     ! Reset convergence flag
     iconv = .false.
@@ -234,8 +234,8 @@ contains
 
     ! Print out to user
     if (cmfd_power_monitor .and. master) then
-      write(OUTPUT_UNIT,FMT='(I0,":",T10,"k-eff: ",F0.8,T30,"k-error: ",1PE12.5,T55, &
-           "src-error: ",1PE12.5)') iter, k_n, kerr, serr
+      write(OUTPUT_UNIT,FMT='(I0,":",T10,"k-eff: ",F0.8,T30,"k-error: ", &
+           &1PE12.5,T55, "src-error: ",1PE12.5)') iter, k_n, kerr, serr
     end if
 
   end subroutine convergence
@@ -254,7 +254,7 @@ contains
     ! Get problem size
     n = loss % n
 
-    ! Also allocate in cmfd object
+    ! Allocate in cmfd object if not already allocated
     if (adjoint_calc) then
       if (.not. allocated(cmfd%adj_phi)) allocate(cmfd%adj_phi(n))
     else
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index f62437c417..d85951d527 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -58,7 +58,7 @@ contains
     self % n = n
     self % nnz = nnz
 
-    ! Set petsc active by default to false
+    ! Set PETSc active by default to false
     self % petsc_active = .false.
 
   end subroutine matrix_create
@@ -216,7 +216,7 @@ contains
   end subroutine split
 
 !===============================================================================
-! MATRIX_SETUP_PETSC configures the row/col vectors and links to a petsc object
+! MATRIX_SETUP_PETSC configures the row/col vectors and links to a PETSc object
 !===============================================================================
 
   subroutine matrix_setup_petsc(self)
@@ -239,7 +239,7 @@ contains
   end subroutine matrix_setup_petsc
 
 !===============================================================================
-! MATRIX_WRITE_PETSC_BINARY writes a petsc matrix binary file
+! MATRIX_WRITE_PETSC_BINARY writes a PETSc matrix binary file
 !===============================================================================
 
   subroutine matrix_write_petsc_binary(self, filename)
@@ -290,7 +290,7 @@ contains
     integer :: j
     integer :: shift
 
-    ! Set shift by default 0 and change if petsc is active
+    ! Set shift by default 0 and change if PETSc is active
     ! This is because PETSc needs vectors to remain referenced to 0
     shift = 0
     if (self % petsc_active) shift = 1
@@ -301,7 +301,7 @@ contains
       ! Initialize target location in vector
       vec_out % val(i) = ZERO
 
-      ! Begin loop around columns (need to shift if petsc is active)
+      ! Begin loop around columns (need to shift if PETSc is active)
       COLS: do j = self % row(i) + shift, self % row(i + 1) - 1 + shift
 
         vec_out % val(i) = vec_out % val(i) + self % val(j) * &

From e4f55c92467083913916cb7de108e7fbab0ece97 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Fri, 12 Jul 2013 10:03:33 -0400
Subject: [PATCH 27/69] changed code on how columns are printed out

---
 src/eigenvalue.F90 |  2 +-
 src/output.F90     | 48 ++++++++++++++++------------------------------
 2 files changed, 18 insertions(+), 32 deletions(-)

diff --git a/src/eigenvalue.F90 b/src/eigenvalue.F90
index 82ad2d73a4..e67876b734 100644
--- a/src/eigenvalue.F90
+++ b/src/eigenvalue.F90
@@ -43,7 +43,7 @@ contains
     allocate(p)
 
     ! Display column titles
-    call print_columns()
+    if(master) call print_columns()
 
     ! Turn on inactive timer
     call time_inactive % start()
diff --git a/src/output.F90 b/src/output.F90
index c6810dc083..a4ce08fc3b 100644
--- a/src/output.F90
+++ b/src/output.F90
@@ -1214,37 +1214,28 @@ contains
 
   subroutine print_columns()
 
-    if (entropy_on) then
-      if (cmfd_run) then
-        message = " Bat./Gen.      k       Entropy         Average k          CMFD k    CMFD Ent"
-        call write_message(1)
-        message = " =========   ========   ========   ====================   ========   ========"
-        call write_message(1)
-      else
-        message = " Bat./Gen.      k       Entropy         Average k"
-        call write_message(1)
-        message = " =========   ========   ========   ===================="
-        call write_message(1)
-      end if
-    else
-      if (cmfd_run) then
-        message = " Bat./Gen.      k            Average k          CMFD k"
-        call write_message(1)
-        message = " =========   ========   ====================   ========"
-        call write_message(1)
-      else
-        message = " Bat./Gen.      k            Average k"
-        call write_message(1)
-        message = " =========   ========   ===================="
-        call write_message(1)
-      end if
+    write(UNIT=ou, FMT='(2X,A9,3X)', ADVANCE='NO') "Bat./Gen."
+    write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') "   k    "
+    if (entropy_on) write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') "Entropy "
+    write(UNIT=ou, FMT='(A20,3X)', ADVANCE='NO') "     Average k      "
+    if (cmfd_run) then
+      write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') " CMFD k "
     end if
+    write(UNIT=ou, FMT=*)
+    
+    write(UNIT=ou, FMT='(2X,A9,3X)', ADVANCE='NO') "========="
+    write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') "========"
+    if (entropy_on) write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') "========"
+    write(UNIT=ou, FMT='(A20,3X)', ADVANCE='NO') "===================="
+    if (cmfd_run) then
+      write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') "========="
+    end if
+    write(UNIT=ou, FMT=*)
 
   end subroutine print_columns
 
 !===============================================================================
-! PRINT_GENERATION displays information for a generation of neutrons. For now,
-! if the user has entropy on, it will print out the entropy
+! PRINT_GENERATION displays information for a generation of neutrons.
 !===============================================================================
 
   subroutine print_generation()
@@ -1294,15 +1285,10 @@ contains
       write(UNIT=OUTPUT_UNIT, FMT='(23X)', ADVANCE='NO')
     end if
 
-
     ! write out cmfd keff if it is active
     if (cmfd_on) write(UNIT=OUTPUT_UNIT, FMT='(3X, F8.5)', ADVANCE='NO') &
          cmfd % keff 
 
-    ! write out cmfd entopy
-    if (cmfd_on .and. entropy_on) write(UNIT=OUTPUT_UNIT, &
-         FMT='(3X, F8.5)', ADVANCE='NO') cmfd % entropy
-
     ! next line
     write(UNIT=OUTPUT_UNIT, FMT=*)
 

From 212589556d2d87531f7cc08c3f8f6ba8113308e4 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Fri, 12 Jul 2013 12:20:30 -0400
Subject: [PATCH 28/69] add new column display for CMFD, added dominance ratio
 calculation, redid neutron balance, cmfd can display batchwise entropy, RMS
 of neutron balance, RMS of source deviation or dominance ratio

---
 src/DEPENDENCIES          |  7 +----
 src/OBJECTS               |  1 -
 src/cmfd_data.F90         | 35 ++++++++++-------------
 src/cmfd_execute.F90      |  9 ++++--
 src/cmfd_header.F90       | 31 ++++++++++++++++++--
 src/cmfd_input.F90        | 13 ++++++---
 src/cmfd_output.F90       | 59 ---------------------------------------
 src/cmfd_power_solver.F90 | 22 ++++++++++++---
 src/constants.F90         |  5 ++--
 src/finalize.F90          | 11 +++-----
 src/global.F90            |  8 ++++--
 src/output.F90            | 34 ++++++++++++++++++++--
 src/templates/cmfd_t.xml  |  3 +-
 13 files changed, 120 insertions(+), 118 deletions(-)
 delete mode 100644 src/cmfd_output.F90

diff --git a/src/DEPENDENCIES b/src/DEPENDENCIES
index bb1f7a7cd6..b4c9c7980c 100644
--- a/src/DEPENDENCIES
+++ b/src/DEPENDENCIES
@@ -6,11 +6,6 @@ energy_grid.o: global.o
 energy_grid.o: list_header.o
 energy_grid.o: output.o
 
-cmfd_output.o: cmfd_data.o
-cmfd_output.o: cmfd_header.o
-cmfd_output.o: constants.o
-cmfd_output.o: global.o
-
 ace_header.o: constants.o
 ace_header.o: endf_header.o
 
@@ -104,6 +99,7 @@ fission.o: global.o
 fission.o: interpolation.o
 fission.o: search.o
 
+cmfd_input.o: cmfd_header.o
 cmfd_input.o: error.o
 cmfd_input.o: global.o
 cmfd_input.o: mesh_header.o
@@ -194,7 +190,6 @@ string.o: constants.o
 string.o: error.o
 string.o: global.o
 
-finalize.o: cmfd_output.o
 finalize.o: global.o
 finalize.o: hdf5_interface.o
 finalize.o: output.o
diff --git a/src/OBJECTS b/src/OBJECTS
index 5075a5ee53..4a2c48a717 100644
--- a/src/OBJECTS
+++ b/src/OBJECTS
@@ -8,7 +8,6 @@ cmfd_header.o \
 cmfd_input.o \
 cmfd_jfnk_solver.o \
 cmfd_loss_operator.o \
-cmfd_output.o \
 cmfd_power_solver.o \
 cmfd_prod_operator.o \
 constants.o \
diff --git a/src/cmfd_data.F90 b/src/cmfd_data.F90
index 4e81c2aaf4..ef971f47e4 100644
--- a/src/cmfd_data.F90
+++ b/src/cmfd_data.F90
@@ -24,9 +24,6 @@ contains
     use constants,           only: CMFD_NOACCEL
     use global,              only: cmfd, cmfd_coremap, cmfd_downscatter
 
-    ! Initialize cmfd object
-    if (.not.allocated(cmfd%flux)) call allocate_cmfd(cmfd)
-
     ! Check for core map and set it up
     if ((cmfd_coremap) .and. (cmfd%mat_dim == CMFD_NOACCEL)) call set_coremap()
 
@@ -37,7 +34,7 @@ contains
     if (cmfd_downscatter) call compute_effective_downscatter()
 
     ! Check neutron balance
-!   call neutron_balance(670)
+    call neutron_balance()
 
     ! Calculate dtilde
     call compute_dtilde()
@@ -370,10 +367,10 @@ contains
 ! NEUTRON_BALANCE writes a file that contains n. bal. info for all cmfd mesh
 !===============================================================================
 
-  subroutine neutron_balance(uid)
+  subroutine neutron_balance()
 
     use constants,    only: ONE, ZERO, CMFD_NOACCEL, CMFD_NORES
-    use global,       only: cmfd, keff
+    use global,       only: cmfd, keff, current_batch
 
     integer :: nx           ! number of mesh cells in x direction
     integer :: ny           ! number of mesh cells in y direction
@@ -385,15 +382,13 @@ contains
     integer :: g            ! iteration counter for g
     integer :: h            ! iteration counter for outgoing groups
     integer :: l            ! iteration counter for leakage
-    integer :: uid
+    integer :: cnt          ! number of locations to count neutron balance
     real(8) :: leakage      ! leakage term in neutron balance
     real(8) :: interactions ! total number of interactions in balance
     real(8) :: scattering   ! scattering term in neutron balance
     real(8) :: fission      ! fission term in neutron balance
     real(8) :: res          ! residual of neutron balance
-
-    ! Check if keff is close to 0 (happens on first active batch)
-    if (keff < 1e-8_8) return
+    real(8) :: rms          ! RMS of the residual
 
     ! Extract spatial and energy indices from object
     nx = cmfd % indices(1)
@@ -404,6 +399,10 @@ contains
     ! Allocate res dataspace
     if (.not. allocated(cmfd%resnb)) allocate(cmfd%resnb(ng,nx,ny,nz))
 
+    ! Reset rms and cnt
+    rms = ZERO
+    cnt = 0
+
     ! Begin loop around space and energy groups
     ZLOOP: do k = 1, nz
 
@@ -457,16 +456,9 @@ contains
             ! Bank res in cmfd object
             cmfd%resnb(g,i,j,k) = res
 
-            ! Write out info to file
-            write(uid,'(A,1X,I0,1X,I0,1X,I0,1X,A,1X,I0,1X,A,1PE11.4)') &
-                          'Location',i,j,k,' Group:',g,'Balance:',res
-            write(uid,100) 'Leakage:',leakage
-            write(uid,100) 'Interactions:',interactions
-            write(uid,100) 'Scattering:',scattering
-            write(uid,100) 'Fission:',fission
-            write(uid,100) 'k-eff:',keff
-            write(uid,100) 'k-eff balanced:',fission/(leakage+interactions-scattering)
-            write(uid,100) 'Balance:',keff - fission/(leakage+interactions-scattering)
+            ! Take square for RMS calculation
+            rms = rms + res**2
+            cnt = cnt + 1
 
           end do GROUPG
 
@@ -476,7 +468,8 @@ contains
 
     end do ZLOOP
 
- 100 FORMAT(A,1X,1PE11.4)
+    ! Calculate RMS and record in vector for this batch
+    cmfd % balance(current_batch) = sqrt(ONE/dble(cnt)*rms)
 
   end subroutine neutron_balance
 
diff --git a/src/cmfd_execute.F90 b/src/cmfd_execute.F90
index 60452c8850..a514b0cd8b 100644
--- a/src/cmfd_execute.F90
+++ b/src/cmfd_execute.F90
@@ -132,7 +132,8 @@ contains
   subroutine calc_fission_source()
 
     use constants,  only: CMFD_NOACCEL, ZERO, TWO
-    use global,     only: cmfd, cmfd_coremap, master, mpi_err, entropy_on
+    use global,     only: cmfd, cmfd_coremap, master, mpi_err, entropy_on, &
+                          current_batch
 #ifdef MPI
     use mpi
 #endif
@@ -223,7 +224,7 @@ contains
         end where
 
         ! Sum that source
-        cmfd % entropy = -sum(source)
+        cmfd % entropy(current_batch) = -sum(source)
 
         ! Deallocate tmp array
         if (allocated(source)) deallocate(source)
@@ -233,6 +234,10 @@ contains
       ! Normalize source so average is 1.0
       cmfd % cmfd_src = cmfd % cmfd_src/sum(cmfd % cmfd_src)*cmfd % norm
 
+      ! Calculate differences between normalized sources
+      cmfd % src_cmp(current_batch) = sqrt(ONE/cmfd % norm * &
+             sum((cmfd % cmfd_src - cmfd % openmc_src)**2))
+
     end if
 
     ! Broadcast full source to all procs
diff --git a/src/cmfd_header.F90 b/src/cmfd_header.F90
index 027172ee27..3d6b3fd61a 100644
--- a/src/cmfd_header.F90
+++ b/src/cmfd_header.F90
@@ -69,7 +69,16 @@ module cmfd_header
     real(8) :: norm = ONE
 
     ! "Shannon entropy" from cmfd fission source
-    real(8) :: entropy 
+    real(8), allocatable :: entropy(:)
+
+    ! RMS of neutron balance equations
+    real(8), allocatable :: balance(:)
+
+    ! RMS deviation of OpenMC and CMFD normalized source
+    real(8), allocatable :: src_cmp(:)
+
+    ! Dominance ratio
+    real(8), allocatable :: dom(:)
 
   end type cmfd_type
 
@@ -79,9 +88,11 @@ contains
 ! ALLOCATE_CMFD allocates all data in of cmfd type
 !==============================================================================
 
-  subroutine allocate_cmfd(this)
+  subroutine allocate_cmfd(this, n_batches, entropy_on)
 
-    type(cmfd_type) :: this 
+    integer :: n_batches
+    logical :: entropy_on
+    type(cmfd_type) :: this
 
     integer :: nx  ! number of mesh cells in x direction
     integer :: ny  ! number of mesh cells in y direction
@@ -120,6 +131,12 @@ contains
     if (.not. allocated(this % sourcecounts)) allocate(this % sourcecounts(ng,nx,ny,nz))
     if (.not. allocated(this % weightfactors)) allocate(this % weightfactors(ng,nx,ny,nz))
 
+    ! Allocate batchwise parameters
+    if (.not. allocated(this % entropy) .and. entropy_on) allocate(this % entropy(n_batches))
+    if (.not. allocated(this % balance)) allocate(this % balance(n_batches))
+    if (.not. allocated(this % src_cmp)) allocate(this % src_cmp(n_batches))
+    if (.not. allocated(this % dom)) allocate(this % dom(n_batches))
+
     ! Set everthing to 0 except weight multiply factors if feedback isnt on
     this % flux          = ZERO
     this % totalxs       = ZERO
@@ -135,6 +152,10 @@ contains
     this % openmc_src    = ZERO
     this % sourcecounts  = ZERO
     this % weightfactors = ONE
+    this % balance       = ZERO
+    this % src_cmp       = ZERO
+    this % dom           = ZERO
+    if (entropy_on) this % entropy = ZERO
 
   end subroutine allocate_cmfd
 
@@ -164,6 +185,10 @@ contains
     if (allocated(this % weightfactors)) deallocate(this % weightfactors)
     if (allocated(this % cmfd_src))      deallocate(this % cmfd_src)
     if (allocated(this % openmc_src))    deallocate(this % openmc_src)
+    if (allocated(this % balance))       deallocate(this % balance)
+    if (allocated(this % src_cmp))       deallocate(this % src_cmp)
+    if (allocated(this % dom))           deallocate(this % dom)
+    if (allocated(this % entropy))       deallocate(this % entropy)
 
   end subroutine deallocate_cmfd
 
diff --git a/src/cmfd_input.F90 b/src/cmfd_input.F90
index b8e90cb262..d14c65ee93 100644
--- a/src/cmfd_input.F90
+++ b/src/cmfd_input.F90
@@ -18,6 +18,8 @@ contains
 
   subroutine configure_cmfd()
 
+    use cmfd_header,  only: allocate_cmfd
+
     integer :: new_comm ! new mpi communicator
     integer :: color    ! color group of processor
 
@@ -44,11 +46,14 @@ contains
     call PetscInitialize(PETSC_NULL_CHARACTER,mpi_err)
 # endif
 
-    ! initialize timers
+    ! Initialize timers
     call time_cmfd % reset()
     call time_cmfdbuild % reset()
     call time_cmfdsolve % reset()
 
+    ! Allocate cmfd object
+    call allocate_cmfd(cmfd, n_batches, entropy_on)
+
   end subroutine configure_cmfd
 
 !===============================================================================
@@ -162,10 +167,7 @@ contains
          cmfd_power_monitor = .true.
 
     ! Output logicals
-    call lower_case(write_balance_)
     call lower_case(write_matrices_)
-    if (write_balance_ == 'true' .or. write_balance_ == '1') &
-         cmfd_write_balance = .true.
     if (write_matrices_ == 'true' .or. write_matrices_ == '1') &
          cmfd_write_matrices = .true.
 
@@ -188,6 +190,9 @@ contains
     ! Last flush before active batches
     cmfd_act_flush = active_flush_
 
+    ! Get display
+    cmfd_display = display_
+
     ! Create tally objects
     call create_cmfd_tally()
 
diff --git a/src/cmfd_output.F90 b/src/cmfd_output.F90
deleted file mode 100644
index eb2bf69f93..0000000000
--- a/src/cmfd_output.F90
+++ /dev/null
@@ -1,59 +0,0 @@
-module cmfd_output
-
-! This modules cleans up cmfd objects and outputs neutron balance info
-
-  implicit none
-  private
-  public :: finalize_cmfd
-
-contains
-
-!===============================================================================
-! FINALIZE_CMFD finalizes PETSc and frees memory associated with CMFD
-!===============================================================================
-
-  subroutine finalize_cmfd() 
-
-    use global,      only: cmfd, cmfd_write_balance, &
-                           master, mpi_err
-    use cmfd_header, only: deallocate_cmfd
-
-    ! Finalize PETSc 
-#ifdef PETSC
-    call PetscFinalize(mpi_err)
-# endif
-
-    ! Write out final neutron balance
-    if (master .and. cmfd_write_balance) call write_neutron_balance()
-
-    ! Deallocate cmfd object
-    call deallocate_cmfd(cmfd)
-
-  end subroutine finalize_cmfd 
-
-!===============================================================================
-! WRITE_NEUTRON_BALANCE writes an ASCII neutron balance file out
-!===============================================================================
-
-  subroutine write_neutron_balance()
-
-    use cmfd_data,    only: neutron_balance
-    use constants,    only: CMFD_BALANCE, MAX_FILE_LEN
-    use global,       only: path_output
-
-    character(MAX_FILE_LEN) :: filename
-
-    filename = trim(path_output) // 'neutron_balance.out'
-
-    ! Open file for output
-    open(UNIT=CMFD_BALANCE, FILE=filename, ACTION='write')
-
-    ! Write out the tally
-    call neutron_balance(CMFD_BALANCE) 
-
-    ! Close file
-    close(CMFD_BALANCE)
-
-  end subroutine write_neutron_balance
-
-end module cmfd_output
diff --git a/src/cmfd_power_solver.F90 b/src/cmfd_power_solver.F90
index 5c8333cbc1..1d96ea2944 100644
--- a/src/cmfd_power_solver.F90
+++ b/src/cmfd_power_solver.F90
@@ -17,7 +17,9 @@ module cmfd_power_solver
   real(8) :: k_o                  ! old k-eigenvalue
   real(8) :: ktol = 1.e-8_8       ! tolerance on keff
   real(8) :: stol = 1.e-8_8       ! tolerance on source
-  logical :: adjoint_calc = .false. ! run an adjoint calculation
+  real(8) :: norm_n               ! current norm of source vector
+  real(8) :: norm_o               ! old norm of source vector
+  logical :: adjoint_calc         ! run an adjoint calculation
   type(Matrix) :: loss            ! cmfd loss matrix
   type(Matrix) :: prod            ! cmfd prod matrix
   type(Vector) :: phi_n           ! new flux vector
@@ -47,6 +49,7 @@ contains
     if (present(s_tol)) stol = s_tol
 
     ! Check for adjoint execution
+    adjoint_calc = .false.
     if (present(adjoint)) adjoint_calc = adjoint
 
     ! Check for physical adjoint
@@ -91,7 +94,7 @@ contains
 
   subroutine init_data(adjoint)
 
-    use constants, only: ONE
+    use constants, only: ONE, ZERO
 
     logical :: adjoint
 
@@ -136,6 +139,10 @@ contains
     call S_o % setup_petsc()
     call S_n % setup_petsc()
 
+    ! Set norms to 0
+    norm_n = ZERO
+    norm_o = ZERO
+
   end subroutine init_data
 
 !===============================================================================
@@ -194,12 +201,13 @@ contains
       ! Check convergence
       call convergence(i)
 
-      ! To break or not to break
+      ! Break loop if converged
       if (iconv) exit
 
       ! Record old values
       phi_o % val = phi_n % val
       k_o = k_n
+      norm_o = norm_n
 
     end do
 
@@ -232,6 +240,9 @@ contains
     ! Check for convergence
     if(kerr < ktol .and. serr < stol) iconv = .true.
 
+    ! Save the L2 norm of the source
+    norm_n = serr
+
     ! Print out to user
     if (cmfd_power_monitor .and. master) then
       write(OUTPUT_UNIT,FMT='(I0,":",T10,"k-eff: ",F0.8,T30,"k-error: ", &
@@ -246,7 +257,7 @@ contains
 
   subroutine extract_results()
 
-    use global, only: cmfd, cmfd_write_matrices
+    use global, only: cmfd, cmfd_write_matrices, current_batch
 
     character(len=25)    :: filename  ! name of file to write data 
     integer              :: n         ! problem size
@@ -282,6 +293,9 @@ contains
       cmfd%phi = cmfd%phi/sqrt(sum(cmfd%phi*cmfd%phi))
     end if
 
+    ! Save dominance ratio
+    cmfd % dom(current_batch) = norm_n/norm_o
+
     ! Write out results
     if (cmfd_write_matrices) then
       if (adjoint_calc) then
diff --git a/src/constants.F90 b/src/constants.F90
index 6734a7bcd7..1a4080f622 100644
--- a/src/constants.F90
+++ b/src/constants.F90
@@ -363,9 +363,8 @@ module constants
   integer, parameter :: UNIT_TALLY    = 12 ! unit # for writing tally file
   integer, parameter :: UNIT_PLOT     = 13 ! unit # for writing plot file
   integer, parameter :: UNIT_XS       = 14 ! unit # for writing xs summary file
-  integer, parameter :: CMFD_BALANCE  = 15 ! unit # for writing cmfd balance file
-  integer, parameter :: UNIT_PARTICLE = 16 ! unit # for writing particle restart
-  integer, parameter :: UNIT_OUTPUT   = 17 ! unit # for writing output
+  integer, parameter :: UNIT_PARTICLE = 15 ! unit # for writing particle restart
+  integer, parameter :: UNIT_OUTPUT   = 16 ! unit # for writing output
 
   !=============================================================================
   ! CMFD CONSTANTS
diff --git a/src/finalize.F90 b/src/finalize.F90
index ac64e90fba..540726d447 100644
--- a/src/finalize.F90
+++ b/src/finalize.F90
@@ -1,8 +1,5 @@
 module finalize
 
-# ifdef PETSC
-  use cmfd_output,    only: finalize_cmfd
-# endif
   use global
   use output,         only: print_runtime, print_results, &
                             print_overlap_check, write_tallies
@@ -42,11 +39,11 @@ contains
     end if
 
 #ifdef PETSC
-    ! finalize cmfd
-    if (cmfd_run) call finalize_cmfd()
+    ! Finalize PETSc
+    call PetscFinalize(mpi_err)
 #endif
 
-    ! stop timers and show timing statistics
+    ! Stop timers and show timing statistics
     call time_finalize % stop()
     call time_total % stop()
     if (master .and. (run_mode /= MODE_PLOTTING .and. &
@@ -56,7 +53,7 @@ contains
       if (check_overlaps) call print_overlap_check()
     end if
 
-    ! deallocate arrays
+    ! Deallocate arrays
     call free_memory()
 
 #ifdef HDF5
diff --git a/src/global.F90 b/src/global.F90
index 14b2c066e1..9c130cd729 100644
--- a/src/global.F90
+++ b/src/global.F90
@@ -340,16 +340,18 @@ module global
   logical :: cmfd_power_monitor = .false.
 
   ! Cmfd output
-  logical :: cmfd_write_balance  = .false.
   logical :: cmfd_write_matrices = .false.
 
   ! Run an adjoint calculation (last batch only)
   logical :: cmfd_run_adjoint = .false.
 
-  ! Cmfd run logicals
+  ! CMFD run logicals
   logical :: cmfd_on             = .false.
   logical :: cmfd_tally_on       = .true. 
 
+  ! CMFD display info
+  character(len=25) :: cmfd_display
+
   ! Information about state points to be written
   integer :: n_state_points = 0
   type(SetInt) :: statepoint_batch
@@ -413,7 +415,7 @@ contains
     if (allocated(source_bank)) deallocate(source_bank)
     if (allocated(entropy_p)) deallocate(entropy_p)
 
-    ! Deallocate cmfd
+    ! Deallocate CMFD
     call deallocate_cmfd(cmfd)
 
     ! Deallocate tally node lists
diff --git a/src/output.F90 b/src/output.F90
index a4ce08fc3b..2b6b04511d 100644
--- a/src/output.F90
+++ b/src/output.F90
@@ -1220,6 +1220,16 @@ contains
     write(UNIT=ou, FMT='(A20,3X)', ADVANCE='NO') "     Average k      "
     if (cmfd_run) then
       write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') " CMFD k "
+      select case(trim(cmfd_display))
+        case('entropy')
+          write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') "CMFD Ent"
+        case('balance')
+          write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') "RMS Bal "
+        case('source')
+          write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') "RMS Src "
+        case('dominance')
+          write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') "Dom Rat "
+      end select
     end if
     write(UNIT=ou, FMT=*)
     
@@ -1228,7 +1238,9 @@ contains
     if (entropy_on) write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') "========"
     write(UNIT=ou, FMT='(A20,3X)', ADVANCE='NO') "===================="
     if (cmfd_run) then
-      write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') "========="
+      write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') "========"
+      if (cmfd_display /= '') &
+        write(UNIT=ou, FMT='(A8,3X)', ADVANCE='NO') "========"
     end if
     write(UNIT=ou, FMT=*)
 
@@ -1285,9 +1297,25 @@ contains
       write(UNIT=OUTPUT_UNIT, FMT='(23X)', ADVANCE='NO')
     end if
 
-    ! write out cmfd keff if it is active
-    if (cmfd_on) write(UNIT=OUTPUT_UNIT, FMT='(3X, F8.5)', ADVANCE='NO') &
+    ! write out cmfd keff if it is active and other display info
+    if (cmfd_on) then
+      write(UNIT=OUTPUT_UNIT, FMT='(3X, F8.5)', ADVANCE='NO') &
          cmfd % keff 
+      select case(trim(cmfd_display))
+        case('entropy')
+          write(UNIT=OUTPUT_UNIT, FMT='(3X, F8.5)', ADVANCE='NO') &
+            cmfd % entropy(current_batch)
+        case('balance')
+          write(UNIT=OUTPUT_UNIT, FMT='(3X, F8.5)', ADVANCE='NO') &
+            cmfd % balance(current_batch)
+        case('source')
+          write(UNIT=OUTPUT_UNIT, FMT='(3X, F8.5)', ADVANCE='NO') &
+            cmfd % src_cmp(current_batch)
+        case('dominance')
+          write(UNIT=OUTPUT_UNIT, FMT='(3X, F8.5)', ADVANCE='NO') &
+            cmfd % dom(current_batch)
+      end select
+    end if
 
     ! next line
     write(UNIT=OUTPUT_UNIT, FMT=*)
diff --git a/src/templates/cmfd_t.xml b/src/templates/cmfd_t.xml
index 631f2dd44d..0e49e0e5ab 100644
--- a/src/templates/cmfd_t.xml
+++ b/src/templates/cmfd_t.xml
@@ -17,13 +17,11 @@
   <variable name="norm_"           tag="norm"           type="double"        default="1.0"                 />
   <variable name="feedback_"       tag="feedback"       type="word"          length="5"                    />
   <variable name="reset_"          tag="reset"          type="word"          length="5"                    />
-  <variable name="balance_"        tag="balance"        type="word"          length="5"                    />
   <variable name="downscatter_"    tag="downscatter"    type="word"          length="5"                    />
   <variable name="solver_"         tag="solver"         type="word"          default="'power'" length="25" />
   <variable name="snes_monitor_"   tag="snes_monitor"   type="word"          length="5"                    />
   <variable name="ksp_monitor_"    tag="ksp_monitor"    type="word"          length="5"                    />
   <variable name="power_monitor_"  tag="power_monitor"  type="word"          length="5"                    />
-  <variable name="write_balance_"  tag="write_balance"  type="word"          length="5"                    />
   <variable name="write_matrices_" tag="write_matrices" type="word"          length="5"                    />
   <variable name="run_adjoint_"    tag="run_adjoint"    type="word"          length="5"                    />
   <variable name="begin_"          tag="begin"          type="integer"       default="1"                   />
@@ -31,5 +29,6 @@
   <variable name="active_flush_"   tag="active_flush"   type="integer"       default="0"                   />
   <variable name="inactive_flush_" tag="inactive_flush" type="integer"       default="9999"                />
   <variable name="num_flushes_"    tag="num_flushes"    type="integer"       default="1"                   />
+  <variable name="display_"        tag="display"        type="word"          default="''" length="25"      />
 
 </template>

From d28a83cd690186e59d087746a50af27ccbf10f4b Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Fri, 12 Jul 2013 15:37:24 -0400
Subject: [PATCH 29/69] GNU compiler has runtime error with the allocatable
 pointer sometimes, therefore now using an allocatable array and the pointer
 references that

---
 src/vector_header.F90 | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/vector_header.F90 b/src/vector_header.F90
index a201dc9c21..b91f926b07 100644
--- a/src/vector_header.F90
+++ b/src/vector_header.F90
@@ -11,7 +11,8 @@ module vector_header
 
   type, public :: Vector 
     integer :: n        ! number of rows/cols in matrix
-    real(8), pointer :: val(:) ! matrix value vector
+    real(8), allocatable :: data(:) ! where vector data is stored
+    real(8), pointer :: val(:) ! pointer to vector data
 #  ifdef PETSC
     Vec :: petsc_vec
 #  endif
@@ -35,10 +36,11 @@ contains
   subroutine vector_create(self, n)
 
     integer       :: n
-    class(Vector) :: self
+    class(Vector), target :: self
 
     ! Preallocate vector
-    if (.not.associated(self % val)) allocate(self % val(n))
+    if (.not.allocated(self % data)) allocate(self % data(n))
+    self % val => self % data(1:n)
 
     ! Set n
     self % n = n
@@ -63,7 +65,8 @@ contains
     if (self % petsc_active) call VecDestroy(self % petsc_vec, petsc_err)
 #endif
 
-    if (associated(self % val)) deallocate(self % val)
+    if (associated(self % val)) nullify(self % val)
+    if (allocated(self % data)) deallocate(self % data)
 
   end subroutine vector_destroy
 

From 547b71fbfaf43ce24ba89315c2467cdafb3a78c4 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Fri, 12 Jul 2013 15:38:07 -0400
Subject: [PATCH 30/69] add warning if user tries to get dominance ratio
 information and not using power iteration

---
 src/cmfd_input.F90 | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/cmfd_input.F90 b/src/cmfd_input.F90
index d14c65ee93..45cac6b549 100644
--- a/src/cmfd_input.F90
+++ b/src/cmfd_input.F90
@@ -62,7 +62,7 @@ contains
 
   subroutine read_cmfd_xml()
     
-    use error,   only: fatal_error
+    use error,   only: fatal_error, warning
     use output,  only: write_message
     use string,  only: lower_case
     use xml_data_cmfd_t
@@ -192,6 +192,12 @@ contains
 
     ! Get display
     cmfd_display = display_
+    if (trim(cmfd_display) == 'dominance' .and. &
+        trim(cmfd_solver_type) /= 'power') then
+      message = 'Dominance Ratio only aviable with power iteration solver'
+      call warning()
+      cmfd_display = ''
+    end if
 
     ! Create tally objects
     call create_cmfd_tally()

From 70030f788c9e7b0fb16999e99fc03f66510e5a41 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Fri, 12 Jul 2013 16:44:35 -0400
Subject: [PATCH 31/69] matrix row and column properties only accessible
 through get_row and get_col now

---
 src/cmfd_jfnk_solver.F90 | 21 +++----------
 src/matrix_header.F90    | 64 +++++++++++++++++++++++++++++++++-------
 2 files changed, 57 insertions(+), 28 deletions(-)

diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
index 70593a97ef..c84c0321a4 100644
--- a/src/cmfd_jfnk_solver.F90
+++ b/src/cmfd_jfnk_solver.F90
@@ -167,7 +167,6 @@ contains
     integer      :: jloss  ! loop counter for loss matrix cols
     integer      :: jprod  ! loop counter for prod matrix cols
     integer      :: n      ! problem size
-    integer      :: shift  ! integer to account for index shift for Petsc
     real(8)      :: lambda ! eigenvalue
     real(8)      :: val    ! temporary real scalar
     type(Vector) :: flux   ! flux vector
@@ -176,10 +175,6 @@ contains
     ! Get the problem size
     n = loss % n
 
-    ! Check to use shift
-    shift = 0
-    if (loss % petsc_active) shift = 1
-
     ! Get flux and eigenvalue
     flux % val => x % val(1:n)
     lambda = x % val(n + 1)
@@ -199,18 +194,16 @@ contains
       call jac_prec % new_row()
 
       ! Begin loop around columns of loss matrix
-      COLS_LOSS: do jloss = loss % row(i) + shift, &
-                            loss % row(i + 1) - 1 + shift
+      COLS_LOSS: do jloss = loss % get_row(i), loss % get_row(i + 1) - 1
 
         ! Start with the value in the loss matrix
         val = loss % val(jloss)
 
         ! Loop around columns of prod matrix
-        COLS_PROD: do jprod = prod % row(i) + shift, &
-                              prod % row(i + 1) - 1 + shift
+        COLS_PROD: do jprod = prod % get_row(i), prod % get_row(i + 1) - 1
 
           ! See if columns agree with loss matrix
-          if (prod % col(jprod) == loss % col(jloss)) then
+          if (prod % get_col(jprod) == loss % get_col(jloss)) then
             val = val - lambda*prod % val(jprod)
             exit
           end if
@@ -218,7 +211,7 @@ contains
         end do COLS_PROD 
 
         ! Record value in jacobian
-        call jac_prec % add_value(loss % col(jloss) + shift, val)
+        call jac_prec % add_value(loss % get_col(jloss), val)
 
       end do COLS_LOSS
 
@@ -241,12 +234,6 @@ contains
     ! CRS requires a final value in row
     call jac_prec % new_row()
 
-    ! If the Jacobian is already associated with PETSc need to reshift row/col
-    if (jac_prec % petsc_active) then
-      jac_prec % row = jac_prec % row - 1
-      jac_prec % col = jac_prec % col - 1
-    end if
-
     ! Free all allocated memory
     flux % val => null()
     call fsrc % destroy()
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index d85951d527..5b7aa7f725 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -12,8 +12,8 @@ module matrix_header
     integer :: nnz      ! number of nonzeros in matrix
     integer :: n_kount  ! counter for length of matrix
     integer :: nz_kount ! counter for number of non zeros
-    integer, allocatable :: row(:) ! csr row vector
-    integer, allocatable :: col(:) ! column vector
+    integer, private, allocatable :: row(:) ! csr row vector
+    integer, private, allocatable :: col(:) ! column vector
     real(8), allocatable :: val(:) ! matrix value vector
 #  ifdef PETSC
     Mat :: petsc_mat
@@ -25,6 +25,8 @@ module matrix_header
      procedure :: add_value    => matrix_add_value
      procedure :: new_row      => matrix_new_row
      procedure :: assemble     => matrix_assemble
+     procedure :: get_row      => matrix_get_row
+     procedure :: get_col      => matrix_get_col
      procedure :: setup_petsc        => matrix_setup_petsc
      procedure :: write_petsc_binary => matrix_write_petsc_binary
      procedure :: transpose          => matrix_transpose
@@ -91,8 +93,15 @@ contains
     real(8)       :: val
     class(Matrix) :: self
 
+    ! Record the data
     self % col(self % nz_kount) = col
     self % val(self % nz_kount) = val
+
+    ! Need to adjust column indices if PETSc is active
+    if (self % petsc_active) self % col(self % nz_kount) = &
+                             self % col(self % nz_kount) - 1
+
+    ! Increment the number of nonzeros currently stored
     self % nz_kount = self % nz_kount + 1
 
   end subroutine matrix_add_value
@@ -105,7 +114,14 @@ contains
 
     class(Matrix) :: self
 
+    ! Record the current number of nonzeros
     self % row(self % n_kount) = self % nz_kount
+
+    ! If PETSc is active, we have to reference indices off 0
+    if (self % petsc_active) self % row(self % n_kount) = &
+                             self % row(self % n_kount) - 1
+
+    ! Increment the current row that we are on
     self % n_kount = self % n_kount + 1
 
   end subroutine matrix_new_row
@@ -215,6 +231,38 @@ contains
 
   end subroutine split
 
+!===============================================================================
+! MATRIX_GET_ROW
+!===============================================================================
+
+  function matrix_get_row(self, i) result(row)
+
+    class(Matrix) :: self
+    integer       :: i
+    integer       :: row
+
+    row = self % row(i)
+
+    if (self % petsc_active) row = row + 1
+
+  end function matrix_get_row
+
+!===============================================================================
+! MATRIX_GET_COL
+!===============================================================================
+
+  function matrix_get_col(self, i) result(col)
+
+    class(Matrix) :: self
+    integer       :: i
+    integer       :: col
+
+    col = self % col(i)
+
+    if (self % petsc_active) col = col + 1
+
+  end function matrix_get_col
+
 !===============================================================================
 ! MATRIX_SETUP_PETSC configures the row/col vectors and links to a PETSc object
 !===============================================================================
@@ -288,12 +336,6 @@ contains
 
     integer :: i
     integer :: j
-    integer :: shift
-
-    ! Set shift by default 0 and change if PETSc is active
-    ! This is because PETSc needs vectors to remain referenced to 0
-    shift = 0
-    if (self % petsc_active) shift = 1
 
     ! Begin loop around rows
     ROWS: do i = 1, self % n
@@ -301,11 +343,11 @@ contains
       ! Initialize target location in vector
       vec_out % val(i) = ZERO
 
-      ! Begin loop around columns (need to shift if PETSc is active)
-      COLS: do j = self % row(i) + shift, self % row(i + 1) - 1 + shift
+      ! Begin loop around columns
+      COLS: do j = self % get_row(i), self % get_row(i + 1) - 1
 
         vec_out % val(i) = vec_out % val(i) + self % val(j) * &
-                           vec_in % val(self % col(j) + shift)
+                           vec_in % val(self % get_col(j))
 
       end do COLS
 

From 3ae4e46cb8cef003645064053c93292d27611688 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Fri, 12 Jul 2013 16:49:12 -0400
Subject: [PATCH 32/69] added CMFD timers to JFNK solver

---
 src/cmfd_jfnk_solver.F90 | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
index c84c0321a4..c4b0688917 100644
--- a/src/cmfd_jfnk_solver.F90
+++ b/src/cmfd_jfnk_solver.F90
@@ -28,6 +28,8 @@ contains
 
   subroutine cmfd_jfnk_execute(adjoint)
 
+    use global,  only: time_cmfdbuild, time_cmfdsolve
+
     logical, optional :: adjoint ! adjoint calculation
 
     ! Check for adjoint
@@ -37,6 +39,9 @@ contains
     ! Seed with power iteration to help converge to fundamental mode
     call cmfd_power_execute(k_tol=1.E-3_8, s_tol=1.E-3_8, adjoint=adjoint_calc)
 
+    ! Start timer for build
+    call time_cmfdbuild % start()
+
     ! Initialize data
     call init_data()
 
@@ -48,8 +53,13 @@ contains
     jfnk_data % jac_proc_ptr => build_jacobian_matrix
     call jfnk % set_functions(jfnk_data, resvec, jac_prec)
 
+    ! Stop timer for build
+    call time_cmfdbuild % stop()
+
     ! Solve the system
+    call time_cmfdsolve % start()
     call jfnk % solve(xvec)
+    call time_cmfdsolve % stop()
 
     ! Extracts results to cmfd object
     call extract_results()

From 75e583ec07cf8bc0806fb5637451d0a4df29319f Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Wed, 17 Jul 2013 15:34:13 -0400
Subject: [PATCH 33/69] Petsc finalizes only when it is a cmfd run

---
 src/finalize.F90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/finalize.F90 b/src/finalize.F90
index 540726d447..dbc4099c92 100644
--- a/src/finalize.F90
+++ b/src/finalize.F90
@@ -40,7 +40,7 @@ contains
 
 #ifdef PETSC
     ! Finalize PETSc
-    call PetscFinalize(mpi_err)
+    if (cmfd_run) call PetscFinalize(mpi_err)
 #endif
 
     ! Stop timers and show timing statistics

From af38bb0100ab7d4a46e9c0c5896e47ea221fcf8d Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Fri, 19 Jul 2013 13:58:55 -0400
Subject: [PATCH 34/69] changed kount to count

---
 src/cmfd_data.F90        | 24 ++++++++++++------------
 src/cmfd_jfnk_solver.F90 |  4 ++--
 src/matrix_header.F90    | 26 +++++++++++++-------------
 3 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/src/cmfd_data.F90 b/src/cmfd_data.F90
index ef971f47e4..11b49e88ad 100644
--- a/src/cmfd_data.F90
+++ b/src/cmfd_data.F90
@@ -311,13 +311,13 @@ contains
     use constants,  only: CMFD_NOACCEL
     use global,     only: cmfd
 
-    integer :: kount=1 ! counter for unique fuel assemblies
-    integer :: nx      ! number of mesh cells in x direction
-    integer :: ny      ! number of mesh cells in y direction
-    integer :: nz      ! number of mesh cells in z direction
-    integer :: i       ! iteration counter for x
-    integer :: j       ! iteration counter for y
-    integer :: k       ! iteration counter for z
+    integer :: counter=1 ! counter for unique fuel assemblies
+    integer :: nx        ! number of mesh cells in x direction
+    integer :: ny        ! number of mesh cells in y direction
+    integer :: nz        ! number of mesh cells in z direction
+    integer :: i         ! iteration counter for x
+    integer :: j         ! iteration counter for y
+    integer :: k         ! iteration counter for z
 
     ! Extract spatial indices from object
     nx = cmfd % indices(1)
@@ -347,11 +347,11 @@ contains
           else
 
             ! Must be a fuel --> give unique id number
-            cmfd % coremap(i,j,k) = kount
-            cmfd % indexmap(kount,1) = i
-            cmfd % indexmap(kount,2) = j
-            cmfd % indexmap(kount,3) = k
-            kount = kount + 1
+            cmfd % coremap(i,j,k) = counter
+            cmfd % indexmap(counter,1) = i
+            cmfd % indexmap(counter,2) = j
+            cmfd % indexmap(counter,3) = k
+            counter = counter + 1
 
           end if
 
diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
index c4b0688917..04f28ac426 100644
--- a/src/cmfd_jfnk_solver.F90
+++ b/src/cmfd_jfnk_solver.F90
@@ -194,8 +194,8 @@ contains
     call prod % vector_multiply(flux, fsrc)
 
     ! Reset counters in Jacobian matrix
-    jac_prec % n_kount  = 1
-    jac_prec % nz_kount = 1
+    jac_prec % n_count  = 1
+    jac_prec % nz_count = 1
 
     ! Begin loop around rows of Jacobian matrix
     ROWS: do i = 1, n
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index 5b7aa7f725..fe5c2c5bcf 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -10,8 +10,8 @@ module matrix_header
   type, public :: Matrix
     integer :: n        ! number of rows/cols in matrix
     integer :: nnz      ! number of nonzeros in matrix
-    integer :: n_kount  ! counter for length of matrix
-    integer :: nz_kount ! counter for number of non zeros
+    integer :: n_count  ! counter for length of matrix
+    integer :: nz_count ! counter for number of non zeros
     integer, private, allocatable :: row(:) ! csr row vector
     integer, private, allocatable :: col(:) ! column vector
     real(8), allocatable :: val(:) ! matrix value vector
@@ -53,8 +53,8 @@ contains
     if (.not.allocated(self % val)) allocate(self % val(nnz))
 
     ! Set counters to 1
-    self % n_kount  = 1
-    self % nz_kount = 1
+    self % n_count  = 1
+    self % nz_count = 1
 
     ! Set n and nnz
     self % n = n
@@ -94,15 +94,15 @@ contains
     class(Matrix) :: self
 
     ! Record the data
-    self % col(self % nz_kount) = col
-    self % val(self % nz_kount) = val
+    self % col(self % nz_count) = col
+    self % val(self % nz_count) = val
 
     ! Need to adjust column indices if PETSc is active
-    if (self % petsc_active) self % col(self % nz_kount) = &
-                             self % col(self % nz_kount) - 1
+    if (self % petsc_active) self % col(self % nz_count) = &
+                             self % col(self % nz_count) - 1
 
     ! Increment the number of nonzeros currently stored
-    self % nz_kount = self % nz_kount + 1
+    self % nz_count = self % nz_count + 1
 
   end subroutine matrix_add_value
 
@@ -115,14 +115,14 @@ contains
     class(Matrix) :: self
 
     ! Record the current number of nonzeros
-    self % row(self % n_kount) = self % nz_kount
+    self % row(self % n_count) = self % nz_count
 
     ! If PETSc is active, we have to reference indices off 0
-    if (self % petsc_active) self % row(self % n_kount) = &
-                             self % row(self % n_kount) - 1
+    if (self % petsc_active) self % row(self % n_count) = &
+                             self % row(self % n_count) - 1
 
     ! Increment the current row that we are on
-    self % n_kount = self % n_kount + 1
+    self % n_count = self % n_count + 1
 
   end subroutine matrix_new_row
 

From c0772d6b485e43bdb33f1cf8b0a3fb79550d230a Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Sat, 27 Jul 2013 15:08:58 -0400
Subject: [PATCH 35/69] added more compiler directives

---
 src/cmfd_execute.F90 |  5 ++++-
 src/petsc_solver.F90 | 20 ++++++--------------
 2 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/src/cmfd_execute.F90 b/src/cmfd_execute.F90
index a514b0cd8b..ebfaa0aea4 100644
--- a/src/cmfd_execute.F90
+++ b/src/cmfd_execute.F90
@@ -240,8 +240,10 @@ contains
 
     end if
 
+#ifdef MPI
     ! Broadcast full source to all procs
     call MPI_BCAST(cmfd % cmfd_src, n, MPI_REAL8, 0, MPI_COMM_WORLD, mpi_err)
+#endif
 
   end subroutine calc_fission_source
 
@@ -330,9 +332,10 @@ contains
       end if
 
       ! Broadcast weight factors to all procs
+#ifdef MPI
       call MPI_BCAST(cmfd % weightfactors, ng*nx*ny*nz, MPI_REAL8, 0, &
            MPI_COMM_WORLD, mpi_err)
-
+#endif
    end if
 
     ! Begin loop over source bank
diff --git a/src/petsc_solver.F90 b/src/petsc_solver.F90
index 8318ca174b..87c288b936 100644
--- a/src/petsc_solver.F90
+++ b/src/petsc_solver.F90
@@ -196,6 +196,7 @@ contains
     type(Vector)      :: res
     type(Matrix)      :: jac_prec
 
+# ifdef PETSC
     ! Set residual procedure
     call SNESSetFunction(self % snes, res % petsc_vec, jfnk_compute_residual, &
          ctx, petsc_err)
@@ -210,6 +211,7 @@ contains
     ! Set up Jacobian Lags
     call SNESSetLagJacobian(self % snes, -2, petsc_err)
     call SNESSetLagPreconditioner(self % snes, -1, petsc_err)
+#endif
 
   end subroutine jfnk_set_functions
 
@@ -231,7 +233,7 @@ contains
 !===============================================================================
 ! JFNK_COMPUTE_RESIDUAL buffer routine to user specifed residual routine
 !===============================================================================
-
+# ifdef PETSC
   subroutine jfnk_compute_residual(snes, x, res, ctx, ierr)
 
     SNES           :: snes
@@ -248,26 +250,22 @@ contains
     ! for the residual and x vectors here
 
     ! Need to point an OpenMC vector to PETSc vector
-#ifdef PETSC
     call VecGetArrayF90(x, xvec % val, ierr)
     call VecGetArrayF90(res, resvec % val, ierr)
-#endif
 
     ! Call user residual routine
     call ctx % res_proc_ptr(xvec, resvec)
 
     ! Need to restore the PETSc vector
-#ifdef PETSC
     call VecRestoreArrayF90(x, xvec % val, ierr)
     call VecRestoreArrayF90(res, resvec % val, ierr)
-#endif
 
   end subroutine jfnk_compute_residual
-
+#endif
 !===============================================================================
 ! JFNK_COMPUTE_JACOBIAN
 !===============================================================================
-
+#ifdef PETSC
   subroutine jfnk_compute_jacobian(snes, x, jac_mf, jac_prec, flag, ctx, &
              ierr)
 
@@ -285,23 +283,17 @@ contains
     ! matrix
 
     ! Need to point OpenMC vector to PETSc Vector
-#ifdef PETSC
     call VecGetArrayF90(x, xvec % val, ierr)
-#endif
 
     ! Evaluate user jacobian routine
     call ctx % jac_proc_ptr(xvec)
 
     ! Restore the PETSc vector
-#ifdef PETSC
     call VecRestoreArrayF90(x, xvec % val, ierr)
-#endif
 
-#ifdef PETSC
     call MatAssemblyBegin(jac_mf, MAT_FINAL_ASSEMBLY, petsc_err)
     call MatAssemblyEnd(jac_mf, MAT_FINAL_ASSEMBLY, petsc_err)
-#endif
 
   end subroutine jfnk_compute_jacobian
-
+#endif
 end module petsc_solver

From 3cac0afe98f9f9d8a551b36dc95b42a0078687c9 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Sat, 27 Jul 2013 15:10:45 -0400
Subject: [PATCH 36/69] changed CMFD userguide documentation

---
 docs/source/usersguide/input.rst | 66 +++++++++++++++++++++-----------
 1 file changed, 44 insertions(+), 22 deletions(-)

diff --git a/docs/source/usersguide/input.rst b/docs/source/usersguide/input.rst
index d9b43c8214..2bd833cfa5 100644
--- a/docs/source/usersguide/input.rst
+++ b/docs/source/usersguide/input.rst
@@ -1131,6 +1131,22 @@ The ``<begin>`` element controls what batch CMFD calculations should begin.
 
   *Default*: 1
 
+``<display>`` Element
+---------------------
+
+The ``<display>`` element sets one additional CMFD output column. Options are:
+
+* "balance" - prints the RMS [%] of the resdiual from the neutron balance equation
+  on CMFD tallies.
+* "dominance" - prints the estimated dominance ratio from the CMFD iterations.
+  **This will only work for power iteration eigensolver**.
+* "entropy" - prints the *entropy* of the CMFD predicted fission source.
+  **Can only be used if OpenMC entropy is active as well**.
+* "source" - prints the RMS [%] between the OpenMC fission source and CMFD
+  fission source.
+
+  *Default*: None
+
 ``<feedback>`` Element
 ----------------------
 
@@ -1150,14 +1166,14 @@ with "true" and off with "false"
 
   *Default*: true
 
-``<keff_tol>`` Element
-----------------------
+``<inactive_flush>`` Element
+----------------------------
 
-The ``<keff_tol>`` element specifies acceptance criteria of a CMFD eigenvalue.
-If the CMFD eigenvalue and OpenMC batch eigenvalue are within this tolerance, 
-CMFD is allowed to modify source neutron weights. 
+The ``<inactive_flush>`` element controls when CMFD tallies are reset during
+inactive batches. The integer set here is the interval at which this reset
+occurs. The amout of resets is controlled with the ``<num_flushes>`` element.
 
-  *Default*: 0.005
+  *Defualt*: 9999
 
 ``<ksp_monitor>`` Element
 -------------------------
@@ -1241,14 +1257,13 @@ not impact the calculation.
 
   *Default*: 1.0
 
-``<n_procs_cmfd>`` Element
---------------------------
+``<num_flushes>`` Element
+-------------------------
 
-The ``<n_procs_cmfd>`` element is used to set the number of processors used 
-for CMFD calculation. It should be less than or equal to the number of 
-processors used during OpenMC. 
+The ``<num_flushes>`` element controls the number of CMFD tally resets that
+occur during inactive CMFD batches.
 
-  *Default*: 1
+  *Default*: 9999
 
 ``<power_monitor>`` Element
 ---------------------------
@@ -1256,26 +1271,33 @@ processors used during OpenMC.
 The ``<power_monitor>`` element is used to view the convergence of power iteration. 
 This option can be turned on with "true" and turned off with "false".
 
+  *Default*: false
+
+``<run_adjoint>`` Element
+-------------------------
+
+The ``<run_adjoint>`` element can be turned on with "true" to have an adjoint
+calculation be performed on the last batch when CMFD is active.
 
   *Default*: false
 
-``<write_balance>`` Element
----------------------------
+``<snes_monitor>`` Element
+--------------------------
 
-The ``<write_balance>`` element is used to view the balance of OpenMC tally
-residuals for every coarse mesh region and energy group. This option can be 
-turned on with "true" and off with "false". 
+The ``<snes_monitor>`` element is used to view the convergence of the nonlinear SNES
+function in PETSc. This option can be turned on with "true" and turned off with "false".
 
 
   *Default*: false
 
-``<write_hdf5>`` Element
-------------------------
+``<solver>`` Element
+--------------------
 
-The ``<write_hdf5>`` element can be turned on with "true" to get an 
-HDF5 output file of CMFD results. 
+The ``<solver>`` element controls whether the CMFD eigenproblem is solved with
+standard power iteration or nonlinear Jacobian-free Newton Krylov (JFNK). 
+By setting "power", power iteration is used and by setting "jfnk", JFNK is used.
 
-  *Default*: false
+  *Default*: power
 
 ``<write_matrices>`` Element
 ----------------------------

From c7e46dc462645e3560980bdadc2dfa65ea7324e4 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Sat, 27 Jul 2013 17:03:02 -0400
Subject: [PATCH 37/69] important CMFD information now written to statepoint,
 also CMFD can be restarted now

---
 src/cmfd_execute.F90     |  3 ++
 src/cmfd_header.F90      |  6 ++++
 src/hdf5_interface.F90   | 47 ++++++++++++++++++++++++++++
 src/output.F90           |  2 +-
 src/output_interface.F90 | 67 ++++++++++++++++++++++++++++++++++++++++
 src/state_point.F90      | 43 ++++++++++++++++++++++++++
 6 files changed, 167 insertions(+), 1 deletion(-)

diff --git a/src/cmfd_execute.F90 b/src/cmfd_execute.F90
index ebfaa0aea4..33f63b1b2f 100644
--- a/src/cmfd_execute.F90
+++ b/src/cmfd_execute.F90
@@ -46,6 +46,9 @@ contains
         call fatal_error() 
       end if
 
+      ! Save k-effective
+      cmfd % k_cmfd(current_batch) = cmfd % keff
+
       ! check to perform adjoint on last batch
       if (current_batch == n_batches .and. cmfd_run_adjoint) then
         if (trim(cmfd_solver_type) == 'power') then
diff --git a/src/cmfd_header.F90 b/src/cmfd_header.F90
index 3d6b3fd61a..3eebb0f6d9 100644
--- a/src/cmfd_header.F90
+++ b/src/cmfd_header.F90
@@ -80,6 +80,9 @@ module cmfd_header
     ! Dominance ratio
     real(8), allocatable :: dom(:)
 
+    ! List of CMFD k
+    real(8), allocatable :: k_cmfd(:)
+
   end type cmfd_type
 
 contains
@@ -136,6 +139,7 @@ contains
     if (.not. allocated(this % balance)) allocate(this % balance(n_batches))
     if (.not. allocated(this % src_cmp)) allocate(this % src_cmp(n_batches))
     if (.not. allocated(this % dom)) allocate(this % dom(n_batches))
+    if (.not. allocated(this % k_cmfd)) allocate(this % k_cmfd(n_batches))
 
     ! Set everthing to 0 except weight multiply factors if feedback isnt on
     this % flux          = ZERO
@@ -155,6 +159,7 @@ contains
     this % balance       = ZERO
     this % src_cmp       = ZERO
     this % dom           = ZERO
+    this % k_cmfd        = ZERO
     if (entropy_on) this % entropy = ZERO
 
   end subroutine allocate_cmfd
@@ -188,6 +193,7 @@ contains
     if (allocated(this % balance))       deallocate(this % balance)
     if (allocated(this % src_cmp))       deallocate(this % src_cmp)
     if (allocated(this % dom))           deallocate(this % dom)
+    if (allocated(this % k_cmfd))        deallocate(this % k_cmfd)
     if (allocated(this % entropy))       deallocate(this % entropy)
 
   end subroutine deallocate_cmfd
diff --git a/src/hdf5_interface.F90 b/src/hdf5_interface.F90
index 471a4397cb..0886fbf731 100644
--- a/src/hdf5_interface.F90
+++ b/src/hdf5_interface.F90
@@ -23,12 +23,16 @@ module hdf5_interface
   integer(HSIZE_T) :: dims1(1)   ! dims type for 1-D array
   integer(HSIZE_T) :: dims2(2)   ! dims type for 2-D array
   integer(HSIZE_T) :: dims3(3)   ! dims type for 3-D array
+  integer(HSIZE_T) :: dims4(4)   ! dims type for 4-D array
   type(c_ptr)      :: f_ptr      ! pointer to data
 
   ! Generic HDF5 write procedure interface
   interface hdf5_write_data
     module procedure hdf5_write_double
     module procedure hdf5_write_double_1Darray
+    module procedure hdf5_write_double_2Darray
+    module procedure hdf5_write_double_3Darray
+    module procedure hdf5_write_double_4Darray
     module procedure hdf5_write_integer
     module procedure hdf5_write_integer_1Darray
     module procedure hdf5_write_long
@@ -39,6 +43,7 @@ module hdf5_interface
   interface hdf5_read_data
     module procedure hdf5_read_double
     module procedure hdf5_read_double_1Darray
+    module procedure hdf5_read_double_4Darray
     module procedure hdf5_read_integer
     module procedure hdf5_read_integer_1Darray
     module procedure hdf5_read_long
@@ -381,6 +386,28 @@ contains
 
   end subroutine hdf5_write_double_3Darray
 
+!===============================================================================
+! HDF5_WRITE_DOUBLE_4DARRAY writes double precision 4-D aray
+!===============================================================================
+
+  subroutine hdf5_write_double_4Darray(group, name, buffer, length)
+
+    integer,        intent(in) :: length(4) ! length of array dimensions
+    integer(HID_T), intent(in) :: group     ! name of group
+    character(*),   intent(in) :: name      ! name of data
+    real(8),        intent(in) :: buffer(length(1),length(2),& ! data
+                                         length(3),length(4))
+
+    ! Set rank and dimensions
+    hdf5_rank = 4
+    dims4 = length
+
+    ! Write data
+    call h5ltmake_dataset_double_f(group, name, hdf5_rank, dims4, &
+         buffer, hdf5_err)
+
+  end subroutine hdf5_write_double_4Darray
+
 !===============================================================================
 ! HDF5_WRITE_STRING writes string data
 !===============================================================================
@@ -552,6 +579,26 @@ contains
 
   end subroutine hdf5_read_double_1Darray
 
+!===============================================================================
+! HDF5_READ_DOUBLE_4DARRAY reads double precision 4-D array
+!===============================================================================
+
+  subroutine hdf5_read_double_4Darray(group, name, buffer, length)
+
+    integer(HID_T), intent(in)    :: group     ! name of group
+    integer,        intent(in)    :: length(:) ! length of array
+    character(*),   intent(in)    :: name      ! name of data
+    real(8),        intent(inout) :: buffer(length(1), length(2),& ! read data to here
+                                            length(3), length(4))
+
+    ! Set dimensions of data
+    dims4 = length
+
+    ! Read data
+    call h5ltread_dataset_double_f(group, name, buffer, dims4, hdf5_err)
+
+  end subroutine hdf5_read_double_4Darray
+
 !===============================================================================
 ! HDF5_READ_STRING reads string data
 !===============================================================================
diff --git a/src/output.F90 b/src/output.F90
index 2b6b04511d..68bb46764a 100644
--- a/src/output.F90
+++ b/src/output.F90
@@ -1300,7 +1300,7 @@ contains
     ! write out cmfd keff if it is active and other display info
     if (cmfd_on) then
       write(UNIT=OUTPUT_UNIT, FMT='(3X, F8.5)', ADVANCE='NO') &
-         cmfd % keff 
+         cmfd % k_cmfd(current_batch) 
       select case(trim(cmfd_display))
         case('entropy')
           write(UNIT=OUTPUT_UNIT, FMT='(3X, F8.5)', ADVANCE='NO') &
diff --git a/src/output_interface.F90 b/src/output_interface.F90
index 5f421a6d25..17f14f16d5 100644
--- a/src/output_interface.F90
+++ b/src/output_interface.F90
@@ -19,6 +19,7 @@ module output_interface
     module procedure write_double_1Darray
     module procedure write_double_2Darray
     module procedure write_double_3Darray
+    module procedure write_double_4Darray
     module procedure write_integer
     module procedure write_integer_1Darray
     module procedure write_integer_2Darray
@@ -31,6 +32,7 @@ module output_interface
   interface read_data
     module procedure read_double
     module procedure read_double_1Darray
+    module procedure read_double_4Darray
     module procedure read_integer
     module procedure read_integer_1Darray
     module procedure read_long
@@ -367,6 +369,71 @@ contains
 
   end subroutine write_double_3Darray
 
+!===============================================================================
+! WRITE_DOUBLE_4DARRAY writes double precision 4-D array data
+!===============================================================================
+
+  subroutine write_double_4Darray(buffer, name, group, length)
+
+    integer,      intent(in)           :: length(4) ! length of each dimension
+    real(8),      intent(in)           :: buffer(length(1),length(2),&
+                                                 length(3),length(4))
+    character(*), intent(in)           :: name ! name of data
+    character(*), intent(in), optional :: group ! HDF5 group name
+
+#ifdef HDF5
+    ! Check if HDF5 group should be created/opened
+    if (present(group)) then
+      call hdf5_open_group(group)
+    else
+      temp_group = hdf5_fh
+    endif
+
+    ! Write the data
+    call hdf5_write_double_4Darray(temp_group, name, buffer, length)
+
+    ! Check if HDF5 group should be closed
+    if (present(group)) call hdf5_close_group()
+#else
+    message = 'Double precision 4-D array writing not currently supported'
+    call warning()
+#endif
+
+  end subroutine write_double_4Darray
+
+!===============================================================================
+! READ_DOUBLE_4DARRAY reads double precision 4-D array data
+!===============================================================================
+
+  subroutine read_double_4Darray(buffer, name, group, length, option)
+
+    integer,      intent(in)           :: length(4) ! length of each dimension
+    real(8),      intent(inout)        :: buffer(length(1),length(2),&
+                                                 length(3),length(4))
+    character(*), intent(in)           :: name ! name of data
+    character(*), intent(in), optional :: group ! HDF5 group name
+    character(*), intent(in), optional :: option    ! read option
+
+#ifdef HDF5
+    ! Check if HDF5 group should be created/opened
+    if (present(group)) then
+      call hdf5_open_group(group)
+    else
+      temp_group = hdf5_fh
+    endif
+
+    ! Write the data
+    call hdf5_read_double_4Darray(temp_group, name, buffer, length)
+
+    ! Check if HDF5 group should be closed
+    if (present(group)) call hdf5_close_group()
+#else
+    message = 'Double precision 4-D array reading not currently supported'
+    call warning()
+#endif
+
+  end subroutine read_double_4Darray
+
 !===============================================================================
 ! WRITE_INTEGER writes integer scalar data
 !===============================================================================
diff --git a/src/state_point.F90 b/src/state_point.F90
index 5020f8f3d0..dd3f3e785f 100644
--- a/src/state_point.F90
+++ b/src/state_point.F90
@@ -94,6 +94,27 @@ contains
         call write_data(k_col_tra, "k_col_tra")
         call write_data(k_abs_tra, "k_abs_tra")
         call write_data(k_combined, "k_combined", length=2)
+
+        ! Write out CMFD info
+        if (cmfd_on) then
+          call write_data(1, "cmfd_on")
+          call write_data(cmfd % k_cmfd, "k_cmfd", length=current_batch, &
+               group="cmfd")
+          call write_data(cmfd % cmfd_src, "cmfd_src", &
+               length=(/cmfd % indices(4), cmfd % indices(1), &
+               cmfd % indices(2), cmfd % indices(3)/), &
+               group="cmfd")
+          if (entropy_on) call write_data(cmfd % entropy, "cmfd_entropy", &
+                            length=current_batch, group="cmfd")
+          call write_data(cmfd % balance, "cmfd_balance", &
+               length=current_batch, group="cmfd")
+          call write_data(cmfd % dom, "cmfd_dominance", &
+               length = current_batch, group="cmfd")
+          call write_data(cmfd % src_cmp, "cmfd_srccmp", &
+               length = current_batch, group="cmfd")
+        else
+          call write_data(0, "cmfd_on")
+        end if
       end if
 
       ! Write number of meshes
@@ -516,6 +537,28 @@ contains
 
       ! Take maximum of statepoint n_inactive and input n_inactive
       n_inactive = max(n_inactive, int_array(1))
+
+      ! Read in to see if CMFD was on
+      call read_data(int_array(1), "cmfd_on", option="collective")
+
+      ! Write out CMFD info
+      if (int_array(1) == 1) then
+        call read_data(cmfd % k_cmfd, "k_cmfd", length=restart_batch, &
+             group="cmfd", option="collective")
+        call read_data(cmfd % cmfd_src, "cmfd_src", &
+             length=(/cmfd % indices(4), cmfd % indices(1), &
+             cmfd % indices(2), cmfd % indices(3)/), &
+             group="cmfd", option="collective")
+        if (entropy_on) call read_data(cmfd % entropy, "cmfd_entropy", &
+                          length=restart_batch, group="cmfd", &
+                          option="collective")
+        call read_data(cmfd % balance, "cmfd_balance", &
+             length=restart_batch, group="cmfd", option="collective")
+        call read_data(cmfd % dom, "cmfd_dominance", &
+             length = restart_batch, group="cmfd", option="collective")
+        call read_data(cmfd % src_cmp, "cmfd_srccmp", &
+             length = restart_batch, group="cmfd", option="collective")
+      end if
     end if
 
     ! Read number of meshes

From 63a4574134615ec0863929d1c86badc766f2e3cd Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Sat, 27 Jul 2013 17:03:54 -0400
Subject: [PATCH 38/69] Default PETSc version is 3.4.2 and current code will
 only compile with this version

---
 src/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Makefile b/src/Makefile
index 4c301483a2..6c72802265 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -31,7 +31,7 @@ PETSC    = no
 MPI_DIR   = /opt/mpich/3.0.4-$(COMPILER)
 HDF5_DIR  = /opt/hdf5/1.8.11-$(COMPILER)
 PHDF5_DIR = /opt/phdf5/1.8.11-$(COMPILER)
-PETSC_DIR = /opt/petsc/3.3-p6-$(COMPILER)
+PETSC_DIR = /opt/petsc/3.4.2-$(COMPILER)
 
 #===============================================================================
 # Add git SHA-1 hash

From 9d81de66cb03a1a800bd33ce423946c3a1ddde71 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Thu, 8 Aug 2013 18:30:23 -0400
Subject: [PATCH 39/69] fixed bug with for multigroup CMFD and fixed relative
 source error in power iteration

---
 src/cmfd_loss_operator.F90 |  2 +-
 src/cmfd_power_solver.F90  | 53 +++++++++++++++++++++-----------------
 src/cmfd_prod_operator.F90 |  2 +-
 3 files changed, 31 insertions(+), 26 deletions(-)

diff --git a/src/cmfd_loss_operator.F90 b/src/cmfd_loss_operator.F90
index 22fa8dfa14..e09497bedc 100644
--- a/src/cmfd_loss_operator.F90
+++ b/src/cmfd_loss_operator.F90
@@ -410,7 +410,7 @@ contains
     if (cmfd_coremap) then
 
       ! Get indices from indexmap
-      g = mod(irow, ng) + 1
+      g = mod(irow-1, ng) + 1
       i = cmfd % indexmap((irow-1)/ng+1,1)
       j = cmfd % indexmap((irow-1)/ng+1,2)
       k = cmfd % indexmap((irow-1)/ng+1,3)
diff --git a/src/cmfd_power_solver.F90 b/src/cmfd_power_solver.F90
index 1d96ea2944..9c7eff0eda 100644
--- a/src/cmfd_power_solver.F90
+++ b/src/cmfd_power_solver.F90
@@ -19,13 +19,16 @@ module cmfd_power_solver
   real(8) :: stol = 1.e-8_8       ! tolerance on source
   real(8) :: norm_n               ! current norm of source vector
   real(8) :: norm_o               ! old norm of source vector
+  real(8) :: kerr                 ! error in keff
+  real(8) :: serr                 ! error in source
   logical :: adjoint_calc         ! run an adjoint calculation
   type(Matrix) :: loss            ! cmfd loss matrix
   type(Matrix) :: prod            ! cmfd prod matrix
   type(Vector) :: phi_n           ! new flux vector
   type(Vector) :: phi_o           ! old flux vector
-  type(Vector) :: S_n             ! new source vector
-  type(Vector) :: S_o             ! old flux vector
+  type(Vector) :: s_n             ! new source vector
+  type(Vector) :: s_o             ! old flux vector
+  type(Vector) :: serr_v          ! error in source
   type(Petsc_gmres) :: gmres      ! gmres solver
 
 contains
@@ -113,8 +116,9 @@ contains
     call phi_o % create(n)
 
     ! Set up source vectors
-    call S_n % create(n)
-    call S_o % create(n)
+    call s_n % create(n)
+    call s_o % create(n)
+    call serr_v % create(n)
 
     ! Set initial guess
     guess = ONE
@@ -136,8 +140,8 @@ contains
     call prod % setup_petsc()
     call phi_n % setup_petsc()
     call phi_o % setup_petsc()
-    call S_o % setup_petsc()
-    call S_n % setup_petsc()
+    call s_o % setup_petsc()
+    call s_n % setup_petsc()
 
     ! Set norms to 0
     norm_n = ZERO
@@ -181,22 +185,22 @@ contains
     do i = 1, 10000
 
       ! Compute source vector
-      call prod % vector_multiply(phi_o, S_o)
+      call prod % vector_multiply(phi_o, s_o)
 
       ! Normalize source vector
-      S_o % val = S_o % val / k_o
+      s_o % val = s_o % val / k_o
 
       ! Compute new flux vector
-      call gmres % solve(S_o, phi_n)
+      call gmres % solve(s_o, phi_n)
 
       ! Compute new source vector
-      call prod % vector_multiply(phi_n, S_n)
+      call prod % vector_multiply(phi_n, s_n)
 
       ! Compute new k-eigenvalue
-      k_n = sum(S_n % val) / sum(S_o % val)
+      k_n = sum(s_n % val) / sum(s_o % val)
 
       ! Renormalize the old source
-      S_o % val = S_o % val * k_o
+      s_o % val = s_o % val * k_o
 
       ! Check convergence
       call convergence(i)
@@ -219,15 +223,12 @@ contains
 
   subroutine convergence(iter)
 
-    use constants,  only: ONE
+    use constants,  only: ONE, TINY_BIT
     use global,     only: cmfd_power_monitor, master
     use, intrinsic :: ISO_FORTRAN_ENV
 
     integer     :: iter           ! iteration number
 
-    real(8)     :: kerr           ! error in keff
-    real(8)     :: serr           ! error in source
-
     ! Reset convergence flag
     iconv = .false.
 
@@ -235,7 +236,10 @@ contains
     kerr = abs(k_o - k_n)/k_n
 
     ! Calculate max error in source
-    serr = sqrt(ONE/dble(S_n % n) * sum(((S_n % val - S_o % val)/S_n % val)**2))
+    where (s_n % val > TINY_BIT)
+      serr_v % val = ((s_n % val - s_o % val)/s_n % val)**2
+    end where
+    serr = sqrt(ONE/dble(s_n % n) * sum(serr_v % val))
 
     ! Check for convergence
     if(kerr < ktol .and. serr < stol) iconv = .true.
@@ -315,13 +319,14 @@ contains
   subroutine finalize()
 
     ! Destroy all objects 
-    call gmres % destroy() 
-    call loss  % destroy() 
-    call prod  % destroy()
-    call phi_n % destroy()
-    call phi_o % destroy()
-    call S_n   % destroy()
-    call S_o   % destroy()
+    call gmres  % destroy() 
+    call loss   % destroy() 
+    call prod   % destroy()
+    call phi_n  % destroy()
+    call phi_o  % destroy()
+    call s_n    % destroy()
+    call s_o    % destroy()
+    call serr_v % destroy
 
   end subroutine finalize
 
diff --git a/src/cmfd_prod_operator.F90 b/src/cmfd_prod_operator.F90
index 4c6a28fb88..c2db9fc83c 100644
--- a/src/cmfd_prod_operator.F90
+++ b/src/cmfd_prod_operator.F90
@@ -179,7 +179,7 @@ contains
     if (cmfd_coremap) then
 
       ! get indices from indexmap
-      g = mod(irow, ng) + 1
+      g = mod(irow-1, ng) + 1
       i = cmfd % indexmap((irow-1)/ng+1,1)
       j = cmfd % indexmap((irow-1)/ng+1,2)
       k = cmfd % indexmap((irow-1)/ng+1,3)

From a72685818699fba737c85a557a8384084af3f3d2 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Fri, 9 Aug 2013 13:56:48 -0400
Subject: [PATCH 40/69] fixed tally resetting on restart run and writing
 message for core overlay

---
 src/cmfd_execute.F90 | 3 +++
 src/cmfd_input.F90   | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/cmfd_execute.F90 b/src/cmfd_execute.F90
index 33f63b1b2f..f9f3eb2de8 100644
--- a/src/cmfd_execute.F90
+++ b/src/cmfd_execute.F90
@@ -89,6 +89,9 @@ contains
       cmfd_tally_on = .true.
     end if
 
+    ! If this is a restart run and we are just replaying batches leave
+    if (restart_run .and. current_batch <= restart_batch) return
+
     ! Check to flush cmfd tallies for active batches, no more inactive flush
     if (cmfd_run .and. cmfd_act_flush == current_batch) then
       call cmfd_tally_reset()
diff --git a/src/cmfd_input.F90 b/src/cmfd_input.F90
index 45cac6b549..a836279d51 100644
--- a/src/cmfd_input.F90
+++ b/src/cmfd_input.F90
@@ -136,7 +136,7 @@ contains
 
       ! Write to stdout that a core map overlay is active
       message = "Core Map Overlay Activated"
-      call write_message()
+      call write_message(10)
 
     end if
 

From 2b8e57b42dcde209d8f5efb59492c174b0d8da9a Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Thu, 15 Aug 2013 11:21:23 -0400
Subject: [PATCH 41/69] dependencies now sorted alphabetically in build script

---
 src/utils/build_dependencies.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/utils/build_dependencies.py b/src/utils/build_dependencies.py
index a2fd261f26..c88ddbfb62 100755
--- a/src/utils/build_dependencies.py
+++ b/src/utils/build_dependencies.py
@@ -20,7 +20,7 @@ for src in glob.iglob('*.F90'):
         dependencies[module] = sorted(list(deps))
 
 
-for module in dependencies.keys():
+for module in sorted(dependencies.keys()):
     for dep in dependencies[module]:
         print("{0}.o: {1}.o".format(module, dep))
     print('')

From 85e8ec9e24bb14bcd23b57c1d7a3780e023ba2a2 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Thu, 15 Aug 2013 13:57:14 -0400
Subject: [PATCH 42/69] fixed up cmfd statepoint writing/reading, updated
 python statepoint file to reflect changes and incremented statepoint revision
 number

---
 src/cmfd_header.F90     |  7 +++----
 src/cmfd_input.F90      |  2 +-
 src/constants.F90       |  2 +-
 src/state_point.F90     | 11 +++++++----
 src/utils/statepoint.py | 17 +++++++++++++++++
 5 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/src/cmfd_header.F90 b/src/cmfd_header.F90
index 3eebb0f6d9..42e6b30ef9 100644
--- a/src/cmfd_header.F90
+++ b/src/cmfd_header.F90
@@ -91,10 +91,9 @@ contains
 ! ALLOCATE_CMFD allocates all data in of cmfd type
 !==============================================================================
 
-  subroutine allocate_cmfd(this, n_batches, entropy_on)
+  subroutine allocate_cmfd(this, n_batches)
 
     integer :: n_batches
-    logical :: entropy_on
     type(cmfd_type) :: this
 
     integer :: nx  ! number of mesh cells in x direction
@@ -135,7 +134,7 @@ contains
     if (.not. allocated(this % weightfactors)) allocate(this % weightfactors(ng,nx,ny,nz))
 
     ! Allocate batchwise parameters
-    if (.not. allocated(this % entropy) .and. entropy_on) allocate(this % entropy(n_batches))
+    if (.not. allocated(this % entropy)) allocate(this % entropy(n_batches))
     if (.not. allocated(this % balance)) allocate(this % balance(n_batches))
     if (.not. allocated(this % src_cmp)) allocate(this % src_cmp(n_batches))
     if (.not. allocated(this % dom)) allocate(this % dom(n_batches))
@@ -160,7 +159,7 @@ contains
     this % src_cmp       = ZERO
     this % dom           = ZERO
     this % k_cmfd        = ZERO
-    if (entropy_on) this % entropy = ZERO
+    this % entropy       = ZERO
 
   end subroutine allocate_cmfd
 
diff --git a/src/cmfd_input.F90 b/src/cmfd_input.F90
index a836279d51..de23b89b09 100644
--- a/src/cmfd_input.F90
+++ b/src/cmfd_input.F90
@@ -52,7 +52,7 @@ contains
     call time_cmfdsolve % reset()
 
     ! Allocate cmfd object
-    call allocate_cmfd(cmfd, n_batches, entropy_on)
+    call allocate_cmfd(cmfd, n_batches)
 
   end subroutine configure_cmfd
 
diff --git a/src/constants.F90 b/src/constants.F90
index 1a4080f622..22b1e8e84e 100644
--- a/src/constants.F90
+++ b/src/constants.F90
@@ -11,7 +11,7 @@ module constants
   integer, parameter :: VERSION_RELEASE = 2
 
   ! Revision numbers for binary files
-  integer, parameter :: REVISION_STATEPOINT       = 9
+  integer, parameter :: REVISION_STATEPOINT       = 10
   integer, parameter :: REVISION_PARTICLE_RESTART = 1
 
   ! Binary file types
diff --git a/src/state_point.F90 b/src/state_point.F90
index dd3f3e785f..b0d88946eb 100644
--- a/src/state_point.F90
+++ b/src/state_point.F90
@@ -98,14 +98,15 @@ contains
         ! Write out CMFD info
         if (cmfd_on) then
           call write_data(1, "cmfd_on")
+          call write_data(cmfd % indices, "indicies", length=4, group="cmfd")
           call write_data(cmfd % k_cmfd, "k_cmfd", length=current_batch, &
                group="cmfd")
           call write_data(cmfd % cmfd_src, "cmfd_src", &
                length=(/cmfd % indices(4), cmfd % indices(1), &
                cmfd % indices(2), cmfd % indices(3)/), &
                group="cmfd")
-          if (entropy_on) call write_data(cmfd % entropy, "cmfd_entropy", &
-                            length=current_batch, group="cmfd")
+          call write_data(cmfd % entropy, "cmfd_entropy", &
+                          length=current_batch, group="cmfd")
           call write_data(cmfd % balance, "cmfd_balance", &
                length=current_batch, group="cmfd")
           call write_data(cmfd % dom, "cmfd_dominance", &
@@ -543,14 +544,16 @@ contains
 
       ! Write out CMFD info
       if (int_array(1) == 1) then
+        call read_data(cmfd % indices, "indicies", length=4, group="cmfd", &
+                       option="collective")
         call read_data(cmfd % k_cmfd, "k_cmfd", length=restart_batch, &
              group="cmfd", option="collective")
         call read_data(cmfd % cmfd_src, "cmfd_src", &
              length=(/cmfd % indices(4), cmfd % indices(1), &
              cmfd % indices(2), cmfd % indices(3)/), &
              group="cmfd", option="collective")
-        if (entropy_on) call read_data(cmfd % entropy, "cmfd_entropy", &
-                          length=restart_batch, group="cmfd", &
+        call read_data(cmfd % entropy, "cmfd_entropy", &
+                       length=restart_batch, group="cmfd", &
                           option="collective")
         call read_data(cmfd % balance, "cmfd_balance", &
              length=restart_batch, group="cmfd", option="collective")
diff --git a/src/utils/statepoint.py b/src/utils/statepoint.py
index 917ba19e18..ec43435b91 100644
--- a/src/utils/statepoint.py
+++ b/src/utils/statepoint.py
@@ -192,6 +192,23 @@ class StatePoint(object):
                 self.k_abs_tra = self._get_double(path='k_abs_tra')[0]
                 self.k_combined = self._get_double(2, path='k_combined')
 
+            # Read CMFD information
+            cmfd_present = self._get_int(path='cmfd_on')[0]
+            if cmfd_present == 1:
+                self.cmfd_indices = self._get_double(4, path='cmfd/indicies')
+                self.k_cmfd = self._get_double(self.current_batch,
+                                               path='cmfd/k_cmfd')
+                self.cmfd_src = self._get_double(np.product(self.cmfd_indices,
+                                                 path='cmfd/cmfd_src'))
+                self.cmfd_entropy = self._get_double(self.current_batch,
+                                                     path='cmfd/cmfd_entropy')
+                self.cmfd_balance = self._get_double(self.current_batch,
+                                                     path='cmfd/cmfd_balance')
+                self.cmfd_dominance = self._get_double(self.current_batch,
+                                                       path='cmfd/cmfd_dominance')
+                self.cmfd_srccmp = self._get_double(self.current_batch,
+                                                    path='cmfd/cmfd_srccmp')
+
         # Read number of meshes
         n_meshes = self._get_int(path='tallies/n_meshes')[0]
 

From 68a73a4a83f82d70876aa5987138d832790aeeb2 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 20 Aug 2013 09:12:46 -0400
Subject: [PATCH 43/69] added gmres solver for non-petsc compilation

---
 src/DEPENDENCIES                              |   61 +-
 src/OBJECTS                                   |    3 +-
 src/cmfd_jfnk_solver.F90                      |    4 +-
 src/cmfd_power_solver.F90                     |    9 +-
 src/gmres_solver.F90                          | 1231 +++++++++++++++++
 src/input_xml.F90                             |    8 +-
 src/matrix_header.F90                         |    4 +-
 ...{petsc_solver.F90 => solver_interface.F90} |  333 ++++-
 8 files changed, 1536 insertions(+), 117 deletions(-)
 create mode 100644 src/gmres_solver.F90
 rename src/{petsc_solver.F90 => solver_interface.F90} (51%)

diff --git a/src/DEPENDENCIES b/src/DEPENDENCIES
index b4c9c7980c..ab8c31b85f 100644
--- a/src/DEPENDENCIES
+++ b/src/DEPENDENCIES
@@ -6,13 +6,30 @@ energy_grid.o: global.o
 energy_grid.o: list_header.o
 energy_grid.o: output.o
 
-ace_header.o: constants.o
-ace_header.o: endf_header.o
-
 list_header.o: constants.o
 
-petsc_solver.o: matrix_header.o
-petsc_solver.o: vector_header.o
+particle_header.o: constants.o
+
+cmfd_slepc_solver.o: cmfd_loss_operator.o
+cmfd_slepc_solver.o: cmfd_prod_operator.o
+cmfd_slepc_solver.o: constants.o
+cmfd_slepc_solver.o: global.o
+
+global.o: ace_header.o
+global.o: bank_header.o
+global.o: cmfd_header.o
+global.o: constants.o
+global.o: dict_header.o
+global.o: geometry_header.o
+global.o: hdf5_interface.o
+global.o: material_header.o
+global.o: mesh_header.o
+global.o: particle_header.o
+global.o: plot_header.o
+global.o: set_header.o
+global.o: source_header.o
+global.o: tally_header.o
+global.o: timer_header.o
 
 cmfd_loss_operator.o: constants.o
 cmfd_loss_operator.o: global.o
@@ -29,8 +46,6 @@ particle_restart.o: physics.o
 particle_restart.o: random_lcg.o
 particle_restart.o: source.o
 
-doppler.o: constants.o
-
 tally_header.o: constants.o
 
 cmfd_power_solver.o: cmfd_loss_operator.o
@@ -38,7 +53,7 @@ cmfd_power_solver.o: cmfd_prod_operator.o
 cmfd_power_solver.o: constants.o
 cmfd_power_solver.o: global.o
 cmfd_power_solver.o: matrix_header.o
-cmfd_power_solver.o: petsc_solver.o
+cmfd_power_solver.o: solver_interface.o
 cmfd_power_solver.o: vector_header.o
 
 cmfd_data.o: cmfd_header.o
@@ -87,10 +102,8 @@ cmfd_prod_operator.o: constants.o
 cmfd_prod_operator.o: global.o
 cmfd_prod_operator.o: matrix_header.o
 
-cmfd_slepc_solver.o: cmfd_loss_operator.o
-cmfd_slepc_solver.o: cmfd_prod_operator.o
-cmfd_slepc_solver.o: constants.o
-cmfd_slepc_solver.o: global.o
+ace_header.o: constants.o
+ace_header.o: endf_header.o
 
 fission.o: ace_header.o
 fission.o: constants.o
@@ -140,7 +153,7 @@ cmfd_jfnk_solver.o: cmfd_prod_operator.o
 cmfd_jfnk_solver.o: constants.o
 cmfd_jfnk_solver.o: global.o
 cmfd_jfnk_solver.o: matrix_header.o
-cmfd_jfnk_solver.o: petsc_solver.o
+cmfd_jfnk_solver.o: solver_interface.o
 cmfd_jfnk_solver.o: vector_header.o
 
 main.o: constants.o
@@ -170,21 +183,7 @@ interpolation.o: string.o
 
 random_lcg.o: global.o
 
-global.o: ace_header.o
-global.o: bank_header.o
-global.o: cmfd_header.o
-global.o: constants.o
-global.o: dict_header.o
-global.o: geometry_header.o
-global.o: hdf5_interface.o
-global.o: material_header.o
-global.o: mesh_header.o
-global.o: particle_header.o
-global.o: plot_header.o
-global.o: set_header.o
-global.o: source_header.o
-global.o: tally_header.o
-global.o: timer_header.o
+doppler.o: constants.o
 
 string.o: constants.o
 string.o: error.o
@@ -212,7 +211,11 @@ tally.o: search.o
 tally.o: string.o
 tally.o: tally_header.o
 
-particle_header.o: constants.o
+solver_interface.o: error.o
+solver_interface.o: global.o
+solver_interface.o: gmres_solver.o
+solver_interface.o: matrix_header.o
+solver_interface.o: vector_header.o
 
 output_interface.o: constants.o
 output_interface.o: error.o
diff --git a/src/OBJECTS b/src/OBJECTS
index 4a2c48a717..97a3838a94 100644
--- a/src/OBJECTS
+++ b/src/OBJECTS
@@ -25,6 +25,7 @@ fixed_source.o \
 geometry.o \
 geometry_header.o \
 global.o \
+gmres_solver.o \
 hdf5_interface.o \
 hdf5_summary.o \
 initialize.o \
@@ -43,7 +44,6 @@ output_interface.o \
 particle_header.o \
 particle_restart.o \
 particle_restart_write.o \
-petsc_solver.o \
 physics.o \
 plot.o \
 plot_header.o \
@@ -51,6 +51,7 @@ ppmlib.o \
 random_lcg.o \
 search.o \
 set_header.o \
+solver_interface.o \
 source.o \
 source_header.o \
 state_point.o \
diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
index 04f28ac426..3208e62146 100644
--- a/src/cmfd_jfnk_solver.F90
+++ b/src/cmfd_jfnk_solver.F90
@@ -4,7 +4,7 @@ module cmfd_jfnk_solver
   use cmfd_power_solver,   only: cmfd_power_execute
   use cmfd_prod_operator,  only: init_prod_matrix, build_prod_matrix
   use matrix_header,       only: Matrix
-  use petsc_solver,        only: Petsc_jfnk, Jfnk_ctx
+  use solver_interface,    only: JFNKSolver, Jfnk_ctx
   use vector_header,       only: Vector
 
   implicit none
@@ -16,7 +16,7 @@ module cmfd_jfnk_solver
   type(Matrix)     :: jac_prec     ! Jacobian preconditioner object
   type(Matrix)     :: loss         ! CMFD loss matrix
   type(Matrix)     :: prod         ! CMFD production matrix
-  type(Petsc_jfnk) :: jfnk         ! JFNK solver object
+  type(JFNKSolver) :: jfnk         ! JFNK solver object
   type(Vector)     :: resvec       ! JFNK residual vector
   type(Vector)     :: xvec         ! JFNK solution vector
 
diff --git a/src/cmfd_power_solver.F90 b/src/cmfd_power_solver.F90
index 9c7eff0eda..d57f386ee5 100644
--- a/src/cmfd_power_solver.F90
+++ b/src/cmfd_power_solver.F90
@@ -5,7 +5,7 @@ module cmfd_power_solver
   use cmfd_loss_operator, only: init_loss_matrix, build_loss_matrix
   use cmfd_prod_operator, only: init_prod_matrix, build_prod_matrix
   use matrix_header,      only: Matrix
-  use petsc_solver,       only: Petsc_gmres
+  use solver_interface,   only: GMRESSolver
   use vector_header,      only: Vector
 
   implicit none
@@ -29,7 +29,7 @@ module cmfd_power_solver
   type(Vector) :: s_n             ! new source vector
   type(Vector) :: s_o             ! old flux vector
   type(Vector) :: serr_v          ! error in source
-  type(Petsc_gmres) :: gmres      ! gmres solver
+  type(GMRESSolver) :: gmres      ! gmres solver
 
 contains
 
@@ -98,6 +98,7 @@ contains
   subroutine init_data(adjoint)
 
     use constants, only: ONE, ZERO
+    use global,    only: cmfd_write_matrices
 
     logical :: adjoint
 
@@ -136,12 +137,16 @@ contains
     ! Setup petsc for everything
     call loss % assemble()
     call prod % assemble()
+#ifdef PETSC
     call loss % setup_petsc()
     call prod % setup_petsc()
     call phi_n % setup_petsc()
     call phi_o % setup_petsc()
     call s_o % setup_petsc()
     call s_n % setup_petsc()
+    if (cmfd_write_matrices) call loss % write_petsc_binary('loss.bin')
+    if (cmfd_write_matrices) call prod % write_petsc_binary('prod.bin')
+#endif
 
     ! Set norms to 0
     norm_n = ZERO
diff --git a/src/gmres_solver.F90 b/src/gmres_solver.F90
new file mode 100644
index 0000000000..9345f4f9a5
--- /dev/null
+++ b/src/gmres_solver.F90
@@ -0,0 +1,1231 @@
+module gmres_solver
+
+  implicit none
+
+contains
+
+subroutine pgmres ( n, im, rhs, sol, vv, eps, maxits, iout, &
+  aa, ja, ia, alu, jlu, ju, ierr )
+
+!*****************************************************************************80
+!
+!! PGMRES is an ILUT - Preconditioned GMRES solver.
+!                                                                      
+!  Discussion:
+!
+!    This is a simple version of the ILUT preconditioned GMRES algorithm. 
+!    The ILUT preconditioner uses a dual strategy for dropping elements   
+!    instead  of the usual level of-fill-in approach. See details in ILUT 
+!    subroutine documentation. PGMRES uses the L and U matrices generated 
+!    from the subroutine ILUT to precondition the GMRES algorithm.        
+!    The preconditioning is applied to the right. The stopping criterion  
+!    utilized is based simply on reducing the residual norm by epsilon.   
+!    This preconditioning is more reliable than ilu0 but requires more    
+!    storage. It seems to be much less prone to difficulties related to   
+!    strong nonsymmetries in the matrix. We recommend using a nonzero tol 
+!    (tol=.005 or .001 usually give good results) in ILUT. Use a large    
+!    lfil whenever possible (e.g. lfil = 5 to 10). The higher lfil the    
+!    more reliable the code is. Efficiency may also be much improved.     
+!    Note that lfil=n and tol=0.0 in ILUT  will yield the same factors as 
+!    Gaussian elimination without pivoting.                               
+!                                                                      
+!    ILU(0) and MILU(0) are also provided for comparison purposes         
+!    USAGE: first call ILUT or ILU0 or MILU0 to set up preconditioner and 
+!    then call pgmres.                                                    
+!
+!  Modified:
+!
+!    07 January 2004
+!
+!  Author:
+!
+!    Youcef Saad              
+!
+!  Parameters:
+!
+!    Input, integer ( kind = 4 ) N, the order of the matrix.
+!
+!    Input, integer ( kind = 4 ) IM, the size of the Krylov subspace.  IM 
+!    should not exceed 50 in this version.  This restriction can be reset by 
+!    changing the parameter command for KMAX below.
+!                                                
+!    Input/output, real RHS(N), on input, the right hand side vector.
+!    On output, the information in this vector has been destroyed.
+!
+! sol   == real vector of length n containing an initial guess to the  
+!          solution on input. approximate solution on output           
+!
+! eps   == tolerance for stopping criterion. process is stopped        
+!          as soon as ( ||.|| is the euclidean norm):                  
+!          || current residual||/||initial residual|| <= eps           
+!
+! maxits== maximum number of iterations allowed                        
+!
+! iout  == output unit number number for printing intermediate results 
+!          if (iout <= 0) nothing is printed out.                    
+!                                                                      
+!    Input, real AA(*), integer ( kind = 4 ) JA(*), IA(N+1), the matrix in CSR
+!    Compressed Sparse Row format.
+!
+!                                                                      
+! alu,jlu== A matrix stored in Modified Sparse Row format containing   
+!           the L and U factors, as computed by routine ilut.       
+!                                                                      
+! ju     == integer ( kind = 4 ) array of length n containing the pointers to       
+!           the beginning of each row of U in alu, jlu as computed     
+!           by routine ILUT.                                        
+!                                                                      
+! on return:                                                           
+!                                                          
+! sol   == contains an approximate solution (upon successful return).  
+! ierr  == integer ( kind = 4 ). Error message with the following meaning.          
+!          ierr = 0 --> successful return.                            
+!          ierr = 1 --> convergence not achieved in itmax iterations. 
+!          ierr =-1 --> the initial guess seems to be the exact        
+!                       solution (initial residual computed was zero) 
+!                                                                      
+! work arrays:                                                        
+!                                                       
+! vv    == work array of length  n x (im+1) (used to store the Arnoli  
+!          basis)                                                      
+!
+  integer ( kind = 4 ), parameter :: kmax = 50
+  integer ( kind = 4 ) n
+
+  real ( kind = 8 ) aa(*)
+  real ( kind = 8 ) alu(*)
+  real ( kind = 8 ) c(kmax)
+  real ( kind = 8 ) eps
+  real ( kind = 8 ) eps1
+  real ( kind = 8 ), parameter :: epsmac = 1.0D-16
+  real ( kind = 8 ) gam
+  real ( kind = 8 ) hh(kmax+1,kmax)
+  integer ( kind = 4 ) i
+  integer ( kind = 4 ) i1
+  integer ( kind = 4 ) ia(n+1)
+  integer ( kind = 4 ) ierr
+  integer ( kind = 4 ) ii
+  integer ( kind = 4 ) im
+  integer ( kind = 4 ) iout
+  integer ( kind = 4 ) its
+  integer ( kind = 4 ) j
+  integer ( kind = 4 ) ja(*)
+  integer ( kind = 4 ) jj
+  integer ( kind = 4 ) jlu(*)
+  integer ( kind = 4 ) ju(*)
+  integer ( kind = 4 ) k
+  integer ( kind = 4 ) k1
+  integer ( kind = 4 ) maxits
+  integer ( kind = 4 ) n1
+  real ( kind = 8 ) rhs(n)
+  real ( kind = 8 ) ro
+  real ( kind = 8 ) rs(kmax+1)
+  real ( kind = 8 ) s(kmax)
+  real ( kind = 8 ) sol(n)
+  real ( kind = 8 ) t
+  real ( kind = 8 ) vv(n,*)
+!
+!  Arnoldi size should not exceed KMAX=50 in this version.
+!  To reset modify parameter KMAX accordingly.
+!
+  n1 = n + 1
+  its = 0
+!
+!  Outer loop starts here.
+!  Compute initial residual vector.
+!
+  call ope ( n, sol, vv, aa, ja, ia )
+
+  vv(1:n,1) = rhs(1:n) - vv(1:n,1)
+
+  do
+
+    ro = sqrt ( ddot ( n, vv, 1, vv, 1 ) )
+
+    if ( 0 < iout .and. its == 0 ) then
+      write(iout, 199) its, ro
+    end if
+
+    if ( ro == 0.0D+00 ) then
+      ierr = -1
+      exit
+    end if
+
+    t = 1.0D+00 / ro
+    vv(1:n,1) = vv(1:n,1) * t
+
+    if ( its == 0 ) then
+      eps1 = eps * ro
+    end if
+!
+!  Initialize first term of RHS of Hessenberg system.
+!
+     rs(1) = ro
+     i = 0
+
+ 4   continue
+
+     i = i + 1
+     its = its + 1
+     i1 = i + 1
+     call lusol0 ( n, vv(1,i), rhs, alu, jlu, ju )
+     call ope ( n, rhs, vv(1,i1), aa, ja, ia )
+!
+!  Modified Gram - Schmidt.
+!
+     do j = 1, i
+       t = ddot ( n, vv(1,j), 1, vv(1,i1), 1 )
+       hh(j,i) = t
+       call daxpy ( n, -t, vv(1,j), 1, vv(1,i1), 1 )
+     end do
+
+     t = sqrt ( ddot ( n, vv(1,i1), 1, vv(1,i1), 1 ) )
+     hh(i1,i) = t
+
+     if ( t /= 0.0D+00 ) then
+       t = 1.0D+00 / t
+       vv(1:n,i1) = vv(1:n,i1) * t
+     end if
+!
+!  Update factorization of HH.
+!
+    if ( i == 1 ) then
+      go to 121
+    end if
+!
+!  Perform previous transformations on I-th column of H.
+!
+    do k = 2, i
+       k1 = k-1
+       t = hh(k1,i)
+       hh(k1,i) = c(k1) * t + s(k1) * hh(k,i)
+       hh(k,i) = -s(k1) * t + c(k1) * hh(k,i)
+    end do
+
+121 continue
+
+    gam = sqrt ( hh(i,i)**2 + hh(i1,i)**2 )
+!
+!  If GAMMA is zero then any small value will do.
+!  It will affect only residual estimate.
+!
+    if ( gam == 0.0D+00 ) then
+      gam = epsmac
+    end if
+!
+!  Get the next plane rotation.
+!
+    c(i) = hh(i,i) / gam
+    s(i) = hh(i1,i) / gam
+    rs(i1) = -s(i) * rs(i)
+    rs(i) = c(i) * rs(i)
+!
+!  Determine residual norm and test for convergence.
+!
+    hh(i,i) = c(i) * hh(i,i) + s(i) * hh(i1,i)
+    ro = abs ( rs(i1) )
+131 format(1h ,2e14.4)
+
+    if ( 0 < iout ) then
+      write(iout, 199) its, ro
+    end if
+
+    if ( i < im .and. eps1 < ro ) then
+      go to 4
+    end if
+!
+!  Now compute solution.  First solve upper triangular system.
+!
+    rs(i) = rs(i) / hh(i,i)
+
+    do ii = 2, i
+      k = i - ii + 1
+      k1 = k + 1
+      t = rs(k)
+      do j = k1, i
+        t = t - hh(k,j) * rs(j)
+      end do
+      rs(k) = t / hh(k,k)
+    end do
+!
+!  Form linear combination of V(*,i)'s to get solution.
+!
+    t = rs(1)
+    rhs(1:n) = vv(1:n,1) * t
+
+    do j = 2, i
+      t = rs(j)
+      rhs(1:n) = rhs(1:n) + t * vv(1:n,j)
+    end do
+!
+!  Call preconditioner.
+!
+    call lusol0 ( n, rhs, rhs, alu, jlu, ju )
+
+    sol(1:n) = sol(1:n) + rhs(1:n)
+!
+!  Restart outer loop when necessary.
+!
+    if ( ro <= eps1 ) then
+      ierr = 0
+      exit
+    end if
+
+    if ( maxits < its ) then
+      ierr = 1
+      exit
+    end if
+!
+!  Else compute residual vector and continue.
+!
+    do j = 1, i
+      jj = i1 - j + 1
+      rs(jj-1) = -s(jj-1) * rs(jj)
+      rs(jj) = c(jj-1) * rs(jj)
+    end do
+
+    do j = 1, i1
+      t = rs(j)
+      if ( j == 1 ) then
+        t = t - 1.0D+00
+      end if
+      call daxpy ( n, t, vv(1,j), 1,  vv, 1 )
+    end do
+
+199 format(' its =', i4, ' res. norm =', G14.6)
+
+  end do
+
+  return
+end
+
+subroutine ilut ( n, a, ja, ia, lfil, tol, alu, jlu, ju, iwk, wu, wl, jr, &
+  jwl, jwu, ierr )
+
+!*****************************************************************************80
+!
+!! ILUT is an ILUT preconditioner.
+!
+!  Discussion:
+!
+!    This routine carries ouot incomplete LU factorization with dual
+!    truncation mechanism.  Sorting is done for both L and U. 
+!
+!    The dual drop-off strategy works as follows:
+!
+!    1) Theresholding in L and U as set by TOL.  Any element whose size
+!       is less than some tolerance (relative to the norm of current
+!       row in u) is dropped.
+!         
+!    2) Keeping only the largest lenl0+lfil elements in L and the
+!       largest lenu0+lfil elements in U, where lenl0=initial number 
+!       of nonzero elements in a given row of lower part of A 
+!       and lenlu0 is similarly defined.
+!  
+!    Flexibility: one can use tol=0 to get a strategy based on keeping the
+!    largest elements in each row of L and U. Taking tol /= 0 but lfil=n
+!    will give the usual threshold strategy (however, fill-in is then     
+!    unpredictible).                                                      
+!
+!    A must have all nonzero diagonal elements.
+!
+!  Modified:
+!
+!    07 January 2004
+!
+!  Author:
+!
+!    Youcef Saad
+!                                                                      
+!  Parameters:
+!
+!  Parameters:
+!
+!    Input, integer ( kind = 4 ) N, the order of the matrix.
+!
+!    Input, real A(*), integer ( kind = 4 ) JA(*), IA(N+1), the matrix in CSR
+!    Compressed Sparse Row format.
+!
+! lfil    = integer ( kind = 4 ). The fill-in parameter. Each row of L and
+!           each row of U will have a maximum of lfil elements
+!           in addition to the original number of nonzero elements.
+!           Thus storage can be determined beforehand.
+!           lfil must be >= 0.
+!
+! iwk     = integer ( kind = 4 ). The minimum length of arrays alu and jlu
+!
+! On return:
+!
+! alu,jlu = matrix stored in Modified Sparse Row (MSR) format containing
+!           the L and U factors together. The diagonal (stored in
+!           alu(1:n) ) is inverted. Each i-th row of the alu,jlu matrix
+!           contains the i-th row of L (excluding the diagonal entry=1)
+!           followed by the i-th row of U.
+!
+! ju      = integer ( kind = 4 ) array of length n containing the pointers to
+!           the beginning of each row of U in the matrix alu,jlu.
+!
+! ierr    = integer ( kind = 4 ). Error message with the following meaning.
+!           ierr  = 0    --> successful return.
+!           ierr > 0  --> zero pivot encountered at step number ierr.
+!           ierr  = -1   --> Error. input matrix may be wrong.
+!                            (The elimination process has generated a
+!                            row in L or U whose length is >  n.)
+!           ierr  = -2   --> The matrix L overflows the array al.
+!           ierr  = -3   --> The matrix U overflows the array alu.
+!           ierr  = -4   --> Illegal value for lfil.
+!           ierr  = -5   --> zero pivot encountered.
+!
+! work arrays:
+!
+! jr,jwu,jwl, integer ( kind = 4 ) work arrays of length n.
+! wu, wl, real work arrays of length n+1, and n resp.
+!
+  integer ( kind = 4 ) n
+
+  real ( kind = 8 ) a(*)
+  real ( kind = 8 ) alu(*)
+  real ( kind = 8 ) fact
+  integer ( kind = 4 ) ia(n+1)
+  integer ( kind = 4 ) idiag
+  integer ( kind = 4 ) ierr
+  integer ( kind = 4 ) ii
+  integer ( kind = 4 ) iwk
+  integer ( kind = 4 ) j 
+  integer ( kind = 4 ) j1
+  integer ( kind = 4 ) j2
+  integer ( kind = 4 ) ja(*)
+  integer ( kind = 4 ) jj
+  integer ( kind = 4 ) jlu(*)
+  integer ( kind = 4 ) jpos
+  integer ( kind = 4 ) jr(*)
+  integer ( kind = 4 ) jrow
+  integer ( kind = 4 ) ju(*)
+  integer ( kind = 4 ) ju0
+  integer ( kind = 4 ) jwl(n)
+  integer ( kind = 4 ) jwu(n)
+  integer ( kind = 4 ) k
+  integer ( kind = 4 ) len
+  integer ( kind = 4 ) lenl
+  integer ( kind = 4 ) lenl0
+  integer ( kind = 4 ) lenu
+  integer ( kind = 4 ) lenu0
+  integer ( kind = 4 ) lfil
+  integer ( kind = 4 ) nl
+  real ( kind = 8 ) s
+  real ( kind = 8 ) t
+  real ( kind = 8 ) tnorm
+  real ( kind = 8 ) tol
+  real ( kind = 8 ) wl(n)
+  real ( kind = 8 ) wu(n+1)
+
+  if ( lfil < 0 ) then
+    ierr = -4
+    return
+  end if
+!
+!  Initialize JU0 (points to next element to be added to ALU, JLU)
+!  and pointer.
+!
+  ju0 = n + 2
+  jlu(1) = ju0
+!
+!  integer ( kind = 4 ) double pointer array.
+!
+  jr(1:n) = 0
+!
+!  The main loop.
+!
+  do ii = 1, n
+
+    j1 = ia(ii)
+    j2 = ia(ii+1) - 1
+    lenu = 0
+    lenl = 0
+
+    tnorm = 0.0D+00
+    do k = j1, j2
+      tnorm = tnorm + abs ( a(k) )
+    end do
+    tnorm = tnorm / real ( j2-j1+1, kind = 8 )
+!
+!  Unpack L-part and U-part of row of A in arrays WL, WU.
+!
+    do j = j1, j2
+
+      k = ja(j)
+      t = a(j)
+
+      if ( tol * tnorm <= abs ( t ) ) then
+
+        if ( k < ii ) then
+          lenl = lenl + 1
+          jwl(lenl) = k
+          wl(lenl) = t
+          jr(k) = lenl
+        else
+          lenu = lenu+1
+          jwu(lenu) = k
+          wu(lenu) = t
+          jr(k) = lenu
+        end if
+
+      end if
+
+    end do
+
+    lenl0 = lenl
+    lenu0 = lenu
+    jj = 0
+    nl = 0
+!
+!  Eliminate previous rows.
+!
+150 continue
+
+    jj = jj + 1
+
+    if ( lenl < jj ) then
+      go to 160
+    end if
+!
+!  In order to do the elimination in the correct order we need to
+!  exchange the current row number with the one that has
+!  smallest column number, among JJ, JJ+1, ..., LENL.
+!
+    jrow = jwl(jj)
+    k = jj
+!
+!  Determine the smallest column index.
+!
+    do j = jj+1, lenl
+       if ( jwl(j) < jrow ) then
+          jrow = jwl(j)
+          k = j
+       end if
+    end do
+!
+!  Exchange in JWL.
+!
+    j = jwl(jj)
+    jwl(jj) = jrow
+    jwl(k) = j
+!
+!  Exchange in JR.
+!
+    jr(jrow) = jj
+    jr(j) = k
+!
+!  Exchange in WL.
+!
+    s = wl(k)
+    wl(k) = wl(jj)
+    wl(jj) = s
+
+    if ( ii <= jrow ) then
+      go to 160
+    end if
+!
+!  Get the multiplier for row to be eliminated: JROW.
+!
+    fact = wl(jj) * alu(jrow)
+    jr(jrow) = 0
+
+    if ( abs ( fact ) * wu(n+2-jrow) <= tol * tnorm ) then
+      go to 150
+    end if
+!
+!  Combine current row and row JROW.
+!
+    do k = ju(jrow), jlu(jrow+1)-1
+       s = fact * alu(k)
+       j = jlu(k)
+       jpos = jr(j)
+!
+!  If fill-in element and small disregard.
+!
+       if ( abs ( s ) < tol * tnorm .and. jpos == 0 ) then
+         cycle
+       end if
+
+       if ( ii <= j ) then
+!
+!  Dealing with upper part.
+!
+          if ( jpos == 0 ) then
+!
+!  This is a fill-in element.
+!
+             lenu = lenu + 1
+
+             if ( n < lenu ) then
+               go to 995
+             end if
+
+             jwu(lenu) = j
+             jr(j) = lenu
+             wu(lenu) = - s
+          else
+!
+!  No fill-in element.
+!
+             wu(jpos) = wu(jpos) - s
+          end if
+       else
+!
+!  Dealing with lower part.
+!
+          if ( jpos == 0 ) then
+!
+!  This is a fill-in element.
+!
+             lenl = lenl + 1
+
+             if ( n < lenl ) then
+               go to 995
+             end if
+
+             jwl(lenl) = j
+             jr(j) = lenl
+             wl(lenl) = -s
+          else
+!
+!  No fill-in element.
+!
+             wl(jpos) = wl(jpos) - s
+          end if
+       end if
+
+  end do
+
+    nl = nl + 1
+    wl(nl) = fact
+    jwl(nl) = jrow
+  go to 150
+!
+!  Update the L matrix.
+!
+ 160 continue
+
+    len = min ( nl, lenl0 + lfil )
+
+    call bsort2 ( wl, jwl, nl, len )
+
+    do k = 1, len
+
+       if ( iwk < ju0 ) then
+         ierr = -2
+         return
+       end if
+
+       alu(ju0) =  wl(k)
+       jlu(ju0) =  jwl(k)
+       ju0 = ju0 + 1
+
+    end do
+!
+!  Save pointer to beginning of row II of U.
+!
+    ju(ii) = ju0
+!
+!  Reset double pointer JR to zero (L-part - except first
+!  JJ-1 elements which have already been reset).
+!
+  do k = jj, lenl
+    jr(jwl(k)) = 0
+  end do
+!
+!  Be sure that the diagonal element is first in W and JW.
+!
+    idiag = jr(ii)
+
+    if ( idiag == 0 ) then
+      go to 900
+    end if
+
+    if ( idiag /= 1 ) then
+
+       s = wu(1)
+       wu(j) = wu(idiag)
+       wu(idiag) = s
+
+       j = jwu(1)
+       jwu(1) = jwu(idiag)
+       jwu(idiag) = j
+
+    end if
+
+    len = min ( lenu, lenu0 + lfil )
+
+    call bsort2 ( wu(2), jwu(2), lenu-1, len )
+!
+! Update the U-matrix.
+!
+    t = 0.0D+00
+
+    do k = 2, len
+
+       if ( iwk < ju0 ) then
+         ierr = -3
+         return
+       end if
+
+       jlu(ju0) = jwu(k)
+       alu(ju0) = wu(k)
+       t = t + abs ( wu(k) )
+       ju0 = ju0 + 1
+
+    end do
+!
+!  Save norm in WU (backwards). Norm is in fact average absolute value.
+!
+    wu(n+2-ii) = t / real ( len + 1, kind = 8 )
+!
+!  Store inverse of diagonal element of U.
+!
+    if ( wu(1) == 0.0D+00 ) then
+      ierr = -5
+      return
+    end if
+
+    alu(ii) = 1.0D+00 / wu(1)
+!
+!  Update pointer to beginning of next row of U.
+!
+  jlu(ii+1) = ju0
+!
+!  Reset double pointer JR to zero (U-part).
+!
+  do k = 1, lenu
+    jr(jwu(k)) = 0
+  end do
+
+  end do
+
+  ierr = 0
+
+  return
+!
+!  Zero pivot :
+!
+ 900    ierr = ii
+    return
+!
+!  Incomprehensible error. Matrix must be wrong.
+!
+ 995    ierr = -1
+    return
+end
+
+subroutine amux ( n, x, y, a, ja, ia )
+
+!*****************************************************************************80
+!
+!! AMUX multiplies a CSR matrix A times a vector.
+!
+!  Discussion:
+!
+!    This routine multiplies a matrix by a vector using the dot product form.
+!    Matrix A is stored in compressed sparse row storage.
+!
+!  Modified:
+!
+!    07 January 2004
+!
+!  Author:
+!
+!    Youcef Saad
+!
+!  Parameters:
+!
+!    Input, integer ( kind = 4 ) N, the row dimension of the matrix.
+!
+!    Input, real X(*), and array of length equal to the column dimension 
+!    of A.
+!
+!    Input, real A(*), integer ( kind = 4 ) JA(*), IA(NROW+1), the matrix in CSR
+!    Compressed Sparse Row format.
+!
+!    Output, real Y(N), the product A * X.
+!
+  integer ( kind = 4 ) n
+
+  real ( kind = 8 ) a(*)
+  integer ( kind = 4 ) i
+  integer ( kind = 4 ) ia(*)
+  integer ( kind = 4 ) ja(*)
+  integer ( kind = 4 ) k
+  real ( kind = 8 ) t
+  real ( kind = 8 ) x(*)
+  real ( kind = 8 ) y(n)
+
+  do i = 1, n
+!
+!  Compute the inner product of row I with vector X.
+!
+    t = 0.0D+00
+    do k = ia(i), ia(i+1)-1
+      t = t + a(k) * x(ja(k))
+    end do
+
+    y(i) = t
+
+  end do
+
+  return
+end
+subroutine amuxd ( n, x, y, diag, ndiag, idiag, ioff )
+
+!*****************************************************************************80
+!
+!! AMUXD multiplies a DIA matrix times a vector.
+!
+!  Discussion:
+!
+!    This routine multiplies a matrix by a vector when the original matrix 
+!    is stored in the DIA diagonal storage format.
+!
+!  Modified:
+!
+!    07 January 2004
+!
+!  Author:
+!
+!    Youcef Saad
+!
+!  Parameters:
+!
+!    Input, integer ( kind = 4 ) N, the row dimension of the matrix.
+!
+!    Input, real X(*), array of length equal to the column dimension of
+!    the A matrix.
+!
+!    Output, real Y(N), the product A * X.
+!
+!    Input, real DIAG(NDIAG,IDIAG), the diagonals.
+!
+!    Input, integer ( kind = 4 ) NDIAG, the first dimension of array adiag as 
+!    declared in the calling program.
+!
+!    Input, integer ( kind = 4 ) IDIAG, the number of diagonals in the matrix.
+!
+!    Input, integer ( kind = 4 ) IOFF(IDIAG), the offsets of the diagonals of 
+!    the matrix: diag(i,k) contains the element a(i,i+ioff(k)) of the matrix.
+!
+  integer ( kind = 4 ) idiag
+  integer ( kind = 4 ) n
+  integer ( kind = 4 ) ndiag
+
+  real ( kind = 8 ) diag(ndiag,idiag)
+  integer ( kind = 4 ) i1
+  integer ( kind = 4 ) i2
+  integer ( kind = 4 ) io
+  integer ( kind = 4 ) ioff(idiag)
+  integer ( kind = 4 ) j
+  integer ( kind = 4 ) k
+  real ( kind = 8 ) x(n)
+  real ( kind = 8 ) y(n)
+
+  y(1:n) = 0.0D+00
+
+  do j = 1, idiag
+    io = ioff(j)
+    i1 = max ( 1, 1 - io )
+    i2 = min ( n, n - io )
+    do k = i1, i2
+      y(k) = y(k) + diag(k,j) * x(k+io)
+    end do
+  end do
+
+  return
+end
+
+subroutine daxpy ( n, da, dx, incx, dy, incy )
+
+!*****************************************************************************80
+!
+!! DAXPY computes constant times a vector plus a vector.
+!
+!  Discussion:
+!
+!    Uses unrolled loops for increments equal to one.
+!
+!  Author:
+!
+!    Jack Dongarra
+!
+!  Reference:
+!
+!    Dongarra, Moler, Bunch, Stewart,
+!    LINPACK User's Guide,
+!    SIAM, 1979.
+!
+!    Lawson, Hanson, Kincaid, Krogh,
+!    Basic Linear Algebra Subprograms for Fortran Usage,
+!    Algorithm 539,
+!    ACM Transactions on Mathematical Software,
+!    Volume 5, Number 3, September 1979, pages 308-323.
+!
+!  Parameters:
+!
+!    Input, integer ( kind = 4 ) N, the number of elements in DX and DY.
+!
+!    Input, real ( kind = 8 ) DA, the multiplier of DX.
+!
+!    Input, real ( kind = 8 ) DX(*), the first vector.
+!
+!    Input, integer ( kind = 4 ) INCX, the increment between successive entries of DX.
+!
+!    Input/output, real ( kind = 8 ) DY(*), the second vector.
+!    On output, DY(*) has been replaced by DY(*) + DA * DX(*).
+!
+!    Input, integer ( kind = 4 ) INCY, the increment between successive entries of DY.
+!
+  real ( kind = 8 ) da
+  real ( kind = 8 ) dx(*)
+  real ( kind = 8 ) dy(*)
+  integer ( kind = 4 ) i
+  integer ( kind = 4 ) incx
+  integer ( kind = 4 ) incy
+  integer ( kind = 4 ) ix
+  integer ( kind = 4 ) iy
+  integer ( kind = 4 ) m
+  integer ( kind = 4 ) n
+
+  if ( n <= 0 ) then
+    return
+  end if
+
+  if ( da  == 0.0D+00 ) then
+    return
+  end if
+!
+!  Code for unequal increments or equal increments
+!  not equal to 1.
+!
+  if ( incx /= 1 .or. incy /= 1 ) then
+
+    if ( 0 <= incx ) then
+      ix = 1
+    else
+      ix = ( - n + 1 ) * incx + 1
+    end if
+
+    if ( 0 <= incy ) then
+      iy = 1
+    else
+      iy = ( - n + 1 ) * incy + 1
+    end if
+
+    do i = 1, n
+      dy(iy) = dy(iy) + da * dx(ix)
+      ix = ix + incx
+      iy = iy + incy
+    end do
+!
+!  Code for both increments equal to 1.
+!
+  else
+
+    m = mod ( n, 4 )
+
+    do i = 1, m
+      dy(i) = dy(i) + da * dx(i)
+    end do
+
+    do i = m+1, n, 4
+      dy(i  ) = dy(i  ) + da * dx(i  )
+      dy(i+1) = dy(i+1) + da * dx(i+1)
+      dy(i+2) = dy(i+2) + da * dx(i+2)
+      dy(i+3) = dy(i+3) + da * dx(i+3)
+    end do
+
+  end if
+
+  return
+end
+
+subroutine bsort2 ( w, ind, n, ncut )
+
+!*****************************************************************************80
+!
+!! BSORT2 returns the NCUT largest elements of an array, using bubble sort.
+!
+!  Discussion:
+!
+!    This routine carries out a simple bubble sort for getting the NCUT largest
+!    elements in modulus, in array W.  IND is sorted accordingly.
+!    (Ought to be replaced by a more efficient sort especially
+!    if NCUT is not that small).
+!
+!  Modified:
+!
+!    07 January 2004
+!
+!  Author:
+!
+!    Youcef Saad
+!
+!  Parameters:
+!
+  integer ( kind = 4 ) n
+
+  integer ( kind = 4 ) i
+  integer ( kind = 4 ) ind(*)
+  integer ( kind = 4 ) iswp
+  integer ( kind = 4 ) j
+  integer ( kind = 4 ) ncut
+  logical test
+  real ( kind = 8 ) w(n)
+  real ( kind = 8 ) wswp
+
+  i = 1
+
+  do
+
+    test = .false.
+
+    do j = n-1, i, -1
+
+      if ( abs ( w(j) ) < abs ( w(j+1) ) ) then
+!
+!  Swap.
+!
+        wswp = w(j)
+        w(j) = w(j+1)
+        w(j+1) = wswp
+!
+!  Reorder the original ind array accordingly.
+!
+        iswp = ind(j)
+        ind(j) = ind(j+1)
+        ind(j+1) = iswp
+!
+!  Set indicator that sequence is still unsorted.
+!
+        test = .true.
+
+      end if
+
+    end do
+
+    i = i + 1
+
+    if ( .not. test .or. ncut < i ) then
+      exit
+    end if
+
+  end do
+
+  return
+end
+
+subroutine ope ( n, x, y, a, ja, ia )
+
+!*****************************************************************************80
+!
+!! OPE sparse matrix * vector multiplication
+!
+!  Modified:
+!
+!    07 January 2004
+!
+!  Author:
+!
+!    Youcef Saad
+!
+!  Parameters:
+!
+!    Input, integer ( kind = 4 ) N, the order of the matrix.
+!
+!    Input, real X(N), the vector to be multiplied.
+!
+!    Output, real Y(N), the product A * X.
+!
+!    Input, real A(*), integer ( kind = 4 ) JA(*), IA(N+1), the matrix in CSR
+!    Compressed Sparse Row format.
+!
+  integer ( kind = 4 ) n
+
+  real ( kind = 8 ) a(*)
+  integer ( kind = 4 ) i
+  integer ( kind = 4 ) ia(n+1)
+  integer ( kind = 4 ) ja(*)
+  integer ( kind = 4 ) k
+  integer ( kind = 4 ) k1
+  integer ( kind = 4 ) k2
+  real ( kind = 8 ) x(n)
+  real ( kind = 8 ) y(n)
+
+  do i = 1, n
+    k1 = ia(i)
+    k2 = ia(i+1) -1
+    y(i) = 0.0D+00
+    do k = k1, k2
+      y(i) = y(i) + a(k) * x(ja(k))
+    end do
+  end do
+
+  return
+end
+
+subroutine lusol0 ( n, y, x, alu, jlu, ju )
+
+!*****************************************************************************80
+!
+!! LUSOL0 performs a forward followed by a backward solve
+! for LU matrix as produced by  ILUT
+!
+!  Modified:
+!
+!    07 January 2004
+!
+!  Author:
+!
+!    Youcef Saad
+!
+!  Parameters:
+!
+!    Input, integer ( kind = 4 ) N, the order of the matrix.
+!
+!    Input, real Y(N), the right hand side of the linear system.
+!
+!    Output, real X(N), the solution.
+!
+!    ALU, JLU, JU, ...
+!
+  integer ( kind = 4 ) n
+
+  real ( kind = 8 ) alu(*)
+  integer ( kind = 4 ) i
+  integer ( kind = 4 ) jlu(*)
+  integer ( kind = 4 ) ju(*)
+  integer ( kind = 4 ) k
+  real ( kind = 8 ) x(n)
+  real ( kind = 8 ) y(n)
+!
+!  Forward solve
+!
+  do i = 1, n
+    x(i) = y(i)
+    do k = jlu(i), ju(i)-1
+      x(i) = x(i) - alu(k) * x(jlu(k))
+    end do
+  end do
+!
+!  Backward solve.
+!
+  do i = n, 1, -1
+    do k = ju(i), jlu(i+1)-1
+      x(i) = x(i) - alu(k) * x(jlu(k))
+    end do
+    x(i) = alu(i) * x(i)
+  end do
+
+  return
+end
+
+function ddot ( n, dx, incx, dy, incy )
+
+!*****************************************************************************80
+!
+!! DDOT forms the dot product of two vectors.
+!
+!  Discussion:
+!
+!    This routine uses unrolled loops for increments equal to one.
+!
+!  Author:
+!
+!    Jack Dongarra
+!
+!  Reference:
+!
+!    Dongarra, Moler, Bunch, Stewart,
+!    LINPACK User's Guide,
+!    SIAM, 1979.
+!
+!  Parameters:
+!
+!    Input, integer ( kind = 4 ) N, the number of entries in the vectors.
+!
+!    Input, real ( kind = 8 ) DX(*), the first vector.
+!
+!    Input, integer ( kind = 4 ) INCX, the increment between successive entries in X.
+!
+!    Input, real ( kind = 8 ) DY(*), the second vector.
+!
+!    Input, integer ( kind = 4 ) INCY, the increment between successive entries in Y.
+!
+!    Output, real DDOT, the sum of the product of the corresponding
+!    entries of X and Y.
+!
+  real ( kind = 8 ) ddot
+  real ( kind = 8 ) dtemp
+  real ( kind = 8 ) dx(*)
+  real ( kind = 8 ) dy(*)
+  integer ( kind = 4 ) i
+  integer ( kind = 4 ) incx
+  integer ( kind = 4 ) incy
+  integer ( kind = 4 ) ix
+  integer ( kind = 4 ) iy
+  integer ( kind = 4 ) m
+  integer ( kind = 4 ) n
+
+  ddot = 0.0D+00
+  dtemp = 0.0D+00
+
+  if ( n <= 0 ) then
+    return
+  end if
+!
+!  Code for unequal increments or equal increments
+!  not equal to 1.
+!
+  if ( incx /= 1 .or. incy /= 1 ) then
+
+    if ( 0 <= incx ) then
+      ix = 1
+    else
+      ix = ( - n + 1 ) * incx + 1
+    end if
+
+    if ( 0 <= incy ) then
+      iy = 1
+    else
+      iy = ( - n + 1 ) * incy + 1
+    end if
+
+    do i = 1, n
+      dtemp = dtemp + dx(ix) * dy(iy)
+      ix = ix + incx
+      iy = iy + incy
+    end do
+!
+!  Code for both increments equal to 1.
+!
+  else
+
+    m = mod ( n, 5 )
+
+    do i = 1, m
+      dtemp = dtemp + dx(i) * dy(i)
+    end do
+
+    do i = m+1, n, 5
+
+      dtemp = dtemp + dx(i  ) * dy(i  ) &
+                    + dx(i+1) * dy(i+1) &
+                    + dx(i+2) * dy(i+2) &
+                    + dx(i+3) * dy(i+3) &
+                    + dx(i+4) * dy(i+4)
+    end do
+
+  end if
+
+  ddot = dtemp
+
+  return
+end
+
+end module gmres_solver
diff --git a/src/input_xml.F90 b/src/input_xml.F90
index 617788e472..a0731ee70b 100644
--- a/src/input_xml.F90
+++ b/src/input_xml.F90
@@ -557,16 +557,10 @@ contains
       end do
     end if
 
-    ! check for cmfd run
+    ! Check for cmfd run
     call lower_case(run_cmfd_)
     if (run_cmfd_ == 'true' .or. run_cmfd_ == '1') then
       cmfd_run = .true.
-#ifndef PETSC
-      if (master) then
-        message = 'CMFD is not available, compile OpenMC with PETSc'
-        call fatal_error()
-      end if
-#endif
     end if
 
   end subroutine read_settings_xml
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index fe5c2c5bcf..c034c8d2e7 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -12,8 +12,8 @@ module matrix_header
     integer :: nnz      ! number of nonzeros in matrix
     integer :: n_count  ! counter for length of matrix
     integer :: nz_count ! counter for number of non zeros
-    integer, private, allocatable :: row(:) ! csr row vector
-    integer, private, allocatable :: col(:) ! column vector
+    integer, allocatable :: row(:) ! csr row vector
+    integer, allocatable :: col(:) ! column vector
     real(8), allocatable :: val(:) ! matrix value vector
 #  ifdef PETSC
     Mat :: petsc_mat
diff --git a/src/petsc_solver.F90 b/src/solver_interface.F90
similarity index 51%
rename from src/petsc_solver.F90
rename to src/solver_interface.F90
index 87c288b936..47bb0c14ab 100644
--- a/src/petsc_solver.F90
+++ b/src/solver_interface.F90
@@ -1,8 +1,14 @@
-module petsc_solver
+module solver_interface
 
+  use error,          only: fatal_error
+  use global,         only: message
   use matrix_header,  only: Matrix
   use vector_header,  only: Vector
 
+#ifndef PETSC
+  use gmres_solver,   only: ilut, pgmres
+#endif
+
   implicit none
   private
 
@@ -10,18 +16,44 @@ module petsc_solver
 #  include <finclude/petsc.h90>
 #endif
 
-  ! PETSc GMRES solver type 
-  type, public :: Petsc_gmres 
+  ! GMRES solver type 
+  type, public :: GMRESSolver 
 #ifdef PETSC
     KSP :: ksp
     PC  :: pc
+#else
+    integer :: im      ! iterations before restart
+    integer :: iout    ! unit number for printing gmres iters
+    integer :: maxits  ! maximum iterations
+    integer :: lfil    ! fill in parameter
+    integer :: iwk     ! size of MSR storage
+    integer, allocatable :: ju(:) ! row pointers for ILUT result
+    integer, allocatable :: jlu(:) ! MSR storage of j
+    integer, allocatable :: jr(:) ! temp work array
+    integer, allocatable :: jwu(:) ! temp work array
+    integer, allocatable :: jwl(:) ! temp work array
+    real(8) :: tol     ! gmres tolerance
+    real(8) :: droptol ! ILU drop tolerance
+    real(8), allocatable :: alu(:) ! MSR storage of ILU A
+    real(8), allocatable :: vv(:) ! GMRES work array
+    real(8), allocatable :: wu(:) ! temp work array
+    real(8), allocatable :: wl(:) ! temp work array
+    real(8), allocatable :: rhs(:) ! copy of rhs array
+    type(Matrix), pointer :: A ! coefficient matrix
 #endif
    contains
+#ifdef PETSC
+     procedure :: create       => petsc_gmres_create
+     procedure :: set_oper     => petsc_gmres_set_oper
+     procedure :: destroy      => petsc_gmres_destroy
+     procedure :: solve        => petsc_gmres_solve
+#else
      procedure :: create       => gmres_create
      procedure :: set_oper     => gmres_set_oper
      procedure :: destroy      => gmres_destroy
      procedure :: solve        => gmres_solve
-  end type Petsc_gmres
+#endif
+  end type GMRESSolver
 
   ! Derived type to contain list of data needed to be passed to procedures
   type, public :: Jfnk_ctx
@@ -29,8 +61,8 @@ module petsc_solver
     procedure (jac_interface), pointer, nopass :: jac_proc_ptr
   end type Jfnk_ctx
 
-  ! Petsc SNES JFNK solver type 
-  type, public :: Petsc_jfnk
+  ! JFNK solver type 
+  type, public :: JFNKSolver 
 #ifdef PETSC
     KSP  :: ksp
     PC   :: pc
@@ -39,13 +71,18 @@ module petsc_solver
     SNESLineSearch :: ls
 #endif
    contains
+#ifdef PETSC
+     procedure :: create        => petsc_jfnk_create
+     procedure :: destroy       => petsc_jfnk_destroy
+     procedure :: set_functions => petsc_jfnk_set_functions
+     procedure :: solve         => petsc_jfnk_solve
+#else
      procedure :: create        => jfnk_create
      procedure :: destroy       => jfnk_destroy
      procedure :: set_functions => jfnk_set_functions
      procedure :: solve         => jfnk_solve
-  end type Petsc_jfnk
-
-  integer :: petsc_err
+#endif
+  end type JFNKSolver
 
   ! Abstract interface stating how jacobian and residual routines look
   abstract interface
@@ -61,30 +98,28 @@ module petsc_solver
     end subroutine jac_interface
   end interface
 
+#ifdef PETSC
+  integer :: petsc_err ! petsc error code
+#endif
+
 contains
 
+#ifndef PETSC
 !===============================================================================
 ! GMRES_CREATE sets up a PETSc GMRES solver
 !===============================================================================
 
   subroutine gmres_create(self)
 
-    class(Petsc_gmres) :: self
+    class(GMRESSolver) :: self
 
-    integer :: ilu_levels = 5
-    real(8) :: rtol = 1.0e-10_8
-    real(8) :: atol = 1.0e-10_8
-
-#ifdef PETSC
-    call KSPCreate(PETSC_COMM_WORLD, self % ksp, petsc_err)
-    call KSPSetTolerances(self % ksp, rtol, atol, &
-         PETSC_DEFAULT_DOUBLE_PRECISION, PETSC_DEFAULT_INTEGER, petsc_err)
-    call KSPSetType(self % ksp, KSPGMRES, petsc_err)
-    call KSPSetInitialGuessNonzero(self % ksp, PETSC_TRUE, petsc_err)
-    call KSPGetPC(self % ksp, self % pc, petsc_err)
-    call PCFactorSetLevels(self % pc, ilu_levels, petsc_err)
-    call KSPSetFromOptions(self % ksp, petsc_err)
-#endif
+    ! Set up default values
+    self % im = 40
+    self % iout = 0
+    self % maxits = 1000
+    self % lfil = 0
+    self % tol = 1.e-10_8
+    self % droptol = 0.0_8
 
   end subroutine gmres_create
 
@@ -94,15 +129,35 @@ contains
 
   subroutine gmres_set_oper(self, prec_mat, mat_in)
 
-    class(Petsc_gmres) :: self
+    class(GMRESSolver) :: self
     type(Matrix)       :: prec_mat
-    type(Matrix)       :: mat_in
+    type(Matrix), target :: mat_in
 
-#ifdef PETSC
-    call KSPSetOperators(self % ksp, mat_in % petsc_mat, prec_mat % petsc_mat, &
-         SAME_NONZERO_PATTERN, petsc_err)
-    call KSPSetUp(self % ksp, petsc_err) 
-#endif
+    integer :: ierr
+
+    ! Set pointer to matrix
+    self % A => mat_in
+
+    ! Set up data
+    self % iwk = self % A % nnz + 1
+
+    ! Set up arrays
+    allocate(self % ju(self % A % n))
+    allocate(self % jlu(self % iwk))
+    allocate(self % alu(self % iwk))
+    allocate(self % jr(self % A % n))
+    allocate(self % jwu(self % A % n))
+    allocate(self % jwl(self % A % n))
+    allocate(self % vv(self % A % n * (self % im + 1)))
+    allocate(self % wu(self % A % n + 1))
+    allocate(self % wl(self % A % n))
+    allocate(self % rhs(self % A % n))
+
+    ! Perform ILU Preconditioning
+    call ilut(self % A % n, self % A % val, self % A % col, self % A % row, &
+              self % lfil, self % droptol, self % alu, self % jlu, self % ju, &
+              self % iwk, self % wu, self % wl, self % jr, self % jwl, &
+              self % jwu, ierr)
 
   end subroutine gmres_set_oper
 
@@ -112,11 +167,19 @@ contains
 
   subroutine gmres_destroy(self)
 
-   class(Petsc_gmres) :: self
+   class(GMRESSolver) :: self
 
-#ifdef PETSC
-    call KSPDestroy(self % ksp, petsc_err)
-#endif
+   ! Destroy all arrays associated with GMRES solver
+   deallocate(self % ju)
+   deallocate(self % jlu)
+   deallocate(self % alu)
+   deallocate(self % jr)
+   deallocate(self % jwu)
+   deallocate(self % jwl)
+   deallocate(self % vv)
+   deallocate(self % wu)
+   deallocate(self % wl)
+   deallocate(self % rhs)
 
   end subroutine gmres_destroy
 
@@ -126,13 +189,20 @@ contains
 
   subroutine gmres_solve(self, b, x)
 
-    class(Petsc_gmres) :: self
+    class(GMRESSolver) :: self
     type(Vector)       :: b
     type(Vector)       :: x
 
-#ifdef PETSC
-    call KSPSolve(self % ksp, b % petsc_vec, x % petsc_vec, petsc_err)
-#endif
+    integer :: ierr ! error code
+
+    ! Copy rhs
+    self % rhs = b % val
+
+    ! Perform GMRES calculation
+    call pgmres(self % A % n, self % im, self % rhs, x % val, self % vv, &
+                self % tol, self % maxits, self % iout, self % A % val, &
+                self % A % col, self % A % row, self % alu, self % jlu, &
+                self % ju, ierr)
 
   end subroutine gmres_solve
 
@@ -142,9 +212,130 @@ contains
 
   subroutine jfnk_create(self)
 
-    class(Petsc_jfnk) :: self
+    class(JFNKSolver) :: self
+
+    message = 'JFNK cannot be used without PETSc.'
+    call fatal_error()
+
+  end subroutine jfnk_create
+
+!===============================================================================
+! JFNK_DESTROY frees all memory associated with JFNK solver
+!===============================================================================
+
+  subroutine jfnk_destroy(self)
+
+    class(JFNKSolver) :: self
+
+    message = 'JFNK cannot be used without PETSc.'
+    call fatal_error()
+
+  end subroutine jfnk_destroy
+
+!===============================================================================
+! JFNK_SET_FUNCTIONS sets user functions and matrix free objects
+!===============================================================================
+
+  subroutine jfnk_set_functions(self, ctx, res, jac_prec)
+
+    class(JFNKSolver) :: self
+    type(Jfnk_ctx)    :: ctx
+    type(Vector)      :: res
+    type(Matrix)      :: jac_prec
+
+    message = 'JFNK cannot be used without PETSc.'
+    call fatal_error()
+
+  end subroutine jfnk_set_functions
+
+!===============================================================================
+! JFNK_SOLVE solves the nonlinear system
+!===============================================================================
+
+  subroutine jfnk_solve(self, xvec)
+
+    class(JFNKSolver) :: self
+    type(Vector)      :: xvec
+
+    message = 'JFNK cannot be used without PETSc.'
+    call fatal_error()
+
+  end subroutine jfnk_solve
+
+#else
+!===============================================================================
+! PETSC_GMRES_CREATE sets up a PETSc GMRES solver
+!===============================================================================
+
+  subroutine petsc_gmres_create(self)
+
+    class(GMRESSolver) :: self
+
+    integer :: ilu_levels = 5
+    real(8) :: rtol = 1.0e-10_8
+    real(8) :: atol = 1.0e-10_8
+
+    call KSPCreate(PETSC_COMM_WORLD, self % ksp, petsc_err)
+    call KSPSetTolerances(self % ksp, rtol, atol, &
+         PETSC_DEFAULT_DOUBLE_PRECISION, PETSC_DEFAULT_INTEGER, petsc_err)
+    call KSPSetType(self % ksp, KSPGMRES, petsc_err)
+    call KSPSetInitialGuessNonzero(self % ksp, PETSC_TRUE, petsc_err)
+    call KSPGetPC(self % ksp, self % pc, petsc_err)
+    call PCFactorSetLevels(self % pc, ilu_levels, petsc_err)
+    call KSPSetFromOptions(self % ksp, petsc_err)
+
+  end subroutine petsc_gmres_create
+
+!===============================================================================
+! PETSC_GMRES_SET_OPER sets the matrix opetors for the GMRES solver
+!===============================================================================
+
+  subroutine petsc_gmres_set_oper(self, prec_mat, mat_in)
+
+    class(GMRESSolver) :: self
+    type(Matrix)       :: prec_mat
+    type(Matrix)       :: mat_in
+
+    call KSPSetOperators(self % ksp, mat_in % petsc_mat, prec_mat % petsc_mat, &
+         SAME_NONZERO_PATTERN, petsc_err)
+    call KSPSetUp(self % ksp, petsc_err) 
+
+  end subroutine petsc_gmres_set_oper
+
+!===============================================================================
+! PETSC_GMRES_DESTROY frees all memory associated with the GMRES solver
+!===============================================================================
+
+  subroutine petsc_gmres_destroy(self)
+
+   class(GMRESSolver) :: self
+
+    call KSPDestroy(self % ksp, petsc_err)
+
+  end subroutine petsc_gmres_destroy
+
+!===============================================================================
+! PETSC_GMRES_SOLVE solves the linear system
+!===============================================================================
+
+  subroutine petsc_gmres_solve(self, b, x)
+
+    class(GMRESSolver) :: self
+    type(Vector)       :: b
+    type(Vector)       :: x
+
+    call KSPSolve(self % ksp, b % petsc_vec, x % petsc_vec, petsc_err)
+
+  end subroutine petsc_gmres_solve
+
+!===============================================================================
+! PETSC_JFNK_CREATE sets up a JFNK solver using PETSc SNES
+!===============================================================================
+
+  subroutine petsc_jfnk_create(self)
+
+    class(JFNKSolver) :: self
 
-#ifdef PETSC
     ! Turn on mf_operator option for matrix free jacobian
     call PetscOptionsSetValue("-snes_mf_operator", "TRUE", petsc_err)
 
@@ -166,75 +357,68 @@ contains
     call PCFactorSetLevels(self % pc, 5, petsc_err)
     call PCSetFromOptions(self % pc, petsc_err)
     call KSPSetFromOptions(self % ksp, petsc_err)
-#endif
 
-  end subroutine jfnk_create
+  end subroutine petsc_jfnk_create
 
 !===============================================================================
-! JFNK_DESTROY frees all memory associated with JFNK solver
+! PETSC_JFNK_DESTROY frees all memory associated with JFNK solver
 !===============================================================================
 
-  subroutine jfnk_destroy(self)
+  subroutine petsc_jfnk_destroy(self)
 
-    class(Petsc_jfnk) :: self
+    class(JFNKSolver) :: self
 
-#ifdef PETSC
     call MatDestroy(self % jac_mf, petsc_err)
     call SNESDestroy(self % snes, petsc_err)
-#endif
 
-  end subroutine jfnk_destroy
+  end subroutine petsc_jfnk_destroy
 
 !===============================================================================
-! JFNK_SET_FUNCTIONS sets user functions and matrix free objects
+! PETSC_JFNK_SET_FUNCTIONS sets user functions and matrix free objects
 !===============================================================================
 
-  subroutine jfnk_set_functions(self, ctx, res, jac_prec)
+  subroutine petsc_jfnk_set_functions(self, ctx, res, jac_prec)
 
-    class(Petsc_jfnk) :: self
+    class(JFNKSolver) :: self
     type(Jfnk_ctx)    :: ctx
     type(Vector)      :: res
     type(Matrix)      :: jac_prec
 
-# ifdef PETSC
     ! Set residual procedure
-    call SNESSetFunction(self % snes, res % petsc_vec, jfnk_compute_residual, &
-         ctx, petsc_err)
+    call SNESSetFunction(self % snes, res % petsc_vec, &
+         petsc_jfnk_compute_residual, ctx, petsc_err)
 
     ! Create the matrix free jacobian
     call MatCreateSNESMF(self % snes, self % jac_mf, petsc_err)
 
     ! Set Jacobian procedure
     call SNESSetJacobian(self % snes, self % jac_mf, jac_prec % petsc_mat, &
-         jfnk_compute_jacobian, ctx, petsc_err)
+         petsc_jfnk_compute_jacobian, ctx, petsc_err)
 
     ! Set up Jacobian Lags
     call SNESSetLagJacobian(self % snes, -2, petsc_err)
     call SNESSetLagPreconditioner(self % snes, -1, petsc_err)
-#endif
 
-  end subroutine jfnk_set_functions
+  end subroutine petsc_jfnk_set_functions
 
 !===============================================================================
-! JFNK_SOLVE solves the nonlinear system
+! PETSC_JFNK_SOLVE solves the nonlinear system
 !===============================================================================
 
-  subroutine jfnk_solve(self, xvec)
+  subroutine petsc_jfnk_solve(self, xvec)
 
-    class(Petsc_jfnk) :: self
+    class(JFNKSolver) :: self
     type(Vector)      :: xvec
 
-#ifdef PETSC
     call SNESSolve(self % snes, PETSC_NULL_DOUBLE, xvec % petsc_vec, petsc_err)
-#endif
 
-  end subroutine jfnk_solve
+  end subroutine petsc_jfnk_solve
 
 !===============================================================================
-! JFNK_COMPUTE_RESIDUAL buffer routine to user specifed residual routine
+! PETSC_JFNK_COMPUTE_RESIDUAL buffer routine to user specifed residual routine
 !===============================================================================
-# ifdef PETSC
-  subroutine jfnk_compute_residual(snes, x, res, ctx, ierr)
+
+  subroutine petsc_jfnk_compute_residual(snes, x, res, ctx, ierr)
 
     SNES           :: snes
     Vec            :: x
@@ -260,13 +444,13 @@ contains
     call VecRestoreArrayF90(x, xvec % val, ierr)
     call VecRestoreArrayF90(res, resvec % val, ierr)
 
-  end subroutine jfnk_compute_residual
-#endif
+  end subroutine petsc_jfnk_compute_residual
+
 !===============================================================================
-! JFNK_COMPUTE_JACOBIAN
+! PETSC_JFNK_COMPUTE_JACOBIAN buffer routine to user specified jacobian routine
 !===============================================================================
-#ifdef PETSC
-  subroutine jfnk_compute_jacobian(snes, x, jac_mf, jac_prec, flag, ctx, &
+
+  subroutine petsc_jfnk_compute_jacobian(snes, x, jac_mf, jac_prec, flag, ctx, &
              ierr)
 
     SNES           :: snes
@@ -294,6 +478,7 @@ contains
     call MatAssemblyBegin(jac_mf, MAT_FINAL_ASSEMBLY, petsc_err)
     call MatAssemblyEnd(jac_mf, MAT_FINAL_ASSEMBLY, petsc_err)
 
-  end subroutine jfnk_compute_jacobian
+  end subroutine petsc_jfnk_compute_jacobian
 #endif
-end module petsc_solver
+
+end module solver_interface

From 4b6b1a7b98902ab9313bd95d4de7c848cb1d711d Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 27 Aug 2013 15:50:57 -0400
Subject: [PATCH 44/69] re-added back in fatal error if petsc is not compiled
 when using cmfd

---
 src/input_xml.F90 | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/input_xml.F90 b/src/input_xml.F90
index a0731ee70b..6243f7a215 100644
--- a/src/input_xml.F90
+++ b/src/input_xml.F90
@@ -561,6 +561,10 @@ contains
     call lower_case(run_cmfd_)
     if (run_cmfd_ == 'true' .or. run_cmfd_ == '1') then
       cmfd_run = .true.
+#ifndef PETSC
+      message = 'CMFD is not available, recompile OpenMC with PETSc'
+      call fatal_error()
+#endif
     end if
 
   end subroutine read_settings_xml

From e2d82f35a244aaa23be2b2f45f80707d1a2887c8 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 27 Aug 2013 16:11:27 -0400
Subject: [PATCH 45/69] removed built-in GMRES solver only PETSc available
 again

---
 src/DEPENDENCIES          |    1 -
 src/cmfd_jfnk_solver.F90  |   10 +-
 src/cmfd_power_solver.F90 |   10 +-
 src/gmres_solver.F90      | 1231 -------------------------------------
 src/solver_interface.F90  |  193 +-----
 5 files changed, 19 insertions(+), 1426 deletions(-)
 delete mode 100644 src/gmres_solver.F90

diff --git a/src/DEPENDENCIES b/src/DEPENDENCIES
index 782f48f61a..18ff2eeb0c 100644
--- a/src/DEPENDENCIES
+++ b/src/DEPENDENCIES
@@ -327,7 +327,6 @@ set_header.o: list_header.o
 
 solver_interface.o: error.o
 solver_interface.o: global.o
-solver_interface.o: gmres_solver.o
 solver_interface.o: matrix_header.o
 solver_interface.o: vector_header.o
 
diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
index 3208e62146..9b14b4d1aa 100644
--- a/src/cmfd_jfnk_solver.F90
+++ b/src/cmfd_jfnk_solver.F90
@@ -46,19 +46,25 @@ contains
     call init_data()
 
     ! Initialize solver
+#ifdef PETSC
     call jfnk % create()
+#endif
 
     ! Set up residual and jacobian routines
     jfnk_data % res_proc_ptr => compute_nonlinear_residual
     jfnk_data % jac_proc_ptr => build_jacobian_matrix
+#ifdef PETSC
     call jfnk % set_functions(jfnk_data, resvec, jac_prec)
+#endif
 
     ! Stop timer for build
     call time_cmfdbuild % stop()
 
     ! Solve the system
     call time_cmfdsolve % start()
+#ifdef PETSC
     call jfnk % solve(xvec)
+#endif
     call time_cmfdsolve % stop()
 
     ! Extracts results to cmfd object
@@ -378,7 +384,9 @@ contains
     call jac_prec % destroy()
     call xvec % destroy()
     call resvec % destroy()
-    call jfnk % destroy() 
+#ifdef PETSC
+    call jfnk % destroy()
+#endif
     nullify(jfnk_data % res_proc_ptr)
     nullify(jfnk_data % jac_proc_ptr)
 
diff --git a/src/cmfd_power_solver.F90 b/src/cmfd_power_solver.F90
index d57f386ee5..cd31789407 100644
--- a/src/cmfd_power_solver.F90
+++ b/src/cmfd_power_solver.F90
@@ -63,7 +63,9 @@ contains
     call time_cmfdbuild % start()
 
     ! Initialize solver
+#ifdef PETSC
     call gmres % create()
+#endif
 
     ! Initialize matrices and vectors
     call init_data(physical_adjoint)
@@ -73,7 +75,9 @@ contains
         call compute_adjoint()
 
     ! Set up krylov info
+#ifdef PETSC
     call gmres % set_oper(loss, loss)
+#endif
 
     ! Stop timer for build
     call time_cmfdbuild % stop()
@@ -196,7 +200,9 @@ contains
       s_o % val = s_o % val / k_o
 
       ! Compute new flux vector
+#ifdef PETSC
       call gmres % solve(s_o, phi_n)
+#endif
 
       ! Compute new source vector
       call prod % vector_multiply(phi_n, s_n)
@@ -324,7 +330,9 @@ contains
   subroutine finalize()
 
     ! Destroy all objects 
-    call gmres  % destroy() 
+#ifdef PETSC
+    call gmres  % destroy()
+#endif
     call loss   % destroy() 
     call prod   % destroy()
     call phi_n  % destroy()
diff --git a/src/gmres_solver.F90 b/src/gmres_solver.F90
deleted file mode 100644
index 9345f4f9a5..0000000000
--- a/src/gmres_solver.F90
+++ /dev/null
@@ -1,1231 +0,0 @@
-module gmres_solver
-
-  implicit none
-
-contains
-
-subroutine pgmres ( n, im, rhs, sol, vv, eps, maxits, iout, &
-  aa, ja, ia, alu, jlu, ju, ierr )
-
-!*****************************************************************************80
-!
-!! PGMRES is an ILUT - Preconditioned GMRES solver.
-!                                                                      
-!  Discussion:
-!
-!    This is a simple version of the ILUT preconditioned GMRES algorithm. 
-!    The ILUT preconditioner uses a dual strategy for dropping elements   
-!    instead  of the usual level of-fill-in approach. See details in ILUT 
-!    subroutine documentation. PGMRES uses the L and U matrices generated 
-!    from the subroutine ILUT to precondition the GMRES algorithm.        
-!    The preconditioning is applied to the right. The stopping criterion  
-!    utilized is based simply on reducing the residual norm by epsilon.   
-!    This preconditioning is more reliable than ilu0 but requires more    
-!    storage. It seems to be much less prone to difficulties related to   
-!    strong nonsymmetries in the matrix. We recommend using a nonzero tol 
-!    (tol=.005 or .001 usually give good results) in ILUT. Use a large    
-!    lfil whenever possible (e.g. lfil = 5 to 10). The higher lfil the    
-!    more reliable the code is. Efficiency may also be much improved.     
-!    Note that lfil=n and tol=0.0 in ILUT  will yield the same factors as 
-!    Gaussian elimination without pivoting.                               
-!                                                                      
-!    ILU(0) and MILU(0) are also provided for comparison purposes         
-!    USAGE: first call ILUT or ILU0 or MILU0 to set up preconditioner and 
-!    then call pgmres.                                                    
-!
-!  Modified:
-!
-!    07 January 2004
-!
-!  Author:
-!
-!    Youcef Saad              
-!
-!  Parameters:
-!
-!    Input, integer ( kind = 4 ) N, the order of the matrix.
-!
-!    Input, integer ( kind = 4 ) IM, the size of the Krylov subspace.  IM 
-!    should not exceed 50 in this version.  This restriction can be reset by 
-!    changing the parameter command for KMAX below.
-!                                                
-!    Input/output, real RHS(N), on input, the right hand side vector.
-!    On output, the information in this vector has been destroyed.
-!
-! sol   == real vector of length n containing an initial guess to the  
-!          solution on input. approximate solution on output           
-!
-! eps   == tolerance for stopping criterion. process is stopped        
-!          as soon as ( ||.|| is the euclidean norm):                  
-!          || current residual||/||initial residual|| <= eps           
-!
-! maxits== maximum number of iterations allowed                        
-!
-! iout  == output unit number number for printing intermediate results 
-!          if (iout <= 0) nothing is printed out.                    
-!                                                                      
-!    Input, real AA(*), integer ( kind = 4 ) JA(*), IA(N+1), the matrix in CSR
-!    Compressed Sparse Row format.
-!
-!                                                                      
-! alu,jlu== A matrix stored in Modified Sparse Row format containing   
-!           the L and U factors, as computed by routine ilut.       
-!                                                                      
-! ju     == integer ( kind = 4 ) array of length n containing the pointers to       
-!           the beginning of each row of U in alu, jlu as computed     
-!           by routine ILUT.                                        
-!                                                                      
-! on return:                                                           
-!                                                          
-! sol   == contains an approximate solution (upon successful return).  
-! ierr  == integer ( kind = 4 ). Error message with the following meaning.          
-!          ierr = 0 --> successful return.                            
-!          ierr = 1 --> convergence not achieved in itmax iterations. 
-!          ierr =-1 --> the initial guess seems to be the exact        
-!                       solution (initial residual computed was zero) 
-!                                                                      
-! work arrays:                                                        
-!                                                       
-! vv    == work array of length  n x (im+1) (used to store the Arnoli  
-!          basis)                                                      
-!
-  integer ( kind = 4 ), parameter :: kmax = 50
-  integer ( kind = 4 ) n
-
-  real ( kind = 8 ) aa(*)
-  real ( kind = 8 ) alu(*)
-  real ( kind = 8 ) c(kmax)
-  real ( kind = 8 ) eps
-  real ( kind = 8 ) eps1
-  real ( kind = 8 ), parameter :: epsmac = 1.0D-16
-  real ( kind = 8 ) gam
-  real ( kind = 8 ) hh(kmax+1,kmax)
-  integer ( kind = 4 ) i
-  integer ( kind = 4 ) i1
-  integer ( kind = 4 ) ia(n+1)
-  integer ( kind = 4 ) ierr
-  integer ( kind = 4 ) ii
-  integer ( kind = 4 ) im
-  integer ( kind = 4 ) iout
-  integer ( kind = 4 ) its
-  integer ( kind = 4 ) j
-  integer ( kind = 4 ) ja(*)
-  integer ( kind = 4 ) jj
-  integer ( kind = 4 ) jlu(*)
-  integer ( kind = 4 ) ju(*)
-  integer ( kind = 4 ) k
-  integer ( kind = 4 ) k1
-  integer ( kind = 4 ) maxits
-  integer ( kind = 4 ) n1
-  real ( kind = 8 ) rhs(n)
-  real ( kind = 8 ) ro
-  real ( kind = 8 ) rs(kmax+1)
-  real ( kind = 8 ) s(kmax)
-  real ( kind = 8 ) sol(n)
-  real ( kind = 8 ) t
-  real ( kind = 8 ) vv(n,*)
-!
-!  Arnoldi size should not exceed KMAX=50 in this version.
-!  To reset modify parameter KMAX accordingly.
-!
-  n1 = n + 1
-  its = 0
-!
-!  Outer loop starts here.
-!  Compute initial residual vector.
-!
-  call ope ( n, sol, vv, aa, ja, ia )
-
-  vv(1:n,1) = rhs(1:n) - vv(1:n,1)
-
-  do
-
-    ro = sqrt ( ddot ( n, vv, 1, vv, 1 ) )
-
-    if ( 0 < iout .and. its == 0 ) then
-      write(iout, 199) its, ro
-    end if
-
-    if ( ro == 0.0D+00 ) then
-      ierr = -1
-      exit
-    end if
-
-    t = 1.0D+00 / ro
-    vv(1:n,1) = vv(1:n,1) * t
-
-    if ( its == 0 ) then
-      eps1 = eps * ro
-    end if
-!
-!  Initialize first term of RHS of Hessenberg system.
-!
-     rs(1) = ro
-     i = 0
-
- 4   continue
-
-     i = i + 1
-     its = its + 1
-     i1 = i + 1
-     call lusol0 ( n, vv(1,i), rhs, alu, jlu, ju )
-     call ope ( n, rhs, vv(1,i1), aa, ja, ia )
-!
-!  Modified Gram - Schmidt.
-!
-     do j = 1, i
-       t = ddot ( n, vv(1,j), 1, vv(1,i1), 1 )
-       hh(j,i) = t
-       call daxpy ( n, -t, vv(1,j), 1, vv(1,i1), 1 )
-     end do
-
-     t = sqrt ( ddot ( n, vv(1,i1), 1, vv(1,i1), 1 ) )
-     hh(i1,i) = t
-
-     if ( t /= 0.0D+00 ) then
-       t = 1.0D+00 / t
-       vv(1:n,i1) = vv(1:n,i1) * t
-     end if
-!
-!  Update factorization of HH.
-!
-    if ( i == 1 ) then
-      go to 121
-    end if
-!
-!  Perform previous transformations on I-th column of H.
-!
-    do k = 2, i
-       k1 = k-1
-       t = hh(k1,i)
-       hh(k1,i) = c(k1) * t + s(k1) * hh(k,i)
-       hh(k,i) = -s(k1) * t + c(k1) * hh(k,i)
-    end do
-
-121 continue
-
-    gam = sqrt ( hh(i,i)**2 + hh(i1,i)**2 )
-!
-!  If GAMMA is zero then any small value will do.
-!  It will affect only residual estimate.
-!
-    if ( gam == 0.0D+00 ) then
-      gam = epsmac
-    end if
-!
-!  Get the next plane rotation.
-!
-    c(i) = hh(i,i) / gam
-    s(i) = hh(i1,i) / gam
-    rs(i1) = -s(i) * rs(i)
-    rs(i) = c(i) * rs(i)
-!
-!  Determine residual norm and test for convergence.
-!
-    hh(i,i) = c(i) * hh(i,i) + s(i) * hh(i1,i)
-    ro = abs ( rs(i1) )
-131 format(1h ,2e14.4)
-
-    if ( 0 < iout ) then
-      write(iout, 199) its, ro
-    end if
-
-    if ( i < im .and. eps1 < ro ) then
-      go to 4
-    end if
-!
-!  Now compute solution.  First solve upper triangular system.
-!
-    rs(i) = rs(i) / hh(i,i)
-
-    do ii = 2, i
-      k = i - ii + 1
-      k1 = k + 1
-      t = rs(k)
-      do j = k1, i
-        t = t - hh(k,j) * rs(j)
-      end do
-      rs(k) = t / hh(k,k)
-    end do
-!
-!  Form linear combination of V(*,i)'s to get solution.
-!
-    t = rs(1)
-    rhs(1:n) = vv(1:n,1) * t
-
-    do j = 2, i
-      t = rs(j)
-      rhs(1:n) = rhs(1:n) + t * vv(1:n,j)
-    end do
-!
-!  Call preconditioner.
-!
-    call lusol0 ( n, rhs, rhs, alu, jlu, ju )
-
-    sol(1:n) = sol(1:n) + rhs(1:n)
-!
-!  Restart outer loop when necessary.
-!
-    if ( ro <= eps1 ) then
-      ierr = 0
-      exit
-    end if
-
-    if ( maxits < its ) then
-      ierr = 1
-      exit
-    end if
-!
-!  Else compute residual vector and continue.
-!
-    do j = 1, i
-      jj = i1 - j + 1
-      rs(jj-1) = -s(jj-1) * rs(jj)
-      rs(jj) = c(jj-1) * rs(jj)
-    end do
-
-    do j = 1, i1
-      t = rs(j)
-      if ( j == 1 ) then
-        t = t - 1.0D+00
-      end if
-      call daxpy ( n, t, vv(1,j), 1,  vv, 1 )
-    end do
-
-199 format(' its =', i4, ' res. norm =', G14.6)
-
-  end do
-
-  return
-end
-
-subroutine ilut ( n, a, ja, ia, lfil, tol, alu, jlu, ju, iwk, wu, wl, jr, &
-  jwl, jwu, ierr )
-
-!*****************************************************************************80
-!
-!! ILUT is an ILUT preconditioner.
-!
-!  Discussion:
-!
-!    This routine carries ouot incomplete LU factorization with dual
-!    truncation mechanism.  Sorting is done for both L and U. 
-!
-!    The dual drop-off strategy works as follows:
-!
-!    1) Theresholding in L and U as set by TOL.  Any element whose size
-!       is less than some tolerance (relative to the norm of current
-!       row in u) is dropped.
-!         
-!    2) Keeping only the largest lenl0+lfil elements in L and the
-!       largest lenu0+lfil elements in U, where lenl0=initial number 
-!       of nonzero elements in a given row of lower part of A 
-!       and lenlu0 is similarly defined.
-!  
-!    Flexibility: one can use tol=0 to get a strategy based on keeping the
-!    largest elements in each row of L and U. Taking tol /= 0 but lfil=n
-!    will give the usual threshold strategy (however, fill-in is then     
-!    unpredictible).                                                      
-!
-!    A must have all nonzero diagonal elements.
-!
-!  Modified:
-!
-!    07 January 2004
-!
-!  Author:
-!
-!    Youcef Saad
-!                                                                      
-!  Parameters:
-!
-!  Parameters:
-!
-!    Input, integer ( kind = 4 ) N, the order of the matrix.
-!
-!    Input, real A(*), integer ( kind = 4 ) JA(*), IA(N+1), the matrix in CSR
-!    Compressed Sparse Row format.
-!
-! lfil    = integer ( kind = 4 ). The fill-in parameter. Each row of L and
-!           each row of U will have a maximum of lfil elements
-!           in addition to the original number of nonzero elements.
-!           Thus storage can be determined beforehand.
-!           lfil must be >= 0.
-!
-! iwk     = integer ( kind = 4 ). The minimum length of arrays alu and jlu
-!
-! On return:
-!
-! alu,jlu = matrix stored in Modified Sparse Row (MSR) format containing
-!           the L and U factors together. The diagonal (stored in
-!           alu(1:n) ) is inverted. Each i-th row of the alu,jlu matrix
-!           contains the i-th row of L (excluding the diagonal entry=1)
-!           followed by the i-th row of U.
-!
-! ju      = integer ( kind = 4 ) array of length n containing the pointers to
-!           the beginning of each row of U in the matrix alu,jlu.
-!
-! ierr    = integer ( kind = 4 ). Error message with the following meaning.
-!           ierr  = 0    --> successful return.
-!           ierr > 0  --> zero pivot encountered at step number ierr.
-!           ierr  = -1   --> Error. input matrix may be wrong.
-!                            (The elimination process has generated a
-!                            row in L or U whose length is >  n.)
-!           ierr  = -2   --> The matrix L overflows the array al.
-!           ierr  = -3   --> The matrix U overflows the array alu.
-!           ierr  = -4   --> Illegal value for lfil.
-!           ierr  = -5   --> zero pivot encountered.
-!
-! work arrays:
-!
-! jr,jwu,jwl, integer ( kind = 4 ) work arrays of length n.
-! wu, wl, real work arrays of length n+1, and n resp.
-!
-  integer ( kind = 4 ) n
-
-  real ( kind = 8 ) a(*)
-  real ( kind = 8 ) alu(*)
-  real ( kind = 8 ) fact
-  integer ( kind = 4 ) ia(n+1)
-  integer ( kind = 4 ) idiag
-  integer ( kind = 4 ) ierr
-  integer ( kind = 4 ) ii
-  integer ( kind = 4 ) iwk
-  integer ( kind = 4 ) j 
-  integer ( kind = 4 ) j1
-  integer ( kind = 4 ) j2
-  integer ( kind = 4 ) ja(*)
-  integer ( kind = 4 ) jj
-  integer ( kind = 4 ) jlu(*)
-  integer ( kind = 4 ) jpos
-  integer ( kind = 4 ) jr(*)
-  integer ( kind = 4 ) jrow
-  integer ( kind = 4 ) ju(*)
-  integer ( kind = 4 ) ju0
-  integer ( kind = 4 ) jwl(n)
-  integer ( kind = 4 ) jwu(n)
-  integer ( kind = 4 ) k
-  integer ( kind = 4 ) len
-  integer ( kind = 4 ) lenl
-  integer ( kind = 4 ) lenl0
-  integer ( kind = 4 ) lenu
-  integer ( kind = 4 ) lenu0
-  integer ( kind = 4 ) lfil
-  integer ( kind = 4 ) nl
-  real ( kind = 8 ) s
-  real ( kind = 8 ) t
-  real ( kind = 8 ) tnorm
-  real ( kind = 8 ) tol
-  real ( kind = 8 ) wl(n)
-  real ( kind = 8 ) wu(n+1)
-
-  if ( lfil < 0 ) then
-    ierr = -4
-    return
-  end if
-!
-!  Initialize JU0 (points to next element to be added to ALU, JLU)
-!  and pointer.
-!
-  ju0 = n + 2
-  jlu(1) = ju0
-!
-!  integer ( kind = 4 ) double pointer array.
-!
-  jr(1:n) = 0
-!
-!  The main loop.
-!
-  do ii = 1, n
-
-    j1 = ia(ii)
-    j2 = ia(ii+1) - 1
-    lenu = 0
-    lenl = 0
-
-    tnorm = 0.0D+00
-    do k = j1, j2
-      tnorm = tnorm + abs ( a(k) )
-    end do
-    tnorm = tnorm / real ( j2-j1+1, kind = 8 )
-!
-!  Unpack L-part and U-part of row of A in arrays WL, WU.
-!
-    do j = j1, j2
-
-      k = ja(j)
-      t = a(j)
-
-      if ( tol * tnorm <= abs ( t ) ) then
-
-        if ( k < ii ) then
-          lenl = lenl + 1
-          jwl(lenl) = k
-          wl(lenl) = t
-          jr(k) = lenl
-        else
-          lenu = lenu+1
-          jwu(lenu) = k
-          wu(lenu) = t
-          jr(k) = lenu
-        end if
-
-      end if
-
-    end do
-
-    lenl0 = lenl
-    lenu0 = lenu
-    jj = 0
-    nl = 0
-!
-!  Eliminate previous rows.
-!
-150 continue
-
-    jj = jj + 1
-
-    if ( lenl < jj ) then
-      go to 160
-    end if
-!
-!  In order to do the elimination in the correct order we need to
-!  exchange the current row number with the one that has
-!  smallest column number, among JJ, JJ+1, ..., LENL.
-!
-    jrow = jwl(jj)
-    k = jj
-!
-!  Determine the smallest column index.
-!
-    do j = jj+1, lenl
-       if ( jwl(j) < jrow ) then
-          jrow = jwl(j)
-          k = j
-       end if
-    end do
-!
-!  Exchange in JWL.
-!
-    j = jwl(jj)
-    jwl(jj) = jrow
-    jwl(k) = j
-!
-!  Exchange in JR.
-!
-    jr(jrow) = jj
-    jr(j) = k
-!
-!  Exchange in WL.
-!
-    s = wl(k)
-    wl(k) = wl(jj)
-    wl(jj) = s
-
-    if ( ii <= jrow ) then
-      go to 160
-    end if
-!
-!  Get the multiplier for row to be eliminated: JROW.
-!
-    fact = wl(jj) * alu(jrow)
-    jr(jrow) = 0
-
-    if ( abs ( fact ) * wu(n+2-jrow) <= tol * tnorm ) then
-      go to 150
-    end if
-!
-!  Combine current row and row JROW.
-!
-    do k = ju(jrow), jlu(jrow+1)-1
-       s = fact * alu(k)
-       j = jlu(k)
-       jpos = jr(j)
-!
-!  If fill-in element and small disregard.
-!
-       if ( abs ( s ) < tol * tnorm .and. jpos == 0 ) then
-         cycle
-       end if
-
-       if ( ii <= j ) then
-!
-!  Dealing with upper part.
-!
-          if ( jpos == 0 ) then
-!
-!  This is a fill-in element.
-!
-             lenu = lenu + 1
-
-             if ( n < lenu ) then
-               go to 995
-             end if
-
-             jwu(lenu) = j
-             jr(j) = lenu
-             wu(lenu) = - s
-          else
-!
-!  No fill-in element.
-!
-             wu(jpos) = wu(jpos) - s
-          end if
-       else
-!
-!  Dealing with lower part.
-!
-          if ( jpos == 0 ) then
-!
-!  This is a fill-in element.
-!
-             lenl = lenl + 1
-
-             if ( n < lenl ) then
-               go to 995
-             end if
-
-             jwl(lenl) = j
-             jr(j) = lenl
-             wl(lenl) = -s
-          else
-!
-!  No fill-in element.
-!
-             wl(jpos) = wl(jpos) - s
-          end if
-       end if
-
-  end do
-
-    nl = nl + 1
-    wl(nl) = fact
-    jwl(nl) = jrow
-  go to 150
-!
-!  Update the L matrix.
-!
- 160 continue
-
-    len = min ( nl, lenl0 + lfil )
-
-    call bsort2 ( wl, jwl, nl, len )
-
-    do k = 1, len
-
-       if ( iwk < ju0 ) then
-         ierr = -2
-         return
-       end if
-
-       alu(ju0) =  wl(k)
-       jlu(ju0) =  jwl(k)
-       ju0 = ju0 + 1
-
-    end do
-!
-!  Save pointer to beginning of row II of U.
-!
-    ju(ii) = ju0
-!
-!  Reset double pointer JR to zero (L-part - except first
-!  JJ-1 elements which have already been reset).
-!
-  do k = jj, lenl
-    jr(jwl(k)) = 0
-  end do
-!
-!  Be sure that the diagonal element is first in W and JW.
-!
-    idiag = jr(ii)
-
-    if ( idiag == 0 ) then
-      go to 900
-    end if
-
-    if ( idiag /= 1 ) then
-
-       s = wu(1)
-       wu(j) = wu(idiag)
-       wu(idiag) = s
-
-       j = jwu(1)
-       jwu(1) = jwu(idiag)
-       jwu(idiag) = j
-
-    end if
-
-    len = min ( lenu, lenu0 + lfil )
-
-    call bsort2 ( wu(2), jwu(2), lenu-1, len )
-!
-! Update the U-matrix.
-!
-    t = 0.0D+00
-
-    do k = 2, len
-
-       if ( iwk < ju0 ) then
-         ierr = -3
-         return
-       end if
-
-       jlu(ju0) = jwu(k)
-       alu(ju0) = wu(k)
-       t = t + abs ( wu(k) )
-       ju0 = ju0 + 1
-
-    end do
-!
-!  Save norm in WU (backwards). Norm is in fact average absolute value.
-!
-    wu(n+2-ii) = t / real ( len + 1, kind = 8 )
-!
-!  Store inverse of diagonal element of U.
-!
-    if ( wu(1) == 0.0D+00 ) then
-      ierr = -5
-      return
-    end if
-
-    alu(ii) = 1.0D+00 / wu(1)
-!
-!  Update pointer to beginning of next row of U.
-!
-  jlu(ii+1) = ju0
-!
-!  Reset double pointer JR to zero (U-part).
-!
-  do k = 1, lenu
-    jr(jwu(k)) = 0
-  end do
-
-  end do
-
-  ierr = 0
-
-  return
-!
-!  Zero pivot :
-!
- 900    ierr = ii
-    return
-!
-!  Incomprehensible error. Matrix must be wrong.
-!
- 995    ierr = -1
-    return
-end
-
-subroutine amux ( n, x, y, a, ja, ia )
-
-!*****************************************************************************80
-!
-!! AMUX multiplies a CSR matrix A times a vector.
-!
-!  Discussion:
-!
-!    This routine multiplies a matrix by a vector using the dot product form.
-!    Matrix A is stored in compressed sparse row storage.
-!
-!  Modified:
-!
-!    07 January 2004
-!
-!  Author:
-!
-!    Youcef Saad
-!
-!  Parameters:
-!
-!    Input, integer ( kind = 4 ) N, the row dimension of the matrix.
-!
-!    Input, real X(*), and array of length equal to the column dimension 
-!    of A.
-!
-!    Input, real A(*), integer ( kind = 4 ) JA(*), IA(NROW+1), the matrix in CSR
-!    Compressed Sparse Row format.
-!
-!    Output, real Y(N), the product A * X.
-!
-  integer ( kind = 4 ) n
-
-  real ( kind = 8 ) a(*)
-  integer ( kind = 4 ) i
-  integer ( kind = 4 ) ia(*)
-  integer ( kind = 4 ) ja(*)
-  integer ( kind = 4 ) k
-  real ( kind = 8 ) t
-  real ( kind = 8 ) x(*)
-  real ( kind = 8 ) y(n)
-
-  do i = 1, n
-!
-!  Compute the inner product of row I with vector X.
-!
-    t = 0.0D+00
-    do k = ia(i), ia(i+1)-1
-      t = t + a(k) * x(ja(k))
-    end do
-
-    y(i) = t
-
-  end do
-
-  return
-end
-subroutine amuxd ( n, x, y, diag, ndiag, idiag, ioff )
-
-!*****************************************************************************80
-!
-!! AMUXD multiplies a DIA matrix times a vector.
-!
-!  Discussion:
-!
-!    This routine multiplies a matrix by a vector when the original matrix 
-!    is stored in the DIA diagonal storage format.
-!
-!  Modified:
-!
-!    07 January 2004
-!
-!  Author:
-!
-!    Youcef Saad
-!
-!  Parameters:
-!
-!    Input, integer ( kind = 4 ) N, the row dimension of the matrix.
-!
-!    Input, real X(*), array of length equal to the column dimension of
-!    the A matrix.
-!
-!    Output, real Y(N), the product A * X.
-!
-!    Input, real DIAG(NDIAG,IDIAG), the diagonals.
-!
-!    Input, integer ( kind = 4 ) NDIAG, the first dimension of array adiag as 
-!    declared in the calling program.
-!
-!    Input, integer ( kind = 4 ) IDIAG, the number of diagonals in the matrix.
-!
-!    Input, integer ( kind = 4 ) IOFF(IDIAG), the offsets of the diagonals of 
-!    the matrix: diag(i,k) contains the element a(i,i+ioff(k)) of the matrix.
-!
-  integer ( kind = 4 ) idiag
-  integer ( kind = 4 ) n
-  integer ( kind = 4 ) ndiag
-
-  real ( kind = 8 ) diag(ndiag,idiag)
-  integer ( kind = 4 ) i1
-  integer ( kind = 4 ) i2
-  integer ( kind = 4 ) io
-  integer ( kind = 4 ) ioff(idiag)
-  integer ( kind = 4 ) j
-  integer ( kind = 4 ) k
-  real ( kind = 8 ) x(n)
-  real ( kind = 8 ) y(n)
-
-  y(1:n) = 0.0D+00
-
-  do j = 1, idiag
-    io = ioff(j)
-    i1 = max ( 1, 1 - io )
-    i2 = min ( n, n - io )
-    do k = i1, i2
-      y(k) = y(k) + diag(k,j) * x(k+io)
-    end do
-  end do
-
-  return
-end
-
-subroutine daxpy ( n, da, dx, incx, dy, incy )
-
-!*****************************************************************************80
-!
-!! DAXPY computes constant times a vector plus a vector.
-!
-!  Discussion:
-!
-!    Uses unrolled loops for increments equal to one.
-!
-!  Author:
-!
-!    Jack Dongarra
-!
-!  Reference:
-!
-!    Dongarra, Moler, Bunch, Stewart,
-!    LINPACK User's Guide,
-!    SIAM, 1979.
-!
-!    Lawson, Hanson, Kincaid, Krogh,
-!    Basic Linear Algebra Subprograms for Fortran Usage,
-!    Algorithm 539,
-!    ACM Transactions on Mathematical Software,
-!    Volume 5, Number 3, September 1979, pages 308-323.
-!
-!  Parameters:
-!
-!    Input, integer ( kind = 4 ) N, the number of elements in DX and DY.
-!
-!    Input, real ( kind = 8 ) DA, the multiplier of DX.
-!
-!    Input, real ( kind = 8 ) DX(*), the first vector.
-!
-!    Input, integer ( kind = 4 ) INCX, the increment between successive entries of DX.
-!
-!    Input/output, real ( kind = 8 ) DY(*), the second vector.
-!    On output, DY(*) has been replaced by DY(*) + DA * DX(*).
-!
-!    Input, integer ( kind = 4 ) INCY, the increment between successive entries of DY.
-!
-  real ( kind = 8 ) da
-  real ( kind = 8 ) dx(*)
-  real ( kind = 8 ) dy(*)
-  integer ( kind = 4 ) i
-  integer ( kind = 4 ) incx
-  integer ( kind = 4 ) incy
-  integer ( kind = 4 ) ix
-  integer ( kind = 4 ) iy
-  integer ( kind = 4 ) m
-  integer ( kind = 4 ) n
-
-  if ( n <= 0 ) then
-    return
-  end if
-
-  if ( da  == 0.0D+00 ) then
-    return
-  end if
-!
-!  Code for unequal increments or equal increments
-!  not equal to 1.
-!
-  if ( incx /= 1 .or. incy /= 1 ) then
-
-    if ( 0 <= incx ) then
-      ix = 1
-    else
-      ix = ( - n + 1 ) * incx + 1
-    end if
-
-    if ( 0 <= incy ) then
-      iy = 1
-    else
-      iy = ( - n + 1 ) * incy + 1
-    end if
-
-    do i = 1, n
-      dy(iy) = dy(iy) + da * dx(ix)
-      ix = ix + incx
-      iy = iy + incy
-    end do
-!
-!  Code for both increments equal to 1.
-!
-  else
-
-    m = mod ( n, 4 )
-
-    do i = 1, m
-      dy(i) = dy(i) + da * dx(i)
-    end do
-
-    do i = m+1, n, 4
-      dy(i  ) = dy(i  ) + da * dx(i  )
-      dy(i+1) = dy(i+1) + da * dx(i+1)
-      dy(i+2) = dy(i+2) + da * dx(i+2)
-      dy(i+3) = dy(i+3) + da * dx(i+3)
-    end do
-
-  end if
-
-  return
-end
-
-subroutine bsort2 ( w, ind, n, ncut )
-
-!*****************************************************************************80
-!
-!! BSORT2 returns the NCUT largest elements of an array, using bubble sort.
-!
-!  Discussion:
-!
-!    This routine carries out a simple bubble sort for getting the NCUT largest
-!    elements in modulus, in array W.  IND is sorted accordingly.
-!    (Ought to be replaced by a more efficient sort especially
-!    if NCUT is not that small).
-!
-!  Modified:
-!
-!    07 January 2004
-!
-!  Author:
-!
-!    Youcef Saad
-!
-!  Parameters:
-!
-  integer ( kind = 4 ) n
-
-  integer ( kind = 4 ) i
-  integer ( kind = 4 ) ind(*)
-  integer ( kind = 4 ) iswp
-  integer ( kind = 4 ) j
-  integer ( kind = 4 ) ncut
-  logical test
-  real ( kind = 8 ) w(n)
-  real ( kind = 8 ) wswp
-
-  i = 1
-
-  do
-
-    test = .false.
-
-    do j = n-1, i, -1
-
-      if ( abs ( w(j) ) < abs ( w(j+1) ) ) then
-!
-!  Swap.
-!
-        wswp = w(j)
-        w(j) = w(j+1)
-        w(j+1) = wswp
-!
-!  Reorder the original ind array accordingly.
-!
-        iswp = ind(j)
-        ind(j) = ind(j+1)
-        ind(j+1) = iswp
-!
-!  Set indicator that sequence is still unsorted.
-!
-        test = .true.
-
-      end if
-
-    end do
-
-    i = i + 1
-
-    if ( .not. test .or. ncut < i ) then
-      exit
-    end if
-
-  end do
-
-  return
-end
-
-subroutine ope ( n, x, y, a, ja, ia )
-
-!*****************************************************************************80
-!
-!! OPE sparse matrix * vector multiplication
-!
-!  Modified:
-!
-!    07 January 2004
-!
-!  Author:
-!
-!    Youcef Saad
-!
-!  Parameters:
-!
-!    Input, integer ( kind = 4 ) N, the order of the matrix.
-!
-!    Input, real X(N), the vector to be multiplied.
-!
-!    Output, real Y(N), the product A * X.
-!
-!    Input, real A(*), integer ( kind = 4 ) JA(*), IA(N+1), the matrix in CSR
-!    Compressed Sparse Row format.
-!
-  integer ( kind = 4 ) n
-
-  real ( kind = 8 ) a(*)
-  integer ( kind = 4 ) i
-  integer ( kind = 4 ) ia(n+1)
-  integer ( kind = 4 ) ja(*)
-  integer ( kind = 4 ) k
-  integer ( kind = 4 ) k1
-  integer ( kind = 4 ) k2
-  real ( kind = 8 ) x(n)
-  real ( kind = 8 ) y(n)
-
-  do i = 1, n
-    k1 = ia(i)
-    k2 = ia(i+1) -1
-    y(i) = 0.0D+00
-    do k = k1, k2
-      y(i) = y(i) + a(k) * x(ja(k))
-    end do
-  end do
-
-  return
-end
-
-subroutine lusol0 ( n, y, x, alu, jlu, ju )
-
-!*****************************************************************************80
-!
-!! LUSOL0 performs a forward followed by a backward solve
-! for LU matrix as produced by  ILUT
-!
-!  Modified:
-!
-!    07 January 2004
-!
-!  Author:
-!
-!    Youcef Saad
-!
-!  Parameters:
-!
-!    Input, integer ( kind = 4 ) N, the order of the matrix.
-!
-!    Input, real Y(N), the right hand side of the linear system.
-!
-!    Output, real X(N), the solution.
-!
-!    ALU, JLU, JU, ...
-!
-  integer ( kind = 4 ) n
-
-  real ( kind = 8 ) alu(*)
-  integer ( kind = 4 ) i
-  integer ( kind = 4 ) jlu(*)
-  integer ( kind = 4 ) ju(*)
-  integer ( kind = 4 ) k
-  real ( kind = 8 ) x(n)
-  real ( kind = 8 ) y(n)
-!
-!  Forward solve
-!
-  do i = 1, n
-    x(i) = y(i)
-    do k = jlu(i), ju(i)-1
-      x(i) = x(i) - alu(k) * x(jlu(k))
-    end do
-  end do
-!
-!  Backward solve.
-!
-  do i = n, 1, -1
-    do k = ju(i), jlu(i+1)-1
-      x(i) = x(i) - alu(k) * x(jlu(k))
-    end do
-    x(i) = alu(i) * x(i)
-  end do
-
-  return
-end
-
-function ddot ( n, dx, incx, dy, incy )
-
-!*****************************************************************************80
-!
-!! DDOT forms the dot product of two vectors.
-!
-!  Discussion:
-!
-!    This routine uses unrolled loops for increments equal to one.
-!
-!  Author:
-!
-!    Jack Dongarra
-!
-!  Reference:
-!
-!    Dongarra, Moler, Bunch, Stewart,
-!    LINPACK User's Guide,
-!    SIAM, 1979.
-!
-!  Parameters:
-!
-!    Input, integer ( kind = 4 ) N, the number of entries in the vectors.
-!
-!    Input, real ( kind = 8 ) DX(*), the first vector.
-!
-!    Input, integer ( kind = 4 ) INCX, the increment between successive entries in X.
-!
-!    Input, real ( kind = 8 ) DY(*), the second vector.
-!
-!    Input, integer ( kind = 4 ) INCY, the increment between successive entries in Y.
-!
-!    Output, real DDOT, the sum of the product of the corresponding
-!    entries of X and Y.
-!
-  real ( kind = 8 ) ddot
-  real ( kind = 8 ) dtemp
-  real ( kind = 8 ) dx(*)
-  real ( kind = 8 ) dy(*)
-  integer ( kind = 4 ) i
-  integer ( kind = 4 ) incx
-  integer ( kind = 4 ) incy
-  integer ( kind = 4 ) ix
-  integer ( kind = 4 ) iy
-  integer ( kind = 4 ) m
-  integer ( kind = 4 ) n
-
-  ddot = 0.0D+00
-  dtemp = 0.0D+00
-
-  if ( n <= 0 ) then
-    return
-  end if
-!
-!  Code for unequal increments or equal increments
-!  not equal to 1.
-!
-  if ( incx /= 1 .or. incy /= 1 ) then
-
-    if ( 0 <= incx ) then
-      ix = 1
-    else
-      ix = ( - n + 1 ) * incx + 1
-    end if
-
-    if ( 0 <= incy ) then
-      iy = 1
-    else
-      iy = ( - n + 1 ) * incy + 1
-    end if
-
-    do i = 1, n
-      dtemp = dtemp + dx(ix) * dy(iy)
-      ix = ix + incx
-      iy = iy + incy
-    end do
-!
-!  Code for both increments equal to 1.
-!
-  else
-
-    m = mod ( n, 5 )
-
-    do i = 1, m
-      dtemp = dtemp + dx(i) * dy(i)
-    end do
-
-    do i = m+1, n, 5
-
-      dtemp = dtemp + dx(i  ) * dy(i  ) &
-                    + dx(i+1) * dy(i+1) &
-                    + dx(i+2) * dy(i+2) &
-                    + dx(i+3) * dy(i+3) &
-                    + dx(i+4) * dy(i+4)
-    end do
-
-  end if
-
-  ddot = dtemp
-
-  return
-end
-
-end module gmres_solver
diff --git a/src/solver_interface.F90 b/src/solver_interface.F90
index 47bb0c14ab..b00bd33585 100644
--- a/src/solver_interface.F90
+++ b/src/solver_interface.F90
@@ -5,10 +5,6 @@ module solver_interface
   use matrix_header,  only: Matrix
   use vector_header,  only: Vector
 
-#ifndef PETSC
-  use gmres_solver,   only: ilut, pgmres
-#endif
-
   implicit none
   private
 
@@ -21,25 +17,6 @@ module solver_interface
 #ifdef PETSC
     KSP :: ksp
     PC  :: pc
-#else
-    integer :: im      ! iterations before restart
-    integer :: iout    ! unit number for printing gmres iters
-    integer :: maxits  ! maximum iterations
-    integer :: lfil    ! fill in parameter
-    integer :: iwk     ! size of MSR storage
-    integer, allocatable :: ju(:) ! row pointers for ILUT result
-    integer, allocatable :: jlu(:) ! MSR storage of j
-    integer, allocatable :: jr(:) ! temp work array
-    integer, allocatable :: jwu(:) ! temp work array
-    integer, allocatable :: jwl(:) ! temp work array
-    real(8) :: tol     ! gmres tolerance
-    real(8) :: droptol ! ILU drop tolerance
-    real(8), allocatable :: alu(:) ! MSR storage of ILU A
-    real(8), allocatable :: vv(:) ! GMRES work array
-    real(8), allocatable :: wu(:) ! temp work array
-    real(8), allocatable :: wl(:) ! temp work array
-    real(8), allocatable :: rhs(:) ! copy of rhs array
-    type(Matrix), pointer :: A ! coefficient matrix
 #endif
    contains
 #ifdef PETSC
@@ -47,11 +24,6 @@ module solver_interface
      procedure :: set_oper     => petsc_gmres_set_oper
      procedure :: destroy      => petsc_gmres_destroy
      procedure :: solve        => petsc_gmres_solve
-#else
-     procedure :: create       => gmres_create
-     procedure :: set_oper     => gmres_set_oper
-     procedure :: destroy      => gmres_destroy
-     procedure :: solve        => gmres_solve
 #endif
   end type GMRESSolver
 
@@ -76,11 +48,6 @@ module solver_interface
      procedure :: destroy       => petsc_jfnk_destroy
      procedure :: set_functions => petsc_jfnk_set_functions
      procedure :: solve         => petsc_jfnk_solve
-#else
-     procedure :: create        => jfnk_create
-     procedure :: destroy       => jfnk_destroy
-     procedure :: set_functions => jfnk_set_functions
-     procedure :: solve         => jfnk_solve
 #endif
   end type JFNKSolver
 
@@ -104,165 +71,7 @@ module solver_interface
 
 contains
 
-#ifndef PETSC
-!===============================================================================
-! GMRES_CREATE sets up a PETSc GMRES solver
-!===============================================================================
-
-  subroutine gmres_create(self)
-
-    class(GMRESSolver) :: self
-
-    ! Set up default values
-    self % im = 40
-    self % iout = 0
-    self % maxits = 1000
-    self % lfil = 0
-    self % tol = 1.e-10_8
-    self % droptol = 0.0_8
-
-  end subroutine gmres_create
-
-!===============================================================================
-! GMRES_SET_OPER sets the matrix opetors for the GMRES solver
-!===============================================================================
-
-  subroutine gmres_set_oper(self, prec_mat, mat_in)
-
-    class(GMRESSolver) :: self
-    type(Matrix)       :: prec_mat
-    type(Matrix), target :: mat_in
-
-    integer :: ierr
-
-    ! Set pointer to matrix
-    self % A => mat_in
-
-    ! Set up data
-    self % iwk = self % A % nnz + 1
-
-    ! Set up arrays
-    allocate(self % ju(self % A % n))
-    allocate(self % jlu(self % iwk))
-    allocate(self % alu(self % iwk))
-    allocate(self % jr(self % A % n))
-    allocate(self % jwu(self % A % n))
-    allocate(self % jwl(self % A % n))
-    allocate(self % vv(self % A % n * (self % im + 1)))
-    allocate(self % wu(self % A % n + 1))
-    allocate(self % wl(self % A % n))
-    allocate(self % rhs(self % A % n))
-
-    ! Perform ILU Preconditioning
-    call ilut(self % A % n, self % A % val, self % A % col, self % A % row, &
-              self % lfil, self % droptol, self % alu, self % jlu, self % ju, &
-              self % iwk, self % wu, self % wl, self % jr, self % jwl, &
-              self % jwu, ierr)
-
-  end subroutine gmres_set_oper
-
-!===============================================================================
-! GMRES_DESTROY frees all memory associated with the GMRES solver
-!===============================================================================
-
-  subroutine gmres_destroy(self)
-
-   class(GMRESSolver) :: self
-
-   ! Destroy all arrays associated with GMRES solver
-   deallocate(self % ju)
-   deallocate(self % jlu)
-   deallocate(self % alu)
-   deallocate(self % jr)
-   deallocate(self % jwu)
-   deallocate(self % jwl)
-   deallocate(self % vv)
-   deallocate(self % wu)
-   deallocate(self % wl)
-   deallocate(self % rhs)
-
-  end subroutine gmres_destroy
-
-!===============================================================================
-! GMRES_SOLVE solves the linear system
-!===============================================================================
-
-  subroutine gmres_solve(self, b, x)
-
-    class(GMRESSolver) :: self
-    type(Vector)       :: b
-    type(Vector)       :: x
-
-    integer :: ierr ! error code
-
-    ! Copy rhs
-    self % rhs = b % val
-
-    ! Perform GMRES calculation
-    call pgmres(self % A % n, self % im, self % rhs, x % val, self % vv, &
-                self % tol, self % maxits, self % iout, self % A % val, &
-                self % A % col, self % A % row, self % alu, self % jlu, &
-                self % ju, ierr)
-
-  end subroutine gmres_solve
-
-!===============================================================================
-! JFNK_CREATE sets up a JFNK solver using PETSc SNES
-!===============================================================================
-
-  subroutine jfnk_create(self)
-
-    class(JFNKSolver) :: self
-
-    message = 'JFNK cannot be used without PETSc.'
-    call fatal_error()
-
-  end subroutine jfnk_create
-
-!===============================================================================
-! JFNK_DESTROY frees all memory associated with JFNK solver
-!===============================================================================
-
-  subroutine jfnk_destroy(self)
-
-    class(JFNKSolver) :: self
-
-    message = 'JFNK cannot be used without PETSc.'
-    call fatal_error()
-
-  end subroutine jfnk_destroy
-
-!===============================================================================
-! JFNK_SET_FUNCTIONS sets user functions and matrix free objects
-!===============================================================================
-
-  subroutine jfnk_set_functions(self, ctx, res, jac_prec)
-
-    class(JFNKSolver) :: self
-    type(Jfnk_ctx)    :: ctx
-    type(Vector)      :: res
-    type(Matrix)      :: jac_prec
-
-    message = 'JFNK cannot be used without PETSc.'
-    call fatal_error()
-
-  end subroutine jfnk_set_functions
-
-!===============================================================================
-! JFNK_SOLVE solves the nonlinear system
-!===============================================================================
-
-  subroutine jfnk_solve(self, xvec)
-
-    class(JFNKSolver) :: self
-    type(Vector)      :: xvec
-
-    message = 'JFNK cannot be used without PETSc.'
-    call fatal_error()
-
-  end subroutine jfnk_solve
-
-#else
+#ifdef PETSC
 !===============================================================================
 ! PETSC_GMRES_CREATE sets up a PETSc GMRES solver
 !===============================================================================

From d020954a5d82d091a52dbc97f4de5437d511444e Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 9 Sep 2013 08:56:47 -0400
Subject: [PATCH 46/69] converted CMFD output writing to new output interface
 and allowed statepoint python file to read in > 1-D arrays

---
 src/state_point.F90     | 62 +++++++++++++++++++++--------------------
 src/utils/statepoint.py | 33 +++++++++++++---------
 2 files changed, 52 insertions(+), 43 deletions(-)

diff --git a/src/state_point.F90 b/src/state_point.F90
index ceaf9021af..9ad9b8d9f2 100644
--- a/src/state_point.F90
+++ b/src/state_point.F90
@@ -20,6 +20,10 @@ module state_point
   use output_interface
   use tally_header,       only: TallyObject
 
+#ifdef MPI
+  use mpi
+#endif
+
   implicit none
 
   type(BinaryOutput) :: sp ! statepoint/source output file
@@ -91,32 +95,32 @@ contains
         call sp % write_data(gen_per_batch, "gen_per_batch")
         call sp % write_data(k_generation, "k_generation", &
              length=current_batch*gen_per_batch)
-        call write_data(entropy, "entropy", length=current_batch*gen_per_batch)
-        call write_data(k_col_abs, "k_col_abs")
-        call write_data(k_col_tra, "k_col_tra")
-        call write_data(k_abs_tra, "k_abs_tra")
-        call write_data(k_combined, "k_combined", length=2)
+        call sp % write_data(entropy, "entropy", length=current_batch*gen_per_batch)
+        call sp % write_data(k_col_abs, "k_col_abs")
+        call sp % write_data(k_col_tra, "k_col_tra")
+        call sp % write_data(k_abs_tra, "k_abs_tra")
+        call sp % write_data(k_combined, "k_combined", length=2)
 
         ! Write out CMFD info
         if (cmfd_on) then
-          call write_data(1, "cmfd_on")
-          call write_data(cmfd % indices, "indicies", length=4, group="cmfd")
-          call write_data(cmfd % k_cmfd, "k_cmfd", length=current_batch, &
+          call sp % write_data(1, "cmfd_on")
+          call sp % write_data(cmfd % indices, "indicies", length=4, group="cmfd")
+          call sp % write_data(cmfd % k_cmfd, "k_cmfd", length=current_batch, &
                group="cmfd")
-          call write_data(cmfd % cmfd_src, "cmfd_src", &
+          call sp % write_data(cmfd % cmfd_src, "cmfd_src", &
                length=(/cmfd % indices(4), cmfd % indices(1), &
                cmfd % indices(2), cmfd % indices(3)/), &
                group="cmfd")
-          call write_data(cmfd % entropy, "cmfd_entropy", &
+          call sp % write_data(cmfd % entropy, "cmfd_entropy", &
                           length=current_batch, group="cmfd")
-          call write_data(cmfd % balance, "cmfd_balance", &
+          call sp % write_data(cmfd % balance, "cmfd_balance", &
                length=current_batch, group="cmfd")
-          call write_data(cmfd % dom, "cmfd_dominance", &
+          call sp % write_data(cmfd % dom, "cmfd_dominance", &
                length = current_batch, group="cmfd")
-          call write_data(cmfd % src_cmp, "cmfd_srccmp", &
+          call sp % write_data(cmfd % src_cmp, "cmfd_srccmp", &
                length = current_batch, group="cmfd")
         else
-          call write_data(0, "cmfd_on")
+          call sp % write_data(0, "cmfd_on")
         end if
       end if
 
@@ -531,27 +535,25 @@ contains
       n_inactive = max(n_inactive, int_array(1))
 
       ! Read in to see if CMFD was on
-      call read_data(int_array(1), "cmfd_on", option="collective")
+      call sp % read_data(int_array(1), "cmfd_on")
 
       ! Write out CMFD info
       if (int_array(1) == 1) then
-        call read_data(cmfd % indices, "indicies", length=4, group="cmfd", &
-                       option="collective")
-        call read_data(cmfd % k_cmfd, "k_cmfd", length=restart_batch, &
-             group="cmfd", option="collective")
-        call read_data(cmfd % cmfd_src, "cmfd_src", &
+        call sp % read_data(cmfd % indices, "indicies", length=4, group="cmfd")
+        call sp % read_data(cmfd % k_cmfd, "k_cmfd", length=restart_batch, &
+             group="cmfd")
+        call sp % read_data(cmfd % cmfd_src, "cmfd_src", &
              length=(/cmfd % indices(4), cmfd % indices(1), &
              cmfd % indices(2), cmfd % indices(3)/), &
-             group="cmfd", option="collective")
-        call read_data(cmfd % entropy, "cmfd_entropy", &
-                       length=restart_batch, group="cmfd", &
-                          option="collective")
-        call read_data(cmfd % balance, "cmfd_balance", &
-             length=restart_batch, group="cmfd", option="collective")
-        call read_data(cmfd % dom, "cmfd_dominance", &
-             length = restart_batch, group="cmfd", option="collective")
-        call read_data(cmfd % src_cmp, "cmfd_srccmp", &
-             length = restart_batch, group="cmfd", option="collective")
+             group="cmfd")
+        call sp % read_data(cmfd % entropy, "cmfd_entropy", &
+                       length=restart_batch, group="cmfd")
+        call sp % read_data(cmfd % balance, "cmfd_balance", &
+             length=restart_batch, group="cmfd")
+        call sp % read_data(cmfd % dom, "cmfd_dominance", &
+             length = restart_batch, group="cmfd")
+        call sp % read_data(cmfd % src_cmp, "cmfd_srccmp", &
+             length = restart_batch, group="cmfd")
       end if
     end if
 
diff --git a/src/utils/statepoint.py b/src/utils/statepoint.py
index ec43435b91..5698506c92 100644
--- a/src/utils/statepoint.py
+++ b/src/utils/statepoint.py
@@ -152,6 +152,8 @@ class StatePoint(object):
 
         # Read statepoint revision
         self.revision = self._get_int(path='revision')[0]
+        if self.revision != 10:
+          raise Exception('Statepoint Revision is not consistent.')
 
         # Read OpenMC version
         if self._hdf5:
@@ -186,28 +188,27 @@ class StatePoint(object):
                 self.current_batch*self.gen_per_batch, path='k_generation')
             self.entropy = self._get_double(
                 self.current_batch*self.gen_per_batch, path='entropy')
-            if self.revision >= 8:
-                self.k_col_abs = self._get_double(path='k_col_abs')[0]
-                self.k_col_tra = self._get_double(path='k_col_tra')[0]
-                self.k_abs_tra = self._get_double(path='k_abs_tra')[0]
-                self.k_combined = self._get_double(2, path='k_combined')
+            self.k_col_abs = self._get_double(path='k_col_abs')[0]
+            self.k_col_tra = self._get_double(path='k_col_tra')[0]
+            self.k_abs_tra = self._get_double(path='k_abs_tra')[0]
+            self.k_combined = self._get_double(2, path='k_combined')
 
             # Read CMFD information
             cmfd_present = self._get_int(path='cmfd_on')[0]
             if cmfd_present == 1:
-                self.cmfd_indices = self._get_double(4, path='cmfd/indicies')
+                self.cmfd_indices = self._get_int(4, path='cmfd/indicies')
                 self.k_cmfd = self._get_double(self.current_batch,
-                                               path='cmfd/k_cmfd')
-                self.cmfd_src = self._get_double(np.product(self.cmfd_indices,
-                                                 path='cmfd/cmfd_src'))
+                              path='cmfd/k_cmfd')
+                self.cmfd_src = self._get_double(np.product(self.cmfd_indices),
+                                path='cmfd/cmfd_src')
                 self.cmfd_entropy = self._get_double(self.current_batch,
-                                                     path='cmfd/cmfd_entropy')
+                                    path='cmfd/cmfd_entropy')
                 self.cmfd_balance = self._get_double(self.current_batch,
-                                                     path='cmfd/cmfd_balance')
+                                    path='cmfd/cmfd_balance')
                 self.cmfd_dominance = self._get_double(self.current_batch,
-                                                       path='cmfd/cmfd_dominance')
+                                      path='cmfd/cmfd_dominance')
                 self.cmfd_srccmp = self._get_double(self.current_batch,
-                                                    path='cmfd/cmfd_srccmp')
+                                   path='cmfd/cmfd_srccmp')
 
         # Read number of meshes
         n_meshes = self._get_int(path='tallies/n_meshes')[0]
@@ -597,6 +598,12 @@ class StatePoint(object):
         else:
             return [float(v) for v in self._get_data(n, 'd', 8)]
 
+    def _get_double(self, n=1, path=None):
+        if self._hdf5:
+            return self._f[path].value
+        else:
+            return self._get_data(n, 'd', 8)
+
     def _get_string(self, n=1, path=None):
         if self._hdf5:
             return str(self._f[path].value)

From 576b45ef23c99bc14005fbb7b50f3a1c87224946 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 9 Sep 2013 10:00:17 -0400
Subject: [PATCH 47/69] updated statepoint utility and cmfd results to handle
 printing more detalied cmfd results

---
 src/utils/statepoint.py                 |   6 +-
 tests/test_cmfd_feed/results.py         |  24 ++
 tests/test_cmfd_feed/results_true.dat   | 317 ++++++++++++++++--------
 tests/test_cmfd_nofeed/results.py       |  24 ++
 tests/test_cmfd_nofeed/results_true.dat | 121 +++++++++
 5 files changed, 392 insertions(+), 100 deletions(-)

diff --git a/src/utils/statepoint.py b/src/utils/statepoint.py
index 5698506c92..96b98ab671 100644
--- a/src/utils/statepoint.py
+++ b/src/utils/statepoint.py
@@ -199,8 +199,10 @@ class StatePoint(object):
                 self.cmfd_indices = self._get_int(4, path='cmfd/indicies')
                 self.k_cmfd = self._get_double(self.current_batch,
                               path='cmfd/k_cmfd')
-                self.cmfd_src = self._get_double(np.product(self.cmfd_indices),
+                self.cmfd_src = self._get_double_array(np.product(self.cmfd_indices),
                                 path='cmfd/cmfd_src')
+                self.cmfd_src = np.reshape(self.cmfd_src,
+                                tuple(self.cmfd_indices), order='F')
                 self.cmfd_entropy = self._get_double(self.current_batch,
                                     path='cmfd/cmfd_entropy')
                 self.cmfd_balance = self._get_double(self.current_batch,
@@ -598,7 +600,7 @@ class StatePoint(object):
         else:
             return [float(v) for v in self._get_data(n, 'd', 8)]
 
-    def _get_double(self, n=1, path=None):
+    def _get_double_array(self, n=1, path=None):
         if self._hdf5:
             return self._f[path].value
         else:
diff --git a/tests/test_cmfd_feed/results.py b/tests/test_cmfd_feed/results.py
index 533a624558..67bbe6ca4f 100644
--- a/tests/test_cmfd_feed/results.py
+++ b/tests/test_cmfd_feed/results.py
@@ -53,6 +53,30 @@ outstr += 'tally 4:\n'
 for item in results4:
   outstr += "{0:12.6E}\n".format(item)
 
+# write out cmfd answers
+outstr += 'cmfd indices\n'
+for item in sp.cmfd_indices:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'k cmfd\n'
+for item in sp.k_cmfd:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd entropy\n'
+for item in sp.cmfd_entropy:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd balance\n'
+for item in sp.cmfd_balance:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd dominance ratio\n'
+for item in sp.cmfd_dominance:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd openmc source comparison\n'
+for item in sp.cmfd_srccmp:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd source\n'
+cmfdsrc = np.reshape(sp.cmfd_src, np.product(sp.cmfd_indices), order='F')
+for item in cmfdsrc:
+  outstr += "{0:12.6E}\n".format(item)
+
 # write results to file
 with open('results_test.dat','w') as fh:
     fh.write(outstr)
diff --git a/tests/test_cmfd_feed/results_true.dat b/tests/test_cmfd_feed/results_true.dat
index ad4c85c761..20ed778cd2 100644
--- a/tests/test_cmfd_feed/results_true.dat
+++ b/tests/test_cmfd_feed/results_true.dat
@@ -1,14 +1,14 @@
 k-combined:
-1.167124E+00 1.217343E-02
+1.167124E+00 1.217344E-02
 tally 1:
 1.126891E+01
-1.275755E+01
-2.086556E+01
-4.369501E+01
+1.275756E+01
+2.086557E+01
+4.369504E+01
 2.853629E+01
-8.169559E+01
-3.404506E+01
-1.165019E+02
+8.169563E+01
+3.404507E+01
+1.165020E+02
 3.723908E+01
 1.389881E+02
 3.760048E+01
@@ -16,86 +16,86 @@ tally 1:
 3.455469E+01
 1.197186E+02
 2.837503E+01
-8.086291E+01
-2.151789E+01
-4.648315E+01
+8.086287E+01
+2.151788E+01
+4.648312E+01
 1.194541E+01
-1.432331E+01
+1.432330E+01
 tally 2:
-2.292173E+01
-2.650792E+01
-1.598183E+01
-1.288692E+01
-2.194143E+00
-2.510557E-01
-4.217816E+01
-8.938583E+01
+2.292174E+01
+2.650794E+01
+1.598184E+01
+1.288693E+01
+2.194144E+00
+2.510560E-01
+4.217817E+01
+8.938588E+01
 2.974466E+01
-4.447038E+01
-3.939269E+00
-7.882413E-01
-5.742289E+01
+4.447040E+01
+3.939271E+00
+7.882419E-01
+5.742290E+01
 1.654599E+02
-4.067581E+01
-8.308454E+01
-5.566955E+00
-1.570072E+00
-6.810586E+01
+4.067582E+01
+8.308457E+01
+5.566956E+00
+1.570073E+00
+6.810587E+01
 2.331019E+02
-4.831543E+01
+4.831544E+01
 1.174673E+02
-6.244999E+00
-1.967938E+00
+6.245000E+00
+1.967939E+00
 7.258131E+01
 2.646756E+02
 5.185146E+01
-1.351425E+02
-6.622488E+00
+1.351426E+02
+6.622489E+00
 2.231695E+00
 7.246115E+01
 2.641295E+02
-5.159542E+01
+5.159541E+01
 1.339441E+02
 6.702239E+00
 2.277825E+00
-6.716397E+01
+6.716396E+01
 2.266815E+02
-4.765063E+01
-1.141627E+02
-6.384098E+00
+4.765062E+01
+1.141626E+02
+6.384097E+00
 2.063760E+00
-5.549542E+01
-1.545361E+02
+5.549541E+01
+1.545360E+02
 3.940819E+01
-7.794880E+01
-5.183375E+00
-1.369483E+00
-4.155685E+01
-8.679439E+01
-2.926744E+01
-4.307145E+01
-3.938852E+00
-7.877949E-01
-2.335180E+01
-2.758241E+01
-1.622683E+01
-1.331996E+01
-2.248921E+00
-2.594934E-01
+7.794877E+01
+5.183373E+00
+1.369482E+00
+4.155684E+01
+8.679434E+01
+2.926743E+01
+4.307143E+01
+3.938851E+00
+7.877944E-01
+2.335179E+01
+2.758239E+01
+1.622682E+01
+1.331995E+01
+2.248920E+00
+2.594933E-01
 tally 3:
 1.537807E+01
 1.193746E+01
 1.044189E+00
 5.627797E-02
 2.861614E+01
-4.118322E+01
+4.118324E+01
 1.846208E+00
 1.729007E-01
-3.921462E+01
-7.723576E+01
+3.921463E+01
+7.723579E+01
 2.427723E+00
 2.983153E-01
-4.653207E+01
+4.653208E+01
 1.089818E+02
 3.128394E+00
 4.945409E-01
@@ -111,15 +111,15 @@ tally 3:
 1.061019E+02
 2.936424E+00
 4.356307E-01
-3.783939E+01
-7.188489E+01
+3.783938E+01
+7.188486E+01
 2.485680E+00
 3.117618E-01
 2.819358E+01
-3.997588E+01
+3.997585E+01
 1.875908E+00
 1.772000E-01
-1.562421E+01
+1.562420E+01
 1.236020E+01
 1.013140E+00
 5.260420E-02
@@ -160,8 +160,8 @@ tally 4:
 0.000000E+00
 0.000000E+00
 0.000000E+00
-3.126365E+00
-4.930682E-01
+3.126366E+00
+4.930686E-01
 0.000000E+00
 0.000000E+00
 0.000000E+00
@@ -208,10 +208,10 @@ tally 4:
 0.000000E+00
 0.000000E+00
 0.000000E+00
-5.456410E+00
-1.499416E+00
-2.727755E+00
-3.788867E-01
+5.456412E+00
+1.499417E+00
+2.727756E+00
+3.788869E-01
 0.000000E+00
 0.000000E+00
 0.000000E+00
@@ -256,10 +256,10 @@ tally 4:
 0.000000E+00
 0.000000E+00
 0.000000E+00
-7.453480E+00
-2.790159E+00
-5.240103E+00
-1.386895E+00
+7.453481E+00
+2.790160E+00
+5.240104E+00
+1.386896E+00
 0.000000E+00
 0.000000E+00
 0.000000E+00
@@ -304,10 +304,10 @@ tally 4:
 0.000000E+00
 0.000000E+00
 0.000000E+00
-8.741563E+00
-3.830225E+00
-7.137968E+00
-2.555042E+00
+8.741565E+00
+3.830226E+00
+7.137969E+00
+2.555043E+00
 0.000000E+00
 0.000000E+00
 0.000000E+00
@@ -354,8 +354,8 @@ tally 4:
 0.000000E+00
 9.221731E+00
 4.266573E+00
-8.474116E+00
-3.604647E+00
+8.474118E+00
+3.604648E+00
 0.000000E+00
 0.000000E+00
 0.000000E+00
@@ -401,9 +401,9 @@ tally 4:
 0.000000E+00
 0.000000E+00
 9.054937E+00
-4.124537E+00
-9.066119E+00
-4.133833E+00
+4.124536E+00
+9.066120E+00
+4.133834E+00
 0.000000E+00
 0.000000E+00
 0.000000E+00
@@ -448,8 +448,8 @@ tally 4:
 0.000000E+00
 0.000000E+00
 0.000000E+00
-8.362253E+00
-3.517245E+00
+8.362251E+00
+3.517244E+00
 9.086548E+00
 4.156118E+00
 0.000000E+00
@@ -496,10 +496,10 @@ tally 4:
 0.000000E+00
 0.000000E+00
 0.000000E+00
-6.871573E+00
-2.369190E+00
-8.399584E+00
-3.549784E+00
+6.871571E+00
+2.369189E+00
+8.399583E+00
+3.549783E+00
 0.000000E+00
 0.000000E+00
 0.000000E+00
@@ -544,10 +544,10 @@ tally 4:
 0.000000E+00
 0.000000E+00
 0.000000E+00
-4.998103E+00
-1.258194E+00
-7.145296E+00
-2.568115E+00
+4.998101E+00
+1.258193E+00
+7.145294E+00
+2.568114E+00
 0.000000E+00
 0.000000E+00
 0.000000E+00
@@ -592,10 +592,10 @@ tally 4:
 0.000000E+00
 0.000000E+00
 0.000000E+00
-2.716076E+00
-3.753568E-01
-5.427593E+00
-1.484154E+00
+2.716075E+00
+3.753566E-01
+5.427591E+00
+1.484153E+00
 0.000000E+00
 0.000000E+00
 0.000000E+00
@@ -642,8 +642,8 @@ tally 4:
 0.000000E+00
 0.000000E+00
 0.000000E+00
-3.056645E+00
-4.702349E-01
+3.056644E+00
+4.702345E-01
 0.000000E+00
 0.000000E+00
 0.000000E+00
@@ -652,3 +652,124 @@ tally 4:
 0.000000E+00
 0.000000E+00
 0.000000E+00
+cmfd indices
+1.000000E+01
+1.000000E+00
+1.000000E+00
+1.000000E+00
+k cmfd
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+1.158665E+00
+1.173348E+00
+1.180514E+00
+1.168646E+00
+1.152485E+00
+1.154989E+00
+1.152285E+00
+1.152275E+00
+1.149476E+00
+1.154912E+00
+1.164393E+00
+1.170145E+00
+1.169161E+00
+1.167966E+00
+1.170153E+00
+1.170989E+00
+cmfd entropy
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+3.229801E+00
+3.226763E+00
+3.225011E+00
+3.230316E+00
+3.232043E+00
+3.228854E+00
+3.227439E+00
+3.228206E+00
+3.226749E+00
+3.222635E+00
+3.218882E+00
+3.219196E+00
+3.218057E+00
+3.220324E+00
+3.219099E+00
+3.219847E+00
+cmfd balance
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+6.244292E-03
+5.210436E-03
+4.212224E-03
+3.321578E-03
+3.067254E-03
+3.076634E-03
+2.601637E-03
+2.666441E-03
+2.490817E-03
+2.571411E-03
+3.013819E-03
+2.802836E-03
+2.725356E-03
+2.475281E-03
+2.279123E-03
+2.110816E-03
+cmfd dominance ratio
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+5.509393E-01
+5.472954E-01
+5.461377E-01
+5.518823E-01
+5.540420E-01
+5.513822E-01
+5.505071E-01
+5.498933E-01
+5.486126E-01
+5.450413E-01
+5.437663E-01
+5.421995E-01
+5.417191E-01
+5.437232E-01
+5.429525E-01
+5.428436E-01
+cmfd openmc source comparison
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+1.136193E-02
+1.059214E-02
+9.675100E-03
+7.296572E-03
+5.812484E-03
+6.138879E-03
+6.070297E-03
+5.791570E-03
+6.020032E-03
+7.287183E-03
+7.068423E-03
+6.380521E-03
+5.828010E-03
+4.741191E-03
+4.848114E-03
+4.686772E-03
+cmfd source
+4.367444E-02
+7.866919E-02
+1.050799E-01
+1.366581E-01
+1.421824E-01
+1.393453E-01
+1.269612E-01
+1.060419E-01
+7.920472E-02
+4.218277E-02
diff --git a/tests/test_cmfd_nofeed/results.py b/tests/test_cmfd_nofeed/results.py
index 533a624558..67bbe6ca4f 100644
--- a/tests/test_cmfd_nofeed/results.py
+++ b/tests/test_cmfd_nofeed/results.py
@@ -53,6 +53,30 @@ outstr += 'tally 4:\n'
 for item in results4:
   outstr += "{0:12.6E}\n".format(item)
 
+# write out cmfd answers
+outstr += 'cmfd indices\n'
+for item in sp.cmfd_indices:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'k cmfd\n'
+for item in sp.k_cmfd:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd entropy\n'
+for item in sp.cmfd_entropy:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd balance\n'
+for item in sp.cmfd_balance:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd dominance ratio\n'
+for item in sp.cmfd_dominance:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd openmc source comparison\n'
+for item in sp.cmfd_srccmp:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd source\n'
+cmfdsrc = np.reshape(sp.cmfd_src, np.product(sp.cmfd_indices), order='F')
+for item in cmfdsrc:
+  outstr += "{0:12.6E}\n".format(item)
+
 # write results to file
 with open('results_test.dat','w') as fh:
     fh.write(outstr)
diff --git a/tests/test_cmfd_nofeed/results_true.dat b/tests/test_cmfd_nofeed/results_true.dat
index 6f00290374..be594a3fba 100644
--- a/tests/test_cmfd_nofeed/results_true.dat
+++ b/tests/test_cmfd_nofeed/results_true.dat
@@ -652,3 +652,124 @@ tally 4:
 0.000000E+00
 0.000000E+00
 0.000000E+00
+cmfd indices
+1.000000E+01
+1.000000E+00
+1.000000E+00
+1.000000E+00
+k cmfd
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+1.158665E+00
+1.177323E+00
+1.179341E+00
+1.169660E+00
+1.179827E+00
+1.180220E+00
+1.177699E+00
+1.179321E+00
+1.180728E+00
+1.180996E+00
+1.186680E+00
+1.181455E+00
+1.178393E+00
+1.176470E+00
+1.172593E+00
+1.167042E+00
+cmfd entropy
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+3.229801E+00
+3.227279E+00
+3.224715E+00
+3.226705E+00
+3.226404E+00
+3.227329E+00
+3.227407E+00
+3.228601E+00
+3.230423E+00
+3.228093E+00
+3.226358E+00
+3.226226E+00
+3.227324E+00
+3.228594E+00
+3.229513E+00
+3.229820E+00
+cmfd balance
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+6.244292E-03
+5.300885E-03
+5.198276E-03
+3.759109E-03
+3.994559E-03
+3.064086E-03
+2.749389E-03
+2.388483E-03
+2.304428E-03
+2.188627E-03
+2.269076E-03
+2.016776E-03
+1.622043E-03
+1.470984E-03
+1.367144E-03
+1.402525E-03
+cmfd dominance ratio
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+5.509393E-01
+5.473805E-01
+5.434034E-01
+5.469278E-01
+5.460554E-01
+5.474063E-01
+5.486271E-01
+5.485095E-01
+5.490619E-01
+5.473524E-01
+5.466605E-01
+5.471154E-01
+5.480465E-01
+5.485550E-01
+5.500141E-01
+5.502154E-01
+cmfd openmc source comparison
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+1.136193E-02
+1.007349E-02
+8.776995E-03
+6.729971E-03
+6.921055E-03
+5.867589E-03
+4.366276E-03
+4.051899E-03
+3.406846E-03
+4.043983E-03
+4.864903E-03
+3.711687E-03
+3.124413E-03
+2.675579E-03
+2.523549E-03
+2.724884E-03
+cmfd source
+4.534769E-02
+8.593793E-02
+1.059054E-01
+1.317554E-01
+1.358070E-01
+1.352224E-01
+1.290227E-01
+1.093443E-01
+7.943056E-02
+4.222668E-02

From 6e77710d308bb05fdf35a96ed65faa40732a3357 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 9 Sep 2013 14:06:32 -0400
Subject: [PATCH 48/69] changed PETSc includes to modules

---
 src/cmfd_input.F90       |   8 +--
 src/matrix_header.F90    |  12 ++---
 src/solver_interface.F90 | 103 ++++++++++++++++++++-------------------
 src/vector_header.F90    |  12 ++---
 4 files changed, 68 insertions(+), 67 deletions(-)

diff --git a/src/cmfd_input.F90 b/src/cmfd_input.F90
index de23b89b09..9a768f4e3b 100644
--- a/src/cmfd_input.F90
+++ b/src/cmfd_input.F90
@@ -2,14 +2,14 @@ module cmfd_input
 
   use global
 
+#ifdef PETSC
+  use petscsys
+#endif
+
   implicit none
   private
   public :: configure_cmfd 
 
-# ifdef PETSC
-#  include <finclude/petsc.h90>
-# endif
-
 contains
 
 !===============================================================================
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index c034c8d2e7..25f1df95a5 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -1,12 +1,12 @@
 module matrix_header
 
+#ifdef PETSC
+  use petscmat
+#endif
+
   implicit none
   private
 
-# ifdef PETSC
-#  include <finclude/petsc.h90>
-# endif
-
   type, public :: Matrix
     integer :: n        ! number of rows/cols in matrix
     integer :: nnz      ! number of nonzeros in matrix
@@ -16,7 +16,7 @@ module matrix_header
     integer, allocatable :: col(:) ! column vector
     real(8), allocatable :: val(:) ! matrix value vector
 #  ifdef PETSC
-    Mat :: petsc_mat
+    type(mat) :: petsc_mat
 #  endif
     logical :: petsc_active
    contains
@@ -296,7 +296,7 @@ contains
     class(Matrix) :: self
 
 #ifdef PETSC
-    PetscViewer :: viewer
+    type(PetscViewer) :: viewer
 
     call PetscViewerBinaryOpen(PETSC_COMM_WORLD, trim(filename), &
          FILE_MODE_WRITE, viewer, petsc_err)
diff --git a/src/solver_interface.F90 b/src/solver_interface.F90
index b00bd33585..a937057d6b 100644
--- a/src/solver_interface.F90
+++ b/src/solver_interface.F90
@@ -5,18 +5,19 @@ module solver_interface
   use matrix_header,  only: Matrix
   use vector_header,  only: Vector
 
+#ifdef PETSC
+  use petscksp
+  use petscsnes
+#endif
+
   implicit none
   private
 
-#ifdef PETSC
-#  include <finclude/petsc.h90>
-#endif
-
   ! GMRES solver type 
   type, public :: GMRESSolver 
 #ifdef PETSC
-    KSP :: ksp
-    PC  :: pc
+    type(ksp) :: ksp_
+    type(pc)  :: pc_
 #endif
    contains
 #ifdef PETSC
@@ -36,11 +37,11 @@ module solver_interface
   ! JFNK solver type 
   type, public :: JFNKSolver 
 #ifdef PETSC
-    KSP  :: ksp
-    PC   :: pc
-    SNES :: snes
-    Mat  :: jac_mf
-    SNESLineSearch :: ls
+    type(ksp)  :: ksp_
+    type(pc)   :: pc_
+    type(snes) :: snes_
+    type(mat)  :: jac_mf
+    integer    :: ls
 #endif
    contains
 #ifdef PETSC
@@ -84,14 +85,14 @@ contains
     real(8) :: rtol = 1.0e-10_8
     real(8) :: atol = 1.0e-10_8
 
-    call KSPCreate(PETSC_COMM_WORLD, self % ksp, petsc_err)
-    call KSPSetTolerances(self % ksp, rtol, atol, &
+    call KSPCreate(PETSC_COMM_WORLD, self % ksp_, petsc_err)
+    call KSPSetTolerances(self % ksp_, rtol, atol, &
          PETSC_DEFAULT_DOUBLE_PRECISION, PETSC_DEFAULT_INTEGER, petsc_err)
-    call KSPSetType(self % ksp, KSPGMRES, petsc_err)
-    call KSPSetInitialGuessNonzero(self % ksp, PETSC_TRUE, petsc_err)
-    call KSPGetPC(self % ksp, self % pc, petsc_err)
-    call PCFactorSetLevels(self % pc, ilu_levels, petsc_err)
-    call KSPSetFromOptions(self % ksp, petsc_err)
+    call KSPSetType(self % ksp_, 'gmres', petsc_err)
+    call KSPSetInitialGuessNonzero(self % ksp_, PETSC_TRUE, petsc_err)
+    call KSPGetPC(self % ksp_, self % pc_, petsc_err)
+    call PCFactorSetLevels(self % pc_, ilu_levels, petsc_err)
+    call KSPSetFromOptions(self % ksp_, petsc_err)
 
   end subroutine petsc_gmres_create
 
@@ -105,9 +106,9 @@ contains
     type(Matrix)       :: prec_mat
     type(Matrix)       :: mat_in
 
-    call KSPSetOperators(self % ksp, mat_in % petsc_mat, prec_mat % petsc_mat, &
+    call KSPSetOperators(self % ksp_, mat_in % petsc_mat, prec_mat % petsc_mat, &
          SAME_NONZERO_PATTERN, petsc_err)
-    call KSPSetUp(self % ksp, petsc_err) 
+    call KSPSetUp(self % ksp_, petsc_err) 
 
   end subroutine petsc_gmres_set_oper
 
@@ -119,7 +120,7 @@ contains
 
    class(GMRESSolver) :: self
 
-    call KSPDestroy(self % ksp, petsc_err)
+    call KSPDestroy(self % ksp_, petsc_err)
 
   end subroutine petsc_gmres_destroy
 
@@ -133,7 +134,7 @@ contains
     type(Vector)       :: b
     type(Vector)       :: x
 
-    call KSPSolve(self % ksp, b % petsc_vec, x % petsc_vec, petsc_err)
+    call KSPSolve(self % ksp_, b % petsc_vec, x % petsc_vec, petsc_err)
 
   end subroutine petsc_gmres_solve
 
@@ -149,23 +150,23 @@ contains
     call PetscOptionsSetValue("-snes_mf_operator", "TRUE", petsc_err)
 
     ! Create the SNES context
-    call SNESCreate(PETSC_COMM_WORLD, self % snes, petsc_err)
+    call SNESCreate(PETSC_COMM_WORLD, self % snes_, petsc_err)
 
     ! Set up the GMRES solver
-    call SNESGetKSP(self % snes, self % ksp, petsc_err)
-    call KSPSetType(self % ksp, KSPGMRES, petsc_err)    
+    call SNESGetKSP(self % snes_, self % ksp_, petsc_err)
+    call KSPSetType(self % ksp_, 'gmres', petsc_err)    
 
     ! Apply options
-    call SNESGetLineSearch(self % snes, self % ls, petsc_err)
-    call SNESLineSearchSetType(self % ls, SNESLINESEARCHBASIC, petsc_err)
-    call SNESSetFromOptions(self % snes, petsc_err)
+    call SNESGetLineSearch(self % snes_, self % ls, petsc_err)
+    call SNESLineSearchSetType(self % ls, 'basic', petsc_err)
+    call SNESSetFromOptions(self % snes_, petsc_err)
 
     ! Set up preconditioner
-    call KSPGetPC(self % ksp, self % pc, petsc_err)
-    call PCSetType(self % pc, PCILU, petsc_err)
-    call PCFactorSetLevels(self % pc, 5, petsc_err)
-    call PCSetFromOptions(self % pc, petsc_err)
-    call KSPSetFromOptions(self % ksp, petsc_err)
+    call KSPGetPC(self % ksp_, self % pc_, petsc_err)
+    call PCSetType(self % pc_, 'ilu', petsc_err)
+    call PCFactorSetLevels(self % pc_, 5, petsc_err)
+    call PCSetFromOptions(self % pc_, petsc_err)
+    call KSPSetFromOptions(self % ksp_, petsc_err)
 
   end subroutine petsc_jfnk_create
 
@@ -178,7 +179,7 @@ contains
     class(JFNKSolver) :: self
 
     call MatDestroy(self % jac_mf, petsc_err)
-    call SNESDestroy(self % snes, petsc_err)
+    call SNESDestroy(self % snes_, petsc_err)
 
   end subroutine petsc_jfnk_destroy
 
@@ -194,19 +195,19 @@ contains
     type(Matrix)      :: jac_prec
 
     ! Set residual procedure
-    call SNESSetFunction(self % snes, res % petsc_vec, &
+    call SNESSetFunction(self % snes_, res % petsc_vec, &
          petsc_jfnk_compute_residual, ctx, petsc_err)
 
     ! Create the matrix free jacobian
-    call MatCreateSNESMF(self % snes, self % jac_mf, petsc_err)
+    call MatCreateSNESMF(self % snes_, self % jac_mf, petsc_err)
 
     ! Set Jacobian procedure
-    call SNESSetJacobian(self % snes, self % jac_mf, jac_prec % petsc_mat, &
+    call SNESSetJacobian(self % snes_, self % jac_mf, jac_prec % petsc_mat, &
          petsc_jfnk_compute_jacobian, ctx, petsc_err)
 
     ! Set up Jacobian Lags
-    call SNESSetLagJacobian(self % snes, -2, petsc_err)
-    call SNESSetLagPreconditioner(self % snes, -1, petsc_err)
+    call SNESSetLagJacobian(self % snes_, -2, petsc_err)
+    call SNESSetLagPreconditioner(self % snes_, -1, petsc_err)
 
   end subroutine petsc_jfnk_set_functions
 
@@ -219,7 +220,7 @@ contains
     class(JFNKSolver) :: self
     type(Vector)      :: xvec
 
-    call SNESSolve(self % snes, PETSC_NULL_DOUBLE, xvec % petsc_vec, petsc_err)
+    call SNESSolve(self % snes_, PETSC_NULL_DOUBLE, xvec % petsc_vec, petsc_err)
 
   end subroutine petsc_jfnk_solve
 
@@ -227,11 +228,11 @@ contains
 ! PETSC_JFNK_COMPUTE_RESIDUAL buffer routine to user specifed residual routine
 !===============================================================================
 
-  subroutine petsc_jfnk_compute_residual(snes, x, res, ctx, ierr)
+  subroutine petsc_jfnk_compute_residual(snes_, x, res, ctx, ierr)
 
-    SNES           :: snes
-    Vec            :: x
-    Vec            :: res
+    type(snes)     :: snes_
+    type(vec)      :: x
+    type(vec)      :: res
     integer        :: ierr
     type(Jfnk_ctx) :: ctx
 
@@ -259,14 +260,14 @@ contains
 ! PETSC_JFNK_COMPUTE_JACOBIAN buffer routine to user specified jacobian routine
 !===============================================================================
 
-  subroutine petsc_jfnk_compute_jacobian(snes, x, jac_mf, jac_prec, flag, ctx, &
-             ierr)
+  subroutine petsc_jfnk_compute_jacobian(snes_, x, jac_mf, jac_prec, flag, &
+             ctx, ierr)
 
-    SNES           :: snes
-    Vec            :: x
-    Mat            :: jac_mf
-    Mat            :: jac_prec
-    MatStructure   :: flag 
+    type(snes)     :: snes_
+    type(vec)      :: x
+    type(mat)      :: jac_mf
+    type(mat)      :: jac_prec
+    integer        :: flag 
     type(Jfnk_ctx) :: ctx
     integer        :: ierr
 
diff --git a/src/vector_header.F90 b/src/vector_header.F90
index b91f926b07..47b865ebec 100644
--- a/src/vector_header.F90
+++ b/src/vector_header.F90
@@ -2,19 +2,19 @@ module vector_header
 
   use constants,  only: ZERO
 
+#ifdef PETSC
+  use petscvec
+#endif
+
   implicit none
   private
 
-# ifdef PETSC
-#  include <finclude/petsc.h90>
-# endif
-
   type, public :: Vector 
     integer :: n        ! number of rows/cols in matrix
     real(8), allocatable :: data(:) ! where vector data is stored
     real(8), pointer :: val(:) ! pointer to vector data
 #  ifdef PETSC
-    Vec :: petsc_vec
+    type(vec) :: petsc_vec
 #  endif
     logical :: petsc_active
    contains
@@ -113,7 +113,7 @@ contains
     class(Vector) :: self
 
 #ifdef PETSC
-    PetscViewer :: viewer
+    type(PetscViewer) :: viewer
 
     call PetscViewerBinaryOpen(PETSC_COMM_WORLD, trim(filename), &
          FILE_MODE_WRITE, viewer, petsc_err)

From 8500207b8429fceea5cf294bb765a16691fe133a Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 9 Sep 2013 14:07:12 -0400
Subject: [PATCH 49/69] dominance ratio precision is loosened due to some
 precision difference when running with MPI

---
 tests/test_cmfd_feed/results.py         |  2 +-
 tests/test_cmfd_feed/results_true.dat   | 40 ++++++++++++-------------
 tests/test_cmfd_nofeed/results.py       |  2 +-
 tests/test_cmfd_nofeed/results_true.dat | 40 ++++++++++++-------------
 4 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/tests/test_cmfd_feed/results.py b/tests/test_cmfd_feed/results.py
index 67bbe6ca4f..f86aaa53e3 100644
--- a/tests/test_cmfd_feed/results.py
+++ b/tests/test_cmfd_feed/results.py
@@ -68,7 +68,7 @@ for item in sp.cmfd_balance:
   outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd dominance ratio\n'
 for item in sp.cmfd_dominance:
-  outstr += "{0:12.6E}\n".format(item)
+  outstr += "{0:10.3E}\n".format(item)
 outstr += 'cmfd openmc source comparison\n'
 for item in sp.cmfd_srccmp:
   outstr += "{0:12.6E}\n".format(item)
diff --git a/tests/test_cmfd_feed/results_true.dat b/tests/test_cmfd_feed/results_true.dat
index 20ed778cd2..0e46fd1371 100644
--- a/tests/test_cmfd_feed/results_true.dat
+++ b/tests/test_cmfd_feed/results_true.dat
@@ -721,26 +721,26 @@ cmfd balance
 2.279123E-03
 2.110816E-03
 cmfd dominance ratio
-0.000000E+00
-0.000000E+00
-0.000000E+00
-0.000000E+00
-5.509393E-01
-5.472954E-01
-5.461377E-01
-5.518823E-01
-5.540420E-01
-5.513822E-01
-5.505071E-01
-5.498933E-01
-5.486126E-01
-5.450413E-01
-5.437663E-01
-5.421995E-01
-5.417191E-01
-5.437232E-01
-5.429525E-01
-5.428436E-01
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 5.509E-01
+ 5.473E-01
+ 5.461E-01
+ 5.519E-01
+ 5.540E-01
+ 5.514E-01
+ 5.505E-01
+ 5.499E-01
+ 5.486E-01
+ 5.450E-01
+ 5.438E-01
+ 5.422E-01
+ 5.417E-01
+ 5.437E-01
+ 5.430E-01
+ 5.428E-01
 cmfd openmc source comparison
 0.000000E+00
 0.000000E+00
diff --git a/tests/test_cmfd_nofeed/results.py b/tests/test_cmfd_nofeed/results.py
index 67bbe6ca4f..f86aaa53e3 100644
--- a/tests/test_cmfd_nofeed/results.py
+++ b/tests/test_cmfd_nofeed/results.py
@@ -68,7 +68,7 @@ for item in sp.cmfd_balance:
   outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd dominance ratio\n'
 for item in sp.cmfd_dominance:
-  outstr += "{0:12.6E}\n".format(item)
+  outstr += "{0:10.3E}\n".format(item)
 outstr += 'cmfd openmc source comparison\n'
 for item in sp.cmfd_srccmp:
   outstr += "{0:12.6E}\n".format(item)
diff --git a/tests/test_cmfd_nofeed/results_true.dat b/tests/test_cmfd_nofeed/results_true.dat
index be594a3fba..683f6a6ec3 100644
--- a/tests/test_cmfd_nofeed/results_true.dat
+++ b/tests/test_cmfd_nofeed/results_true.dat
@@ -721,26 +721,26 @@ cmfd balance
 1.367144E-03
 1.402525E-03
 cmfd dominance ratio
-0.000000E+00
-0.000000E+00
-0.000000E+00
-0.000000E+00
-5.509393E-01
-5.473805E-01
-5.434034E-01
-5.469278E-01
-5.460554E-01
-5.474063E-01
-5.486271E-01
-5.485095E-01
-5.490619E-01
-5.473524E-01
-5.466605E-01
-5.471154E-01
-5.480465E-01
-5.485550E-01
-5.500141E-01
-5.502154E-01
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 5.509E-01
+ 5.474E-01
+ 5.434E-01
+ 5.469E-01
+ 5.461E-01
+ 5.474E-01
+ 5.486E-01
+ 5.485E-01
+ 5.491E-01
+ 5.474E-01
+ 5.467E-01
+ 5.471E-01
+ 5.480E-01
+ 5.486E-01
+ 5.500E-01
+ 5.502E-01
 cmfd openmc source comparison
 0.000000E+00
 0.000000E+00

From ddbaa4a3b89e72fdb27dbdb15bf079f43b575946 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 9 Sep 2013 14:18:18 -0400
Subject: [PATCH 50/69] standardized Makefile between intel and gnu compiler
 options

---
 src/Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index 433095c211..2236a5d019 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -41,12 +41,12 @@ GIT_SHA1 = $(shell git log -1 2>/dev/null | head -n 1 | awk '{print $$2}')
 
 ifeq ($(COMPILER),gnu)
   F90 = gfortran
-  F90FLAGS := -cpp -fbacktrace
+  F90FLAGS := -cpp -std=f2008 -fbacktrace
   LDFLAGS =
 
   # Debugging
   ifeq ($(DEBUG),yes)
-    F90FLAGS += -g -Wall -pedantic -std=f2008 -fbounds-check \
+    F90FLAGS += -g -Wall -pedantic -fbounds-check \
                 -ffpe-trap=invalid,overflow,underflow
     LDFLAGS  += -g
   endif
@@ -69,12 +69,12 @@ endif
 
 ifeq ($(COMPILER),intel)
   F90 = ifort
-  F90FLAGS := -fpp -warn -assume byterecl -traceback
+  F90FLAGS := -fpp -std08 -assume byterecl -traceback
   LDFLAGS =
 
   # Debugging
   ifeq ($(DEBUG),yes)
-    F90FLAGS += -g -ftrapuv -fp-stack-check -check all -fpe0
+    F90FLAGS += -g -warn -ftrapuv -fp-stack-check -check all -fpe0
     LDFLAGS  += -g
   endif
 

From 5af64724f410b0ea440a037e840ee3208abd9b32 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 9 Sep 2013 16:22:09 -0400
Subject: [PATCH 51/69] added PETSc JFNK solver test

---
 tests/test_cmfd_jfnk/cmfd.xml          |  16 +
 tests/test_cmfd_jfnk/geometry.xml      |  43 ++
 tests/test_cmfd_jfnk/materials.xml     |  12 +
 tests/test_cmfd_jfnk/results.py        |  82 +++
 tests/test_cmfd_jfnk/results_true.dat  | 775 +++++++++++++++++++++++++
 tests/test_cmfd_jfnk/settings.xml      |  32 +
 tests/test_cmfd_jfnk/tallies.xml       |  16 +
 tests/test_cmfd_jfnk/test_cmfd_feed.py |  68 +++
 8 files changed, 1044 insertions(+)
 create mode 100644 tests/test_cmfd_jfnk/cmfd.xml
 create mode 100644 tests/test_cmfd_jfnk/geometry.xml
 create mode 100644 tests/test_cmfd_jfnk/materials.xml
 create mode 100644 tests/test_cmfd_jfnk/results.py
 create mode 100644 tests/test_cmfd_jfnk/results_true.dat
 create mode 100644 tests/test_cmfd_jfnk/settings.xml
 create mode 100644 tests/test_cmfd_jfnk/tallies.xml
 create mode 100644 tests/test_cmfd_jfnk/test_cmfd_feed.py

diff --git a/tests/test_cmfd_jfnk/cmfd.xml b/tests/test_cmfd_jfnk/cmfd.xml
new file mode 100644
index 0000000000..d97b098fd2
--- /dev/null
+++ b/tests/test_cmfd_jfnk/cmfd.xml
@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+<cmfd>
+
+  <mesh>
+    <lower_left>  -10 -1 -1 </lower_left>
+    <upper_right>  10  1  1 </upper_right>
+    <dimension> 10 1 1 </dimension>
+    <albedo> 0.0 0.0 1.0 1.0 1.0 1.0 </albedo>
+  </mesh>
+
+  <begin> 5 </begin>
+  <display> balance </display>
+  <solver> jfnk </solver>
+  <feedback> true </feedback>
+
+</cmfd>
diff --git a/tests/test_cmfd_jfnk/geometry.xml b/tests/test_cmfd_jfnk/geometry.xml
new file mode 100644
index 0000000000..57c4aa2285
--- /dev/null
+++ b/tests/test_cmfd_jfnk/geometry.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<geometry>
+
+<!-- Definition of Cells -->
+  <cell id="1">
+    <universe>0</universe>
+    <surfaces>-1 2 -3 4 -5 6</surfaces>
+    <material>1</material>
+  </cell>
+
+<!-- Defition of Surfaces -->
+  <surface id="1">
+    <type>x-plane</type>
+    <coeffs>10</coeffs>
+    <boundary> vacuum </boundary>
+  </surface>
+  <surface id="2">
+    <type>x-plane</type>
+    <coeffs>-10</coeffs>
+    <boundary> vacuum </boundary>
+  </surface>
+  <surface id="3">
+    <type>y-plane</type>
+    <coeffs>1</coeffs>
+    <boundary>reflective</boundary>
+  </surface>
+  <surface id="4">
+    <type>y-plane</type>
+    <coeffs>-1</coeffs>
+    <boundary>reflective</boundary>
+  </surface>
+  <surface id="5">
+    <type>z-plane</type>
+    <coeffs>1</coeffs>
+    <boundary>reflective</boundary>
+  </surface>
+  <surface id="6">
+    <type>z-plane</type>
+    <coeffs>-1</coeffs>
+    <boundary>reflective</boundary>
+  </surface>
+   
+</geometry>
diff --git a/tests/test_cmfd_jfnk/materials.xml b/tests/test_cmfd_jfnk/materials.xml
new file mode 100644
index 0000000000..496774661e
--- /dev/null
+++ b/tests/test_cmfd_jfnk/materials.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0"?>
+<materials>
+
+<!-- Definition of materials -->
+  <material id="1">
+    <density value="19" units="g/cc" />
+    <nuclide name="U-235" xs="70c" wo="0.21" />
+    <nuclide name="U-238" xs="70c" wo="0.68" />
+    <nuclide name="O-16" xs="70c" wo="0.11" />
+  </material>
+
+</materials>
diff --git a/tests/test_cmfd_jfnk/results.py b/tests/test_cmfd_jfnk/results.py
new file mode 100644
index 0000000000..f86aaa53e3
--- /dev/null
+++ b/tests/test_cmfd_jfnk/results.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+
+import sys
+import numpy as np
+
+# import statepoint
+sys.path.append('../../src/utils')
+import statepoint
+
+# read in statepoint file
+if len(sys.argv) > 1:
+    sp = statepoint.StatePoint(sys.argv[1])
+else:
+    sp = statepoint.StatePoint('statepoint.20.binary')
+sp.read_results()
+
+# extract tally results and convert to vector
+results1 = sp.tallies[0].results
+shape1 = results1.shape
+size1 = (np.product(shape1))
+results1 = np.reshape(results1, size1)
+results2 = sp.tallies[1].results
+shape2 = results2.shape
+size2 = (np.product(shape2))
+results2 = np.reshape(results2, size2)
+results3 = sp.tallies[2].results
+shape3 = results3.shape
+size3 = (np.product(shape3))
+results3 = np.reshape(results3, size3)
+results4 = sp.tallies[3].results
+shape4 = results4.shape
+size4 = (np.product(shape4))
+results4 = np.reshape(results4, size4)
+
+# set up output string
+outstr = ''
+ 
+# write out k-combined
+outstr += 'k-combined:\n'
+outstr += "{0:12.6E} {1:12.6E}\n".format(sp.k_combined[0], sp.k_combined[1])
+
+# write out tally results
+outstr += 'tally 1:\n'
+for item in results1:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'tally 2:\n'
+for item in results2:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'tally 3:\n'
+for item in results3:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'tally 4:\n'
+for item in results4:
+  outstr += "{0:12.6E}\n".format(item)
+
+# write out cmfd answers
+outstr += 'cmfd indices\n'
+for item in sp.cmfd_indices:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'k cmfd\n'
+for item in sp.k_cmfd:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd entropy\n'
+for item in sp.cmfd_entropy:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd balance\n'
+for item in sp.cmfd_balance:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd dominance ratio\n'
+for item in sp.cmfd_dominance:
+  outstr += "{0:10.3E}\n".format(item)
+outstr += 'cmfd openmc source comparison\n'
+for item in sp.cmfd_srccmp:
+  outstr += "{0:12.6E}\n".format(item)
+outstr += 'cmfd source\n'
+cmfdsrc = np.reshape(sp.cmfd_src, np.product(sp.cmfd_indices), order='F')
+for item in cmfdsrc:
+  outstr += "{0:12.6E}\n".format(item)
+
+# write results to file
+with open('results_test.dat','w') as fh:
+    fh.write(outstr)
diff --git a/tests/test_cmfd_jfnk/results_true.dat b/tests/test_cmfd_jfnk/results_true.dat
new file mode 100644
index 0000000000..5e78660271
--- /dev/null
+++ b/tests/test_cmfd_jfnk/results_true.dat
@@ -0,0 +1,775 @@
+k-combined:
+1.167124E+00 1.217344E-02
+tally 1:
+1.126891E+01
+1.275756E+01
+2.086557E+01
+4.369504E+01
+2.853629E+01
+8.169563E+01
+3.404507E+01
+1.165020E+02
+3.723908E+01
+1.389881E+02
+3.760048E+01
+1.416691E+02
+3.455469E+01
+1.197186E+02
+2.837503E+01
+8.086287E+01
+2.151788E+01
+4.648312E+01
+1.194541E+01
+1.432330E+01
+tally 2:
+2.292174E+01
+2.650794E+01
+1.598184E+01
+1.288693E+01
+2.194144E+00
+2.510560E-01
+4.217817E+01
+8.938588E+01
+2.974466E+01
+4.447040E+01
+3.939271E+00
+7.882419E-01
+5.742290E+01
+1.654599E+02
+4.067582E+01
+8.308457E+01
+5.566956E+00
+1.570073E+00
+6.810587E+01
+2.331019E+02
+4.831544E+01
+1.174673E+02
+6.245000E+00
+1.967939E+00
+7.258131E+01
+2.646756E+02
+5.185146E+01
+1.351426E+02
+6.622489E+00
+2.231695E+00
+7.246115E+01
+2.641295E+02
+5.159541E+01
+1.339441E+02
+6.702239E+00
+2.277825E+00
+6.716396E+01
+2.266815E+02
+4.765062E+01
+1.141626E+02
+6.384097E+00
+2.063760E+00
+5.549541E+01
+1.545360E+02
+3.940819E+01
+7.794877E+01
+5.183373E+00
+1.369482E+00
+4.155684E+01
+8.679434E+01
+2.926743E+01
+4.307143E+01
+3.938851E+00
+7.877944E-01
+2.335179E+01
+2.758239E+01
+1.622682E+01
+1.331995E+01
+2.248920E+00
+2.594933E-01
+tally 3:
+1.537807E+01
+1.193746E+01
+1.044189E+00
+5.627797E-02
+2.861614E+01
+4.118324E+01
+1.846208E+00
+1.729007E-01
+3.921463E+01
+7.723579E+01
+2.427723E+00
+2.983153E-01
+4.653208E+01
+1.089818E+02
+3.128394E+00
+4.945409E-01
+4.989896E+01
+1.251503E+02
+3.242191E+00
+5.330743E-01
+4.967378E+01
+1.241657E+02
+3.184751E+00
+5.164524E-01
+4.592989E+01
+1.061019E+02
+2.936424E+00
+4.356307E-01
+3.783938E+01
+7.188486E+01
+2.485680E+00
+3.117618E-01
+2.819358E+01
+3.997585E+01
+1.875908E+00
+1.772000E-01
+1.562420E+01
+1.236020E+01
+1.013140E+00
+5.260420E-02
+tally 4:
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+3.126366E+00
+4.930686E-01
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+5.456412E+00
+1.499417E+00
+2.727756E+00
+3.788869E-01
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+7.453481E+00
+2.790160E+00
+5.240104E+00
+1.386896E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+8.741565E+00
+3.830226E+00
+7.137969E+00
+2.555043E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+9.221731E+00
+4.266573E+00
+8.474118E+00
+3.604648E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+9.054937E+00
+4.124536E+00
+9.066120E+00
+4.133834E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+8.362251E+00
+3.517244E+00
+9.086548E+00
+4.156118E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+6.871571E+00
+2.369189E+00
+8.399583E+00
+3.549783E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+4.998101E+00
+1.258193E+00
+7.145294E+00
+2.568114E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+2.716075E+00
+3.753566E-01
+5.427591E+00
+1.484153E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+3.056644E+00
+4.702345E-01
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+cmfd indices
+1.000000E+01
+1.000000E+00
+1.000000E+00
+1.000000E+00
+k cmfd
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+1.158665E+00
+1.173348E+00
+1.180514E+00
+1.168646E+00
+1.152485E+00
+1.154989E+00
+1.152285E+00
+1.152275E+00
+1.149476E+00
+1.154912E+00
+1.164393E+00
+1.170145E+00
+1.169161E+00
+1.167966E+00
+1.170153E+00
+1.170989E+00
+cmfd entropy
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+3.229801E+00
+3.226763E+00
+3.225011E+00
+3.230316E+00
+3.232043E+00
+3.228854E+00
+3.227439E+00
+3.228206E+00
+3.226749E+00
+3.222635E+00
+3.218882E+00
+3.219196E+00
+3.218057E+00
+3.220324E+00
+3.219099E+00
+3.219847E+00
+cmfd balance
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+6.244292E-03
+5.210436E-03
+4.212224E-03
+3.321578E-03
+3.067254E-03
+3.076634E-03
+2.601637E-03
+2.666441E-03
+2.490817E-03
+2.571411E-03
+3.013819E-03
+2.802836E-03
+2.725356E-03
+2.475281E-03
+2.279123E-03
+2.110816E-03
+cmfd dominance ratio
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+ 0.000E+00
+cmfd openmc source comparison
+0.000000E+00
+0.000000E+00
+0.000000E+00
+0.000000E+00
+1.136193E-02
+1.059214E-02
+9.675100E-03
+7.296571E-03
+5.812485E-03
+6.138879E-03
+6.070297E-03
+5.791569E-03
+6.020032E-03
+7.287183E-03
+7.068423E-03
+6.380521E-03
+5.828010E-03
+4.741191E-03
+4.848115E-03
+4.686773E-03
+cmfd source
+4.367444E-02
+7.866919E-02
+1.050799E-01
+1.366581E-01
+1.421824E-01
+1.393453E-01
+1.269612E-01
+1.060419E-01
+7.920472E-02
+4.218277E-02
diff --git a/tests/test_cmfd_jfnk/settings.xml b/tests/test_cmfd_jfnk/settings.xml
new file mode 100644
index 0000000000..41de07f569
--- /dev/null
+++ b/tests/test_cmfd_jfnk/settings.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<settings>
+
+  <!-- Parameters for criticality calculation -->
+  <eigenvalue>
+    <batches>20</batches>
+    <inactive>10</inactive>
+    <particles>1000</particles>
+  </eigenvalue>
+
+  <!-- How verbose output should be -->
+  <verbosity value="7" />
+
+  <!-- Starting source -->
+  <source>
+    <space>
+      <type>box</type>
+      <parameters>-10 -1 -1 10 1 1</parameters>
+    </space>
+  </source>
+
+  <!-- Shannon Entropy --> 
+  <entropy>
+    <dimension> 10 1 1 </dimension>
+    <lower_left> -10.0 -1.0 -1.0 </lower_left>
+    <upper_right> 10.0  1.0  1.0 </upper_right>
+  </entropy>
+
+ <!-- Run CMFD -->
+ <run_cmfd> true </run_cmfd>
+
+</settings>
diff --git a/tests/test_cmfd_jfnk/tallies.xml b/tests/test_cmfd_jfnk/tallies.xml
new file mode 100644
index 0000000000..b20c0ad613
--- /dev/null
+++ b/tests/test_cmfd_jfnk/tallies.xml
@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+<tallies>
+
+  <mesh id="1">
+    <type>rectangular</type>
+    <lower_left>-10 -1 -1 </lower_left>
+    <upper_right>10 1 1</upper_right>
+    <dimension>10 1 1</dimension>
+  </mesh>
+
+  <tally id="1">
+    <filter type="mesh" bins="1" />
+    <scores>flux</scores>
+  </tally>
+
+</tallies>
diff --git a/tests/test_cmfd_jfnk/test_cmfd_feed.py b/tests/test_cmfd_jfnk/test_cmfd_feed.py
new file mode 100644
index 0000000000..db36784888
--- /dev/null
+++ b/tests/test_cmfd_jfnk/test_cmfd_feed.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+
+import os
+from subprocess import Popen, STDOUT, PIPE, call
+import filecmp
+import glob
+
+from nose.plugins.skip import SkipTest
+
+from nose_mpi import NoseMPI
+
+
+pwd = os.path.dirname(__file__)
+
+skipAll = False
+
+def setup():
+    os.putenv('PWD', pwd)
+    os.chdir(pwd)
+    global skipAll
+    skipAll = False
+
+def test_run():
+    openmc_path = pwd + '/../../src/openmc'
+    if int(NoseMPI.mpi_np) > 0:
+        proc = Popen([NoseMPI.mpi_exec, '-np', NoseMPI.mpi_np, openmc_path],
+               stderr=STDOUT, stdout=PIPE)
+    else:
+        proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
+    returncode = proc.wait()
+    output = proc.communicate()[0]
+    print(output)
+    if 'CMFD is not available' in output:
+        global skipAll
+        skipAll = True
+        raise SkipTest
+    assert returncode == 0
+
+def test_created_statepoint():
+    if skipAll:
+        raise SkipTest
+    statepoint = glob.glob(pwd + '/statepoint.20.*')
+    assert len(statepoint) == 1
+    assert statepoint[0].endswith('binary') or statepoint[0].endswith('h5')
+
+def test_output_exists():
+    if skipAll:
+        raise SkipTest
+    assert os.path.exists(pwd + '/tallies.out')
+
+def test_results():
+    if skipAll:
+        raise SkipTest
+    statepoint = glob.glob(pwd + '/statepoint.20.*')
+    call(['python', 'results.py', statepoint[0]])
+    compare = filecmp.cmp('results_test.dat', 'results_true.dat')
+    if not compare:
+      os.rename('results_test.dat', 'results_error.dat')
+    assert compare
+
+def teardown():
+    output = glob.glob(pwd + '/statepoint.20.*')
+    output.append(pwd + '/tallies.out')
+    output.append(pwd + '/results_test.dat')
+    for f in output:
+        if os.path.exists(f):
+            os.remove(f)
+

From b49e9711d4a6a797cf4015dc713e0e4331a4a102 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 9 Sep 2013 16:22:38 -0400
Subject: [PATCH 52/69] reset dominance ratio to 0 in JFNK solver due to
 initial power iteration

---
 src/cmfd_jfnk_solver.F90 | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
index 9b14b4d1aa..4e163c5c93 100644
--- a/src/cmfd_jfnk_solver.F90
+++ b/src/cmfd_jfnk_solver.F90
@@ -81,8 +81,8 @@ contains
 
   subroutine init_data()
 
-    use constants, only: ONE
-    use global,    only: cmfd, cmfd_adjoint_type
+    use constants, only: ZERO, ONE
+    use global,    only: cmfd, cmfd_adjoint_type, current_batch
 
     logical :: physical_adjoint
     integer :: n
@@ -142,6 +142,9 @@ contains
     ! Set up Jacobian for PETSc
     call jac_prec % setup_petsc()
 
+    ! Set dominance ratio to 0
+    cmfd % dom(current_batch) = ZERO
+
   end subroutine init_data
 
 !==============================================================================

From 93af53d634064e603b14b942fe935b5ed2808f60 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 9 Sep 2013 16:24:27 -0400
Subject: [PATCH 53/69] rename python test

---
 tests/test_cmfd_jfnk/{test_cmfd_feed.py => test_cmfd_jfnk.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tests/test_cmfd_jfnk/{test_cmfd_feed.py => test_cmfd_jfnk.py} (100%)

diff --git a/tests/test_cmfd_jfnk/test_cmfd_feed.py b/tests/test_cmfd_jfnk/test_cmfd_jfnk.py
similarity index 100%
rename from tests/test_cmfd_jfnk/test_cmfd_feed.py
rename to tests/test_cmfd_jfnk/test_cmfd_jfnk.py

From 46df1146ada7cef0d97e9477bd41ba3913755fb9 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Mon, 9 Sep 2013 16:51:53 -0400
Subject: [PATCH 54/69] added some documentation on how to configure PETSc and
 HDF5

---
 docs/source/usersguide/install.rst | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/docs/source/usersguide/install.rst b/docs/source/usersguide/install.rst
index d5ea3d9c2d..52476040b4 100644
--- a/docs/source/usersguide/install.rst
+++ b/docs/source/usersguide/install.rst
@@ -71,13 +71,27 @@ Prerequisites
       To compile with support for HDF5_ output (highly recommended), you will
       need to have HDF5 installed on your computer. The installed version will
       need to have been compiled with the same compiler you intend to compile
-      OpenMC with.
+      OpenMC with. HDF5_ must be built with parallel I/O features. An example
+      of configuring HDF5_ is listed below::
+
+           FC=/opt/mpich/3.0.4-gnu/bin/mpif90 CC=/opt/mpich/3.0.4-gnu/bin/mpicc \
+           ./configure --prefix=/opt/hdf5/1.8.11-gnu --enable-fortran \
+                       --enable-fortran2003 --enable-parallel
 
     * PETSc_ for CMFD acceleration
 
-      To enable CMFD acceleration, you will need to have PETSc_ installed on
-      your computer. The installed version will need to have been compiled with
-      the same compiler you intend to compile OpenMC with.
+      To enable CMFD acceleration, you will need to have PETSc_ (3.4.2 or higher)
+      installed on your computer. The installed version will need to have been
+      compiled with the same compiler you intend to compile OpenMC with. OpenMC
+      requires PETSc_ to be configured with Fortran datatypes. An example of
+      configuring PETSc_ is listed below::
+
+           ./configure --prefix=/opt/petsc/3.4.2-gnu --download-f-blas-lapack \
+                       --with-mpi-dir=/opt/mpich/3.0.4-gnu/ --with-shared-libraries=0 \
+                       --with-fortran-datatypes
+
+      The BLAS/LAPACK library is not required to be downloaded and can be linked
+      explicitly (e.g., Intel MLK library).
 
     * git_ version control software for obtaining source code
 

From 6154c4db60eaae55ff835017eda76cde0fd4716d Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Tue, 10 Sep 2013 15:31:12 -0400
Subject: [PATCH 55/69] added more options for running test suite

---
 tests/run_tests.py                 | 53 +++++++++++++++++++++++++----
 tests/test_compile/test_compile.py | 54 ++++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+), 6 deletions(-)

diff --git a/tests/run_tests.py b/tests/run_tests.py
index 5e261fba73..e8650c52f4 100755
--- a/tests/run_tests.py
+++ b/tests/run_tests.py
@@ -67,22 +67,63 @@ sys.path.append(pwd)
 
 # Set list of tests, either default or from command line
 flags = []
-tests = ['compile', 'normal', 'debug', 'optimize', 'hdf5', 'mpi', 'phdf5',
-         'petsc', 'phdf5-petsc', 'phdf5-petsc-optimize']
+tests = ['compile', 'normal', 'debug', 'optimize', 
+         'hdf5', 'hdf5-debug', 'hdf5-optimize',
+         'mpi', 'mpi-debug', 'mpi-optimize',
+         'phdf5', 'phdf5-debug', 'phdf5-optimize',
+         'petsc', 'petsc-debug', 'petsc-optimize',
+         'phdf5-petsc', 'phdf5-petsc-debug', 'phdf5-petsc-optimize']
 if len(sys.argv) > 1:
     flags = [i for i in sys.argv[1:] if i.startswith('-')]
     tests_ = [i for i in sys.argv[1:] if not i.startswith('-')]
-    tests = tests_ if tests_ else tests
+
+    # Check for special subsets of tests
+    tests__ = []
+    for i in tests_:
+
+        # All tests will run all the tests except for compile unless
+        # it is also specified on the command line. Note that specifying
+        # compile all-tests is the same as not specifying any args
+        if i == 'all-tests':
+            tests__ = tests
+            try:
+                idx = tests_.index('compile') # check for compile test
+            except ValueError:
+                del tests__[0]
+            finally:
+                break # don't need to check for anything else
+
+        # This checks for any subsets of tests. The string after
+        # all-XXXX will be used to search through all tests.
+        # Specifying XXXX=normal will run tests that don't contain
+        # debug or optimize substring.
+        if i.startswith('all-'):
+            suffix = i.split('all-')[1]
+            if suffix == 'normal':
+                for j in tests:
+                    if j.rfind('debug') == -1 and \
+                       j.rfind('optimize') == -1:
+                        tests__.append(j)
+            else:
+                for j in tests:
+                    if j.rfind(suffix) != -1:
+                        tests__.append(j)
+        else:
+            tests__.append(i) # append specific test (e.g., mpi-debug)
+    tests = tests__ if tests__ else tests
 
 # Run tests
 results = []
 for name in tests:
     if name == 'compile':
         run_compile()
-    elif name in ['normal', 'debug', 'optimize', 'hdf5']:
+    elif name in ['normal', 'debug', 'optimize', 
+                  'hdf5', 'hdf5-debug', 'hdf5-optimize']:
         run_suite(name=name)
-    elif name in ['mpi', 'phdf5', 'petsc', 'phdf5-petsc',
-                  'phdf5-petsc-optimize']:
+    elif name in ['mpi', 'mpi-debug', 'mpi-optimize',
+                  'phdf5', 'phdf5-debug', 'phdf5-optimize',
+                  'petsc', 'petsc-debug', 'petsc-optimize',
+                  'phdf5-petsc', 'phdf5-petsc-debug', 'phdf5-petsc-optimize']:
         run_suite(name=name, mpi=True)
 
 # print out summary of results
diff --git a/tests/test_compile/test_compile.py b/tests/test_compile/test_compile.py
index 72e4dd6535..a4b3704a08 100644
--- a/tests/test_compile/test_compile.py
+++ b/tests/test_compile/test_compile.py
@@ -53,30 +53,84 @@ def test_mpi():
     assert returncode == 0
     shutil.move('openmc', 'openmc-mpi')
 
+def test_mpi_debug():
+    returncode = run(['make','distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'DEBUG=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-mpi-debug')
+
+def test_mpi_optimize():
+    returncode = run(['make','distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'OPTIMIZE=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-mpi-optimize')
+
 def test_hdf5():
     returncode = run(['make','distclean'])
     returncode = run(['make', compiler, 'HDF5=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-hdf5')
 
+def test_hdf5_debug():
+    returncode = run(['make','distclean'])
+    returncode = run(['make', compiler, 'HDF5=yes', 'DEBUG=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-hdf5-debug')
+
+def test_hdf5_optimize():
+    returncode = run(['make','distclean'])
+    returncode = run(['make', compiler, 'HDF5=yes', 'OPTIMIZE=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-hdf5-optimize')
+
 def test_petsc():
     returncode = run(['make','distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'PETSC=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-petsc')
 
+def test_petsc_debug():
+    returncode = run(['make','distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'PETSC=yes', 'DEBUG=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-petsc-debug')
+
+def test_petsc_optimize():
+    returncode = run(['make','distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'PETSC=yes', 'OPTIMIZE=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-petsc-optimize')
+
 def test_mpi_hdf5():
     returncode = run(['make','distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-phdf5')
 
+def test_mpi_hdf5_debug():
+    returncode = run(['make','distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes', 'DEBUG=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-phdf5-debug')
+
+def test_mpi_hdf5_optimize():
+    returncode = run(['make','distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes', 'OPTIMIZE=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-phdf5-optimize')
+
 def test_mpi_hdf5_petsc():
     returncode = run(['make','distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes', 'PETSC=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-phdf5-petsc')
 
+def test_mpi_hdf5_petsc_debug():
+    returncode = run(['make','distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes', 'PETSC=yes', 'DEBUG=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-phdf5-petsc-debug')
+
 def test_mpi_hdf5_petsc_optimize():
     returncode = run(['make','distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes', 'PETSC=yes', 'OPTIMIZE=yes'])

From c67423342e88c6565723ee43cabc6dc6ef18da7a Mon Sep 17 00:00:00 2001
From: Paul Romano <paul.k.romano@gmail.com>
Date: Sat, 21 Sep 2013 16:48:47 -0400
Subject: [PATCH 56/69] PEP8 fixes in run_tests.py and test_compile.py.

---
 tests/run_tests.py                 |  6 ++---
 tests/test_compile/test_compile.py | 39 +++++++++++++++++++++---------
 2 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/tests/run_tests.py b/tests/run_tests.py
index b10225f7fa..30d330e5d5 100755
--- a/tests/run_tests.py
+++ b/tests/run_tests.py
@@ -87,11 +87,11 @@ if len(sys.argv) > 1:
         if i == 'all-tests':
             tests__ = tests
             try:
-                idx = tests_.index('compile') # check for compile test
+                idx = tests_.index('compile')  # check for compile test
             except ValueError:
                 del tests__[0]
             finally:
-                break # don't need to check for anything else
+                break  # don't need to check for anything else
 
         # This checks for any subsets of tests. The string after
         # all-XXXX will be used to search through all tests.
@@ -109,7 +109,7 @@ if len(sys.argv) > 1:
                     if j.rfind(suffix) != -1:
                         tests__.append(j)
         else:
-            tests__.append(i) # append specific test (e.g., mpi-debug)
+            tests__.append(i)  # append specific test (e.g., mpi-debug)
     tests = tests__ if tests__ else tests
 
 # Run tests
diff --git a/tests/test_compile/test_compile.py b/tests/test_compile/test_compile.py
index bc3b2391d6..997f243d7b 100644
--- a/tests/test_compile/test_compile.py
+++ b/tests/test_compile/test_compile.py
@@ -59,18 +59,21 @@ def test_mpi():
     assert returncode == 0
     shutil.move('openmc', 'openmc-mpi')
 
+
 def test_mpi_debug():
-    returncode = run(['make','distclean'])
+    returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'DEBUG=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-mpi-debug')
 
+
 def test_mpi_optimize():
-    returncode = run(['make','distclean'])
+    returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'OPTIMIZE=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-mpi-optimize')
 
+
 def test_omp():
     returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'OPENMP=yes'])
@@ -84,36 +87,43 @@ def test_hdf5():
     assert returncode == 0
     shutil.move('openmc', 'openmc-hdf5')
 
+
 def test_hdf5_debug():
-    returncode = run(['make','distclean'])
+    returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'HDF5=yes', 'DEBUG=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-hdf5-debug')
 
+
 def test_hdf5_optimize():
-    returncode = run(['make','distclean'])
+    returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'HDF5=yes', 'OPTIMIZE=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-hdf5-optimize')
 
+
 def test_petsc():
     returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'PETSC=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-petsc')
 
+
 def test_petsc_debug():
-    returncode = run(['make','distclean'])
+    returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'PETSC=yes', 'DEBUG=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-petsc-debug')
 
+
 def test_petsc_optimize():
-    returncode = run(['make','distclean'])
-    returncode = run(['make', compiler, 'MPI=yes', 'PETSC=yes', 'OPTIMIZE=yes'])
+    returncode = run(['make', 'distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'PETSC=yes',
+                      'OPTIMIZE=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-petsc-optimize')
 
+
 def test_mpi_omp():
     returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'OPENMP=yes'])
@@ -127,36 +137,43 @@ def test_omp_hdf5():
     assert returncode == 0
     shutil.move('openmc', 'openmc-omp-hdf5')
 
+
 def test_mpi_hdf5():
     returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-phdf5')
 
+
 def test_mpi_hdf5_debug():
-    returncode = run(['make','distclean'])
+    returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes', 'DEBUG=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-phdf5-debug')
 
+
 def test_mpi_hdf5_optimize():
-    returncode = run(['make','distclean'])
+    returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes', 'OPTIMIZE=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-phdf5-optimize')
 
+
 def test_mpi_hdf5_petsc():
     returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes', 'PETSC=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-phdf5-petsc')
 
+
 def test_mpi_hdf5_petsc_debug():
-    returncode = run(['make','distclean'])
-    returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes', 'PETSC=yes', 'DEBUG=yes'])
+    returncode = run(['make', 'distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes', 'PETSC=yes',
+                      'DEBUG=yes'])
     assert returncode == 0
     shutil.move('openmc', 'openmc-phdf5-petsc-debug')
 
+
 def test_mpi_hdf5_petsc_optimize():
     returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes', 'PETSC=yes',

From 0fcde07bc554028da5985f6624b47dd5b61e788b Mon Sep 17 00:00:00 2001
From: Paul Romano <paul.k.romano@gmail.com>
Date: Sat, 21 Sep 2013 16:52:48 -0400
Subject: [PATCH 57/69] Modify build_dependencies.py to skip petsc modules.

---
 src/utils/build_dependencies.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/utils/build_dependencies.py b/src/utils/build_dependencies.py
index c88ddbfb62..19ac978857 100755
--- a/src/utils/build_dependencies.py
+++ b/src/utils/build_dependencies.py
@@ -11,7 +11,8 @@ for src in glob.iglob('*.F90'):
     d = re.findall(r'\n\s*use\s+(\w+)',
                    open(src, 'r').read())
     for name in d:
-        if name in ['mpi', 'hdf5', 'h5lt']:
+        if name in ['mpi', 'hdf5', 'h5lt', 'petscsys', 'petscmat', 'petscksp',
+                    'petscsnes', 'petscvec']:
             continue
         if name.startswith('xml_data_'):
             name = name.replace('xml_data_', 'templates/')

From 4907ed2df222bcddbbfd986ff8dbbfcdb9cb63b8 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Sun, 29 Sep 2013 15:07:42 -0400
Subject: [PATCH 58/69] added intent attribute on all procedure args

---
 src/cmfd_data.F90          |  18 +++---
 src/cmfd_execute.F90       |  41 +++++++-------
 src/cmfd_header.F90        |   6 +-
 src/cmfd_input.F90         |   2 +-
 src/cmfd_jfnk_solver.F90   |  16 +++---
 src/cmfd_loss_operator.F90 | 112 ++++++++++++++++++-------------------
 src/cmfd_power_solver.F90  |  12 ++--
 src/cmfd_prod_operator.F90 |  40 ++++++-------
 src/matrix_header.F90      |  86 ++++++++++++++--------------
 src/solver_interface.F90   |  82 +++++++++++++--------------
 src/vector_header.F90      |  26 ++++-----
 11 files changed, 221 insertions(+), 220 deletions(-)

diff --git a/src/cmfd_data.F90 b/src/cmfd_data.F90
index 0671c79eb6..d0e80aaa90 100644
--- a/src/cmfd_data.F90
+++ b/src/cmfd_data.F90
@@ -767,15 +767,15 @@ contains
     use global,     only: cmfd, cmfd_hold_weights
 
     real(8) :: get_reflector_albedo ! reflector albedo
-    integer :: i                    ! iteration counter for x
-    integer :: j                    ! iteration counter for y
-    integer :: k                    ! iteration counter for z
-    integer :: g                    ! iteration counter for groups
-    integer :: l                    ! iteration counter for leakages
+    integer, intent(in) :: i ! iteration counter for x
+    integer, intent(in) :: j ! iteration counter for y
+    integer, intent(in) :: k ! iteration counter for z
+    integer, intent(in) :: g ! iteration counter for groups
+    integer, intent(in) :: l ! iteration counter for leakages
 
-    integer :: shift_idx            ! parameter to shift index by +1 or -1
-    real(8) :: current(12)          ! partial currents for all faces of mesh cell            
-    real(8) :: albedo               ! the albedo
+    integer :: shift_idx   ! parameter to shift index by +1 or -1
+    real(8) :: current(12) ! partial currents for all faces of mesh cell            
+    real(8) :: albedo      ! the albedo
 
     ! Get partial currents from object
     current = cmfd%current(:,g,i,j,k)
@@ -798,7 +798,7 @@ contains
   end function get_reflector_albedo
 
 !===============================================================================
-! FIX_NEUTRON_BALANCE
+! FIX_NEUTRON_BALANCE is a method to adjust parameters to have perfect balance
 !===============================================================================
 
   subroutine fix_neutron_balance()
diff --git a/src/cmfd_execute.F90 b/src/cmfd_execute.F90
index 17c8157f41..7a35a12144 100644
--- a/src/cmfd_execute.F90
+++ b/src/cmfd_execute.F90
@@ -254,7 +254,7 @@ contains
   end subroutine calc_fission_source
 
 !===============================================================================
-! CMFD_REWEIGHT
+! CMFD_REWEIGHT performs weighting of particles in the source bank
 !===============================================================================
 
   subroutine cmfd_reweight(new_weights)
@@ -271,17 +271,18 @@ contains
     use mpi
 #endif
 
-    integer :: nx                      ! maximum number of cells in x direction
-    integer :: ny                      ! maximum number of cells in y direction
-    integer :: nz                      ! maximum number of cells in z direction
-    integer :: ng                      ! maximum number of energy groups
-    integer :: i                       ! iteration counter
-    integer :: ijk(3)                  ! spatial bin location
-    integer :: e_bin                   ! energy bin of source particle
-    integer :: n_groups                ! number of energy groups
-    logical :: outside                 ! any source sites outside mesh
-    logical :: in_mesh                 ! source site is inside mesh
-    logical :: new_weights             ! calcualte new weights
+    logical, intent(in) :: new_weights ! calcualte new weights
+
+    integer :: nx       ! maximum number of cells in x direction
+    integer :: ny       ! maximum number of cells in y direction
+    integer :: nz       ! maximum number of cells in z direction
+    integer :: ng       ! maximum number of energy groups
+    integer :: i        ! iteration counter
+    integer :: ijk(3)   ! spatial bin location
+    integer :: e_bin    ! energy bin of source particle
+    integer :: n_groups ! number of energy groups
+    logical :: outside  ! any source sites outside mesh
+    logical :: in_mesh  ! source site is inside mesh
 
     type(StructuredMesh), pointer :: m ! point to mesh
     real(8), allocatable :: egrid(:)   ! energy grid
@@ -389,13 +390,13 @@ contains
     use global, only: cmfd, cmfd_coremap
 
     integer :: matidx ! the index location in matrix
-    integer :: i      ! current x index
-    integer :: j      ! current y index
-    integer :: k      ! current z index
-    integer :: g      ! current group index
-    integer :: nx     ! maximum number of cells in x direction
-    integer :: ny     ! maximum number of cells in y direction
-    integer :: ng     ! maximum number of energy groups
+    integer, intent(in) :: i  ! current x index
+    integer, intent(in) :: j  ! current y index
+    integer, intent(in) :: k  ! current z index
+    integer, intent(in) :: g  ! current group index
+    integer, intent(in) :: nx ! maximum number of cells in x direction
+    integer, intent(in) :: ny ! maximum number of cells in y direction
+    integer, intent(in) :: ng ! maximum number of energy groups
 
     ! Check if coremap is used
     if (cmfd_coremap) then
@@ -413,7 +414,7 @@ contains
   end function get_matrix_idx
 
 !===============================================================================
-! CMFD_TALLY_RESET
+! CMFD_TALLY_RESET resets all cmfd tallies
 !===============================================================================
 
   subroutine cmfd_tally_reset()
diff --git a/src/cmfd_header.F90 b/src/cmfd_header.F90
index 42e6b30ef9..3d7786ab8e 100644
--- a/src/cmfd_header.F90
+++ b/src/cmfd_header.F90
@@ -93,8 +93,8 @@ contains
 
   subroutine allocate_cmfd(this, n_batches)
 
-    integer :: n_batches
-    type(cmfd_type) :: this
+    integer, intent(in)            :: n_batches ! number of batches in calc
+    type(cmfd_type), intent(inout) :: this      ! cmfd instance
 
     integer :: nx  ! number of mesh cells in x direction
     integer :: ny  ! number of mesh cells in y direction
@@ -169,7 +169,7 @@ contains
 
   subroutine deallocate_cmfd(this)
 
-    type(cmfd_type) :: this
+    type(cmfd_type), intent(inout) :: this ! cmfd instance
 
     if (allocated(this % egrid))         deallocate(this % egrid)
     if (allocated(this % totalxs))       deallocate(this % totalxs)
diff --git a/src/cmfd_input.F90 b/src/cmfd_input.F90
index 9a768f4e3b..a3a497b10f 100644
--- a/src/cmfd_input.F90
+++ b/src/cmfd_input.F90
@@ -228,7 +228,7 @@ contains
     integer :: ng          ! number of energy groups (default 1)
     integer :: n_filters   ! number of filters
     integer :: i_filter_mesh ! index for mesh filter
-    character(MAX_LINE_LEN) :: filename
+    character(MAX_LINE_LEN) :: filename ! name of cmfd file
     type(TallyObject),    pointer :: t => null()
     type(StructuredMesh), pointer :: m => null()
     type(TallyFilter) :: filters(N_FILTER_TYPES) ! temporary filters
diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
index 4e163c5c93..958a4c3e56 100644
--- a/src/cmfd_jfnk_solver.F90
+++ b/src/cmfd_jfnk_solver.F90
@@ -30,7 +30,7 @@ contains
 
     use global,  only: time_cmfdbuild, time_cmfdsolve
 
-    logical, optional :: adjoint ! adjoint calculation
+    logical, intent(in), optional :: adjoint ! adjoint calculation
 
     ! Check for adjoint
     adjoint_calc = .false.
@@ -84,8 +84,8 @@ contains
     use constants, only: ZERO, ONE
     use global,    only: cmfd, cmfd_adjoint_type, current_batch
 
-    logical :: physical_adjoint
-    integer :: n
+    logical :: physical_adjoint ! physical adjoing calculation logical
+    integer :: n ! size of matrices
 
     ! Set up all matrices
     call init_loss_matrix(loss)
@@ -153,8 +153,8 @@ contains
 
   subroutine init_jacobian_matrix()
 
-    integer :: nnz
-    integer :: n
+    integer :: nnz ! number of nonzeros in matrix
+    integer :: n   ! dimension of matrix
 
     ! Get length of matrix and number of nonzeros total in loss matrix
     nnz = loss % nnz
@@ -179,7 +179,7 @@ contains
 
     use constants,  only: ONE
 
-    type(Vector) :: x
+    type(Vector), intent(in) :: x ! solution vector
 
     integer      :: i      ! loop counter for jacobian rows
     integer      :: jjac   ! loop counter for jacobian cols
@@ -267,8 +267,8 @@ contains
 
     use global,       only: cmfd_write_matrices
 
-    type(Vector) :: x
-    type(Vector) :: res
+    type(Vector), intent(in)  :: x   ! solution vector
+    type(Vector), intent(out) :: res ! residual vector
 
     character(len=25) :: filename
     integer           :: n
diff --git a/src/cmfd_loss_operator.F90 b/src/cmfd_loss_operator.F90
index e09497bedc..6e9917ac5b 100644
--- a/src/cmfd_loss_operator.F90
+++ b/src/cmfd_loss_operator.F90
@@ -16,7 +16,7 @@ contains
 
   subroutine init_loss_matrix(loss_matrix)
 
-    type(Matrix) :: loss_matrix ! cmfd loss matrix
+    type(Matrix), intent(inout) :: loss_matrix ! cmfd loss matrix
 
     integer :: nx   ! maximum number of x cells
     integer :: ny   ! maximum number of y cells
@@ -68,12 +68,12 @@ contains
 
   function preallocate_loss_matrix(nx, ny, nz, ng, n) result(nnz)
 
-    integer, intent(in)    :: nx   ! maximum number of x cells
-    integer, intent(in)    :: ny   ! maximum number of y cells
-    integer, intent(in)    :: nz   ! maximum number of z cells
-    integer, intent(in)    :: ng   ! maximum number of groups
-    integer, intent(in)    :: n    ! total length of matrix
-    integer                :: nnz  ! number of nonzeros
+    integer, intent(in) :: nx  ! maximum number of x cells
+    integer, intent(in) :: ny  ! maximum number of y cells
+    integer, intent(in) :: nz  ! maximum number of z cells
+    integer, intent(in) :: ng  ! maximum number of groups
+    integer, intent(in) :: n   ! total length of matrix
+    integer             :: nnz ! number of nonzeros
 
     integer :: i             ! iteration counter for x
     integer :: j             ! iteration counter for y
@@ -181,39 +181,39 @@ contains
 
   subroutine build_loss_matrix(loss_matrix, adjoint)
 
-    type(Matrix) :: loss_matrix     ! cmfd loss matrix
-    logical, optional :: adjoint    ! set up the adjoint
+    type(Matrix), intent(inout)   :: loss_matrix ! cmfd loss matrix
+    logical, intent(in), optional :: adjoint     ! set up the adjoint
 
-    integer :: nxyz(3,2)            ! single vector containing bound. locations
-    integer :: i                    ! iteration counter for x
-    integer :: j                    ! iteration counter for y
-    integer :: k                    ! iteration counter for z
-    integer :: g                    ! iteration counter for groups
-    integer :: l                    ! iteration counter for leakages
-    integer :: h                    ! energy group when doing scattering
-    integer :: nx                   ! maximum number of x cells
-    integer :: ny                   ! maximum number of y cells
-    integer :: nz                   ! maximum number of z cells
-    integer :: ng                   ! maximum number of groups
-    integer :: neig_mat_idx         ! matrix index of neighbor cell
-    integer :: scatt_mat_idx        ! matrix index for h-->g scattering terms
-    integer :: bound(6)             ! vector for comparing when looking for bound
-    integer :: xyz_idx              ! index for determining if x,y or z leakage
-    integer :: dir_idx              ! index for determining - or + face of cell
-    integer :: neig_idx(3)          ! spatial indices of neighbour
-    integer :: shift_idx            ! parameter to shift index by +1 or -1
-    integer :: irow                 ! iteration counter over row
-    logical :: adjoint_calc         ! is this a physical adjoint calculation?
-    real(8) :: totxs                ! total macro cross section
-    real(8) :: scattxsgg            ! scattering macro cross section g-->g
-    real(8) :: scattxshg            ! scattering macro cross section h-->g
-    real(8) :: dtilde(6)            ! finite difference coupling parameter
-    real(8) :: dhat(6)              ! nonlinear coupling parameter
-    real(8) :: hxyz(3)              ! cell lengths in each direction
-    real(8) :: jn                   ! direction dependent leakage coeff to neig
-    real(8) :: jo(6)                ! leakage coeff in front of cell flux
-    real(8) :: jnet                 ! net leakage from jo
-    real(8) :: val                  ! temporary variable before saving to 
+    integer :: nxyz(3,2)     ! single vector containing bound. locations
+    integer :: i             ! iteration counter for x
+    integer :: j             ! iteration counter for y
+    integer :: k             ! iteration counter for z
+    integer :: g             ! iteration counter for groups
+    integer :: l             ! iteration counter for leakages
+    integer :: h             ! energy group when doing scattering
+    integer :: nx            ! maximum number of x cells
+    integer :: ny            ! maximum number of y cells
+    integer :: nz            ! maximum number of z cells
+    integer :: ng            ! maximum number of groups
+    integer :: neig_mat_idx  ! matrix index of neighbor cell
+    integer :: scatt_mat_idx ! matrix index for h-->g scattering terms
+    integer :: bound(6)      ! vector for comparing when looking for bound
+    integer :: xyz_idx       ! index for determining if x,y or z leakage
+    integer :: dir_idx       ! index for determining - or + face of cell
+    integer :: neig_idx(3)   ! spatial indices of neighbour
+    integer :: shift_idx     ! parameter to shift index by +1 or -1
+    integer :: irow          ! iteration counter over row
+    logical :: adjoint_calc  ! is this a physical adjoint calculation?
+    real(8) :: totxs         ! total macro cross section
+    real(8) :: scattxsgg     ! scattering macro cross section g-->g
+    real(8) :: scattxshg     ! scattering macro cross section h-->g
+    real(8) :: dtilde(6)     ! finite difference coupling parameter
+    real(8) :: dhat(6)       ! nonlinear coupling parameter
+    real(8) :: hxyz(3)       ! cell lengths in each direction
+    real(8) :: jn            ! direction dependent leakage coeff to neig
+    real(8) :: jo(6)         ! leakage coeff in front of cell flux
+    real(8) :: jnet          ! net leakage from jo
+    real(8) :: val           ! temporary variable before saving to 
 
     ! Check for adjoint
     adjoint_calc = .false.
@@ -366,14 +366,14 @@ contains
 
   subroutine indices_to_matrix(g, i, j, k, matidx, ng, nx, ny)
 
-    integer :: matidx ! the index location in matrix
-    integer :: i      ! current x index
-    integer :: j      ! current y index
-    integer :: k      ! current z index
-    integer :: g      ! current group index
-    integer :: nx     ! maximum number of x cells
-    integer :: ny     ! maximum number of y cells
-    integer :: ng     ! maximum number of groups
+    integer, intent(out) :: matidx ! the index location in matrix
+    integer, intent(in)  :: i      ! current x index
+    integer, intent(in)  :: j      ! current y index
+    integer, intent(in)  :: k      ! current z index
+    integer, intent(in)  :: g      ! current group index
+    integer, intent(in)  :: nx     ! maximum number of x cells
+    integer, intent(in)  :: ny     ! maximum number of y cells
+    integer, intent(in)  :: ng     ! maximum number of groups
 
     ! Check if coremap is used
     if (cmfd_coremap) then
@@ -396,15 +396,15 @@ contains
 
   subroutine matrix_to_indices(irow, g, i, j, k, ng, nx, ny, nz)
 
-    integer :: i     ! iteration counter for x
-    integer :: j     ! iteration counter for y
-    integer :: k     ! iteration counter for z
-    integer :: g     ! iteration counter for groups
-    integer :: irow  ! iteration counter over row (0 reference)
-    integer :: nx    ! maximum number of x cells
-    integer :: ny    ! maximum number of y cells
-    integer :: nz    ! maximum number of z cells
-    integer :: ng    ! maximum number of groups
+    integer, intent(out) :: i     ! iteration counter for x
+    integer, intent(out) :: j     ! iteration counter for y
+    integer, intent(out) :: k     ! iteration counter for z
+    integer, intent(out) :: g     ! iteration counter for groups
+    integer, intent(in)  :: irow  ! iteration counter over row (0 reference)
+    integer, intent(in)  :: nx    ! maximum number of x cells
+    integer, intent(in)  :: ny    ! maximum number of y cells
+    integer, intent(in)  :: nz    ! maximum number of z cells
+    integer, intent(in)  :: ng    ! maximum number of groups
 
     ! Check for core map
     if (cmfd_coremap) then
diff --git a/src/cmfd_power_solver.F90 b/src/cmfd_power_solver.F90
index cd31789407..c1309b72cd 100644
--- a/src/cmfd_power_solver.F90
+++ b/src/cmfd_power_solver.F90
@@ -41,11 +41,11 @@ contains
 
     use global,  only: cmfd_adjoint_type, time_cmfdbuild, time_cmfdsolve
 
-    real(8), optional :: k_tol    ! tolerance on keff
-    real(8), optional :: s_tol    ! tolerance on source
-    logical, optional :: adjoint  ! adjoint calc
+    real(8), intent(in), optional :: k_tol    ! tolerance on keff
+    real(8), intent(in), optional :: s_tol    ! tolerance on source
+    logical, intent(in), optional :: adjoint  ! adjoint calc
 
-    logical :: physical_adjoint = .false.
+    logical :: physical_adjoint = .false. ! physical adjoint default false
 
     ! Set tolerances if present
     if (present(k_tol)) ktol = k_tol
@@ -104,7 +104,7 @@ contains
     use constants, only: ONE, ZERO
     use global,    only: cmfd_write_matrices
 
-    logical :: adjoint
+    logical, intent(in) :: adjoint ! adjoint calcualtion
 
     integer :: n      ! problem size
     real(8) :: guess  ! initial guess
@@ -238,7 +238,7 @@ contains
     use global,     only: cmfd_power_monitor, master
     use, intrinsic :: ISO_FORTRAN_ENV
 
-    integer     :: iter           ! iteration number
+    integer, intent(in) :: iter ! iteration number
 
     ! Reset convergence flag
     iconv = .false.
diff --git a/src/cmfd_prod_operator.F90 b/src/cmfd_prod_operator.F90
index c2db9fc83c..e6d055ced8 100644
--- a/src/cmfd_prod_operator.F90
+++ b/src/cmfd_prod_operator.F90
@@ -16,7 +16,7 @@ contains
 
   subroutine init_prod_matrix(prod_matrix)
 
-    type(Matrix) :: prod_matrix
+    type(Matrix), intent(inout) :: prod_matrix ! production matrix
 
     integer :: nx   ! maximum number of x cells
     integer :: ny   ! maximum number of y cells
@@ -50,8 +50,8 @@ contains
 
   subroutine build_prod_matrix(prod_matrix, adjoint)
 
-    type(Matrix) :: prod_matrix
-    logical, optional :: adjoint
+    type(Matrix), intent(inout)   :: prod_matrix ! production matrix
+    logical, intent(in), optional :: adjoint     ! adjoint calculation logical
 
     integer :: i                  ! iteration counter for x
     integer :: j                  ! iteration counter for y
@@ -135,14 +135,14 @@ contains
 
   subroutine indices_to_matrix(g, i, j, k, matidx, ng, nx, ny)
 
-    integer :: matidx ! the index location in matrix
-    integer :: i      ! current x index
-    integer :: j      ! current y index
-    integer :: k      ! current z index
-    integer :: g      ! current group index
-    integer :: nx     ! maximum number of x cells
-    integer :: ny     ! maximum number of y cells
-    integer :: ng     ! maximum number of groups
+    integer, intent(out) :: matidx ! the index location in matrix
+    integer, intent(in)  :: i      ! current x index
+    integer, intent(in)  :: j      ! current y index
+    integer, intent(in)  :: k      ! current z index
+    integer, intent(in)  :: g      ! current group index
+    integer, intent(in)  :: nx     ! maximum number of x cells
+    integer, intent(in)  :: ny     ! maximum number of y cells
+    integer, intent(in)  :: ng     ! maximum number of groups
 
     ! check if coremap is used
     if (cmfd_coremap) then
@@ -165,15 +165,15 @@ contains
 
   subroutine matrix_to_indices(irow, g, i, j, k, ng, nx, ny, nz)
 
-    integer :: i     ! iteration counter for x
-    integer :: j     ! iteration counter for y
-    integer :: k     ! iteration counter for z
-    integer :: g     ! iteration counter for groups
-    integer :: irow  ! iteration counter over row (0 reference)
-    integer :: nx    ! maximum number of x cells
-    integer :: ny    ! maximum number of y cells
-    integer :: nz    ! maximum number of z cells
-    integer :: ng    ! maximum number of groups
+    integer, intent(out) :: i    ! iteration counter for x
+    integer, intent(out) :: j    ! iteration counter for y
+    integer, intent(out) :: k    ! iteration counter for z
+    integer, intent(out) :: g    ! iteration counter for groups
+    integer, intent(in)  :: irow ! iteration counter over row (0 reference)
+    integer, intent(in)  :: nx   ! maximum number of x cells
+    integer, intent(in)  :: ny   ! maximum number of y cells
+    integer, intent(in)  :: nz   ! maximum number of z cells
+    integer, intent(in)  :: ng   ! maximum number of groups
 
     ! check for core map
     if (cmfd_coremap) then
diff --git a/src/matrix_header.F90 b/src/matrix_header.F90
index 25f1df95a5..591eeaaed1 100644
--- a/src/matrix_header.F90
+++ b/src/matrix_header.F90
@@ -43,9 +43,9 @@ contains
 
   subroutine matrix_create(self, n, nnz)
 
-    integer       :: n
-    integer       :: nnz
-    class(Matrix) :: self
+    integer, intent(in) :: n             ! dimension of matrix
+    integer, intent(in) :: nnz           ! number of nonzeros
+    class(Matrix), intent(inout) :: self ! matrix instance
 
     ! Preallocate vectors
     if (.not.allocated(self % row)) allocate(self % row(n+1))
@@ -71,7 +71,7 @@ contains
 
   subroutine matrix_destroy(self)
 
-    class(Matrix) :: self
+    class(Matrix), intent(inout) :: self ! matrix instance
 
 #ifdef PETSC
     if (self % petsc_active) call MatDestroy(self % petsc_mat, petsc_err)
@@ -89,9 +89,9 @@ contains
 
   subroutine matrix_add_value(self, col, val)
 
-    integer       :: col
-    real(8)       :: val
-    class(Matrix) :: self
+    integer, intent(in) :: col           ! col location in matrix
+    real(8), intent(in) :: val           ! value to store in matrix
+    class(Matrix), intent(inout) :: self ! matrix instance
 
     ! Record the data
     self % col(self % nz_count) = col
@@ -112,7 +112,7 @@ contains
 
   subroutine matrix_new_row(self)
 
-    class(Matrix) :: self
+    class(Matrix), intent(inout) :: self ! matrix instance
 
     ! Record the current number of nonzeros
     self % row(self % n_count) = self % nz_count
@@ -132,7 +132,7 @@ contains
 
   subroutine matrix_assemble(self)
 
-    class(Matrix) :: self
+    class(Matrix), intent(inout) :: self ! matrix instance
 
     integer :: i
     integer :: first
@@ -158,12 +158,12 @@ contains
 
   recursive subroutine sort_csr(col, val, first, last)
 
-    integer :: col(:)
-    integer :: first
-    integer :: last
-    real(8) :: val(:)
+    integer :: col(:) ! column vector to sort
+    integer :: first  ! first value in sort
+    integer :: last   ! last value in sort
+    real(8) :: val(:) ! value vector to be sorted like columns
 
-    integer :: mid
+    integer :: mid ! midpoint value
 
     if (first < last) then
       call split(col, val, first, last, mid)
@@ -179,18 +179,18 @@ contains
 
   subroutine split(col, val, low, high, mid)
 
-    integer :: col(:)
-    integer :: low
-    integer :: high
-    integer :: mid
-    real(8) :: val(:)
+    integer :: col(:) ! column vector to sort
+    integer :: low    ! low index to sort
+    integer :: high   ! high index to sort
+    integer :: mid    ! middle of sort
+    real(8) :: val(:) ! value vector to be sorted like columns
 
-    integer :: left
-    integer :: right
-    integer :: iswap
-    integer :: pivot
-    real(8) :: rswap
-    real(8) :: val0
+    integer :: left  ! contains left value in sort
+    integer :: right ! contains right value in sort
+    integer :: iswap ! temporary interger swap
+    integer :: pivot ! pivotting variable for columns
+    real(8) :: rswap ! temporary real swap
+    real(8) :: val0  ! pivot for value vector
 
     left = low
     right = high
@@ -232,14 +232,14 @@ contains
   end subroutine split
 
 !===============================================================================
-! MATRIX_GET_ROW
+! MATRIX_GET_ROW is a method to get row and checks for PETSc C indexing use
 !===============================================================================
 
   function matrix_get_row(self, i) result(row)
 
-    class(Matrix) :: self
-    integer       :: i
-    integer       :: row
+    class(Matrix), intent(in) :: self ! matrix instance
+    integer, intent(in)       :: i    ! row to get
+    integer                   :: row  ! index in col where row begins
 
     row = self % row(i)
 
@@ -248,14 +248,14 @@ contains
   end function matrix_get_row
 
 !===============================================================================
-! MATRIX_GET_COL
+! MATRIX_GET_COL is a method to get column and checks for PETSc C index use
 !===============================================================================
 
   function matrix_get_col(self, i) result(col)
 
-    class(Matrix) :: self
-    integer       :: i
-    integer       :: col
+    class(Matrix), intent(in) :: self ! matrix instance
+    integer, intent(in)       :: i    ! index from row vector
+    integer                   :: col  ! the actual column
 
     col = self % col(i)
 
@@ -269,7 +269,7 @@ contains
 
   subroutine matrix_setup_petsc(self)
 
-    class(Matrix) :: self
+    class(Matrix), intent(inout) :: self ! matrix instance
 
     ! change indices to c notation
     self % row = self % row - 1
@@ -292,11 +292,11 @@ contains
 
   subroutine matrix_write_petsc_binary(self, filename)
 
-    character(*) :: filename
-    class(Matrix) :: self
+    character(*), intent(in)  :: filename ! file name to write to
+    class(Matrix), intent(in) :: self     ! matrix instance
 
 #ifdef PETSC
-    type(PetscViewer) :: viewer
+    type(PetscViewer) :: viewer ! a petsc viewer instance
 
     call PetscViewerBinaryOpen(PETSC_COMM_WORLD, trim(filename), &
          FILE_MODE_WRITE, viewer, petsc_err)
@@ -312,7 +312,7 @@ contains
 
   subroutine matrix_transpose(self)
 
-    class(Matrix) :: self
+    class(Matrix), intent(inout) :: self ! matrix instance
 
 #ifdef PETSC
     call MatTranspose(self % petsc_mat, MAT_REUSE_MATRIX, self % petsc_mat, &
@@ -330,12 +330,12 @@ contains
     use constants,      only: ZERO
     use vector_header,  only: Vector
 
-    class(Matrix) :: self
-    type(Vector)  :: vec_in
-    type(Vector)  :: vec_out
+    class(Matrix), intent(in)   :: self    ! matrix instance
+    type(Vector), intent(in)    :: vec_in  ! vector to multiply matrix against
+    type(Vector), intent(inout) :: vec_out ! resulting vector
 
-    integer :: i
-    integer :: j
+    integer :: i ! row iteration counter
+    integer :: j ! column iteration counter
 
     ! Begin loop around rows
     ROWS: do i = 1, self % n
diff --git a/src/solver_interface.F90 b/src/solver_interface.F90
index a937057d6b..113e3119af 100644
--- a/src/solver_interface.F90
+++ b/src/solver_interface.F90
@@ -16,8 +16,8 @@ module solver_interface
   ! GMRES solver type 
   type, public :: GMRESSolver 
 #ifdef PETSC
-    type(ksp) :: ksp_
-    type(pc)  :: pc_
+    type(ksp) :: ksp_ ! Krylov linear solver instance
+    type(pc)  :: pc_  ! Preconditioner instance
 #endif
    contains
 #ifdef PETSC
@@ -37,11 +37,11 @@ module solver_interface
   ! JFNK solver type 
   type, public :: JFNKSolver 
 #ifdef PETSC
-    type(ksp)  :: ksp_
-    type(pc)   :: pc_
-    type(snes) :: snes_
-    type(mat)  :: jac_mf
-    integer    :: ls
+    type(ksp)  :: ksp_   ! Krylov linear solver instance
+    type(pc)   :: pc_    ! Preconditioner instance
+    type(snes) :: snes_  ! Nonlinear solver instance
+    type(mat)  :: jac_mf ! Matrix free jacobian instance
+    integer    :: ls     ! Line search instance
 #endif
    contains
 #ifdef PETSC
@@ -56,13 +56,13 @@ module solver_interface
   abstract interface
     subroutine res_interface(x, res)
       import :: Vector
-      type(Vector) :: x
-      type(Vector) :: res
+      type(Vector), intent(in)  :: x   ! solution vector
+      type(Vector), intent(out) :: res ! residual vector
     end subroutine res_interface
 
     subroutine jac_interface(x)
       import :: Vector
-      type(Vector) :: x
+      type(Vector), intent(in) :: x ! solution vector
     end subroutine jac_interface
   end interface
 
@@ -79,7 +79,7 @@ contains
 
   subroutine petsc_gmres_create(self)
 
-    class(GMRESSolver) :: self
+    class(GMRESSolver), intent(inout) :: self ! GMRES solver instance
 
     integer :: ilu_levels = 5
     real(8) :: rtol = 1.0e-10_8
@@ -102,9 +102,9 @@ contains
 
   subroutine petsc_gmres_set_oper(self, prec_mat, mat_in)
 
-    class(GMRESSolver) :: self
-    type(Matrix)       :: prec_mat
-    type(Matrix)       :: mat_in
+    class(GMRESSolver), intent(inout) :: self     ! GMRES solver instanace
+    type(Matrix), intent(inout)       :: prec_mat ! preconditioner matrix
+    type(Matrix), intent(inout)       :: mat_in   ! coefficient matrix
 
     call KSPSetOperators(self % ksp_, mat_in % petsc_mat, prec_mat % petsc_mat, &
          SAME_NONZERO_PATTERN, petsc_err)
@@ -118,7 +118,7 @@ contains
 
   subroutine petsc_gmres_destroy(self)
 
-   class(GMRESSolver) :: self
+   class(GMRESSolver), intent(inout) :: self ! GMRES solver instance
 
     call KSPDestroy(self % ksp_, petsc_err)
 
@@ -130,9 +130,9 @@ contains
 
   subroutine petsc_gmres_solve(self, b, x)
 
-    class(GMRESSolver) :: self
-    type(Vector)       :: b
-    type(Vector)       :: x
+    class(GMRESSolver), intent(inout) :: self ! GMRES solver instance
+    type(Vector), intent(inout)       :: b    ! right hand side vector
+    type(Vector), intent(inout)       :: x    ! solution vector
 
     call KSPSolve(self % ksp_, b % petsc_vec, x % petsc_vec, petsc_err)
 
@@ -144,7 +144,7 @@ contains
 
   subroutine petsc_jfnk_create(self)
 
-    class(JFNKSolver) :: self
+    class(JFNKSolver), intent(inout) :: self ! JFNK solver instance
 
     ! Turn on mf_operator option for matrix free jacobian
     call PetscOptionsSetValue("-snes_mf_operator", "TRUE", petsc_err)
@@ -176,7 +176,7 @@ contains
 
   subroutine petsc_jfnk_destroy(self)
 
-    class(JFNKSolver) :: self
+    class(JFNKSolver), intent(inout) :: self ! JFNK solver instance
 
     call MatDestroy(self % jac_mf, petsc_err)
     call SNESDestroy(self % snes_, petsc_err)
@@ -189,10 +189,10 @@ contains
 
   subroutine petsc_jfnk_set_functions(self, ctx, res, jac_prec)
 
-    class(JFNKSolver) :: self
-    type(Jfnk_ctx)    :: ctx
-    type(Vector)      :: res
-    type(Matrix)      :: jac_prec
+    class(JFNKSolver), intent(inout) :: self     ! JFNK solver instance
+    type(Jfnk_ctx),    intent(inout) :: ctx      ! JFNK context instance
+    type(Vector),      intent(inout) :: res      ! residual vector
+    type(Matrix),      intent(inout) :: jac_prec ! preconditioner matrix
 
     ! Set residual procedure
     call SNESSetFunction(self % snes_, res % petsc_vec, &
@@ -217,8 +217,8 @@ contains
 
   subroutine petsc_jfnk_solve(self, xvec)
 
-    class(JFNKSolver) :: self
-    type(Vector)      :: xvec
+    class(JFNKSolver), intent(inout) :: self ! JFNK instance
+    type(Vector), intent(inout)      :: xvec ! solution vector
 
     call SNESSolve(self % snes_, PETSC_NULL_DOUBLE, xvec % petsc_vec, petsc_err)
 
@@ -230,14 +230,14 @@ contains
 
   subroutine petsc_jfnk_compute_residual(snes_, x, res, ctx, ierr)
 
-    type(snes)     :: snes_
-    type(vec)      :: x
-    type(vec)      :: res
-    integer        :: ierr
-    type(Jfnk_ctx) :: ctx
+    type(snes), intent(inout)     :: snes_ ! PETSc SNES object
+    type(vec), intent(inout)      :: x     ! PETSc solution vector
+    type(vec), intent(inout)      :: res   ! PETSc residual vector
+    integer, intent(inout)        :: ierr  ! error code
+    type(Jfnk_ctx), intent(inout) :: ctx ! JFNK context instance
 
-    type(Vector) :: xvec
-    type(Vector) :: resvec
+    type(Vector) :: xvec   ! solution vector
+    type(Vector) :: resvec ! residual vector
 
     ! We need to use the x and res that come from PETSc because the pointer
     ! location changes and therefore we can not use module variables
@@ -263,15 +263,15 @@ contains
   subroutine petsc_jfnk_compute_jacobian(snes_, x, jac_mf, jac_prec, flag, &
              ctx, ierr)
 
-    type(snes)     :: snes_
-    type(vec)      :: x
-    type(mat)      :: jac_mf
-    type(mat)      :: jac_prec
-    integer        :: flag 
-    type(Jfnk_ctx) :: ctx
-    integer        :: ierr
+    type(snes), intent(inout)     :: snes_     ! PETSc snes instance
+    type(vec), intent(inout)      :: x         ! PETSc solution vector
+    type(mat), intent(inout)      :: jac_mf    ! PETSc matrix free jacobian
+    type(mat), intent(inout)      :: jac_prec  ! PETSc matrix jacobian precond.
+    integer, intent(inout)        :: flag      ! unused madatory flag
+    type(Jfnk_ctx), intent(inout) :: ctx       ! JFNK context instance
+    integer, intent(inout)        :: ierr      ! error code
 
-    type(Vector) :: xvec
+    type(Vector) :: xvec ! solution vector
 
     ! Again, we use the vector that comes from Petsc to build the Jacobian
     ! matrix
diff --git a/src/vector_header.F90 b/src/vector_header.F90
index 47b865ebec..3d1bd9afae 100644
--- a/src/vector_header.F90
+++ b/src/vector_header.F90
@@ -14,9 +14,9 @@ module vector_header
     real(8), allocatable :: data(:) ! where vector data is stored
     real(8), pointer :: val(:) ! pointer to vector data
 #  ifdef PETSC
-    type(vec) :: petsc_vec
+    type(vec) :: petsc_vec ! PETSc vector
 #  endif
-    logical :: petsc_active
+    logical :: petsc_active ! Logical if PETSc is being used
    contains
      procedure :: create       => vector_create
      procedure :: destroy      => vector_destroy
@@ -25,7 +25,7 @@ module vector_header
      procedure :: write_petsc_binary => vector_write_petsc_binary
   end type Vector
 
-  integer :: petsc_err
+  integer :: petsc_err ! petsc error code
 
 contains
 
@@ -35,8 +35,8 @@ contains
 
   subroutine vector_create(self, n)
 
-    integer       :: n
-    class(Vector), target :: self
+    integer, intent(in)                    :: n    ! size of vector
+    class(Vector), intent(inout), target   :: self ! vector instance
 
     ! Preallocate vector
     if (.not.allocated(self % data)) allocate(self % data(n))
@@ -59,7 +59,7 @@ contains
 
   subroutine vector_destroy(self)
 
-    class(Vector) :: self
+    class(Vector), intent(inout) :: self ! vector instance
 
 #ifdef PETSC
     if (self % petsc_active) call VecDestroy(self % petsc_vec, petsc_err)
@@ -76,9 +76,9 @@ contains
 
   subroutine vector_add_value(self, idx, val)
 
-    integer       :: idx
-    real(8)       :: val
-    class(Vector) :: self
+    integer, intent(in)          :: idx  ! index location in vector
+    real(8), intent(in)          :: val  ! value to add
+    class(Vector), intent(inout) :: self ! vector instance
 
     self % val(idx) = val
 
@@ -90,7 +90,7 @@ contains
 
   subroutine vector_setup_petsc(self)
 
-    class(Vector) :: self
+    class(Vector), intent(inout) :: self ! vector instance
 
     ! Link to PETSc 
 #ifdef PETSC
@@ -109,11 +109,11 @@ contains
 
   subroutine vector_write_petsc_binary(self, filename)
 
-    character(*) :: filename
-    class(Vector) :: self
+    character(*), intent(in)  :: filename ! name of file to write to
+    class(Vector), intent(in) :: self     ! vector instance
 
 #ifdef PETSC
-    type(PetscViewer) :: viewer
+    type(PetscViewer) :: viewer ! PETSc viewer instance
 
     call PetscViewerBinaryOpen(PETSC_COMM_WORLD, trim(filename), &
          FILE_MODE_WRITE, viewer, petsc_err)

From 80b58c2d0f4cf4701b2f456dc34d4d8a9cd23faf Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Sun, 29 Sep 2013 15:16:51 -0400
Subject: [PATCH 59/69] added space between args

---
 src/cmfd_input.F90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cmfd_input.F90 b/src/cmfd_input.F90
index a3a497b10f..d893d7bc74 100644
--- a/src/cmfd_input.F90
+++ b/src/cmfd_input.F90
@@ -35,7 +35,7 @@ contains
 
     ! Split up procs
 # ifdef PETSC 
-    call MPI_COMM_SPLIT(MPI_COMM_WORLD,color,0,new_comm,mpi_err)
+    call MPI_COMM_SPLIT(MPI_COMM_WORLD, color, 0, new_comm, mpi_err)
 # endif
 
     ! assign to PETSc
@@ -43,7 +43,7 @@ contains
     PETSC_COMM_WORLD = new_comm
 
     ! Initialize PETSc on all procs
-    call PetscInitialize(PETSC_NULL_CHARACTER,mpi_err)
+    call PetscInitialize(PETSC_NULL_CHARACTER, mpi_err)
 # endif
 
     ! Initialize timers

From 7747f798452b4498a3516a2c22f8dee41eca0967 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Sun, 29 Sep 2013 15:43:49 -0400
Subject: [PATCH 60/69] updated statepoint specs for rev. 9 and rev. 10

---
 docs/source/devguide/statepoint.rst | 538 ++++++++++++++++++++++++++++
 1 file changed, 538 insertions(+)

diff --git a/docs/source/devguide/statepoint.rst b/docs/source/devguide/statepoint.rst
index 2b486f12e0..9862a45282 100644
--- a/docs/source/devguide/statepoint.rst
+++ b/docs/source/devguide/statepoint.rst
@@ -4,6 +4,544 @@
 State Point Binary File Specifications
 ======================================
 
+-----------
+Revision 10 
+-----------
+
+**integer(4) FILETYPE_STATEPOINT**
+
+    Flags whether this file is a statepoint file or a particle restart file.
+
+**integer(4) REVISION_STATEPOINT**
+
+    Revision of the binary state point file. Any time a change is made in the
+    format of the state-point file, this integer is incremented.
+
+**integer(4) VERSION_MAJOR**
+
+    Major version number for OpenMC
+
+**integer(4) VERSION_MINOR**
+
+    Minor version number for OpenMC
+
+**integer(4) VERSION_RELEASE**
+
+    Release version number for OpenMC
+
+**character(19) time_stamp**
+
+    Date and time the state point was written.
+
+**character(255) path**
+
+    Absolute path to directory containing input files.
+
+**integer(8) seed**
+
+    Pseudo-random number generator seed.
+
+**integer(4) run_mode**
+
+    run mode used. The modes are described in constants.F90.
+
+**integer(8) n_particles**
+
+    Number of particles used per generation.
+
+**integer(4) n_batches**
+
+    Total number of batches (active + inactive).
+
+**integer(4) current_batch**
+
+    The number of batches already simulated.
+
+if (run_mode == MODE_EIGENVALUE)
+
+    **integer(4) n_inactive**
+
+        Number of inactive batches
+
+    **integer(4) gen_per_batch**
+
+        Number of generations per batch for criticality calculations
+
+    *do i = 1, current_batch \* gen_per_batch*
+
+        **real(8) k_generation(i)**
+
+             k-effective for the i-th total generation
+
+    *do i = 1, current_batch \* gen_per_batch*
+
+        **real(8) entropy(i)**
+
+            Shannon entropy for the i-th total generation
+
+    **real(8) k_col_abs**
+
+        Sum of product of collision/absorption estimates of k-effective
+
+    **real(8) k_col_tra**
+
+        Sum of product of collision/track-length estimates of k-effective
+
+    **real(8) k_abs_tra**
+
+        Sum of product of absorption/track-length estimates of k-effective
+
+    **real(8) k_combined(2)**
+
+        Mean and standard deviation of a combined estimate of k-effective
+
+    **integer(4) cmfd_on**
+
+        Flag that cmfd is on
+
+    if (cmfd_on)
+
+        **integer(4) cmfd % indices**
+
+            Indices for cmfd mesh (i,j,k,g)
+
+        **real(8) cmfd % k_cmfd(1:current_batch)**
+
+            CMFD eigenvalues
+
+        **real(8) cmfd % src(1:I,1:J,1:K,1:G)**
+
+            CMFD fission source
+
+        **real(8) cmfd % entropy(1:current_batch)**
+
+            CMFD estimate of Shannon entropy
+
+        **real(8) cmfd % balance(1:current_batch)**
+
+            RMS of the residual neutron balance equation on CMFD mesh
+
+        **real(8) cmfd % dom(1:current_batch)**
+
+            CMFD estimate of dominance ratio
+
+        **real(8) cmfd % scr_cmp(1:current_batch)**
+
+            RMS comparison of difference between OpenMC and CMFD fission source
+
+**integer(4) n_meshes**
+
+    Number of meshes in tallies.xml file
+
+*do i = 1, n_meshes*
+
+    **integer(4) meshes(i) % id**
+
+        Unique ID of mesh.
+
+    **integer(4) meshes(i) % type**
+
+        Type of mesh.
+
+    **integer(4) meshes(i) % n_dimension**
+
+        Number of dimensions for mesh (2 or 3).
+
+    **integer(4) meshes(i) % dimension(:)**
+
+        Number of mesh cells in each dimension.
+
+    **real(8) meshes(i) % lower_left(:)**
+
+        Coordinates of lower-left corner of mesh.
+
+    **real(8) meshes(i) % upper_right(:)**
+
+        Coordinates of upper-right corner of mesh.
+
+    **real(8) meshes(i) % width(:)**
+
+        Width of each mesh cell in each dimension.
+
+**integer(4) n_tallies**
+
+*do i = 1, n_tallies*
+
+    **integer(4) tallies(i) % id**
+
+        Unique ID of tally.
+
+    **integer(4) tallies(i) % n_realizations**
+
+        Number of realizations for the i-th tally.
+
+    **integer(4) size(tallies(i) % scores, 1)**
+
+        Total number of score bins for the i-th tally
+
+    **integer(4) size(tallies(i) % scores, 2)**
+
+        Total number of filter bins for the i-th tally
+
+    **integer(4) tallies(i) % n_filters**
+
+    *do j = 1, tallies(i) % n_filters*
+
+        **integer(4) tallies(i) % filter(j) % type**
+
+            Type of tally filter.
+
+        **integer(4) tallies(i) % filter(j) % n_bins**
+
+            Number of bins for filter.
+
+        **integer(4)/real(8) tallies(i) % filter(j) % bins(:)**
+
+            Value for each filter bin of this type.
+
+    **integer(4) tallies(i) % n_nuclide_bins**
+
+        Number of nuclide bins. If none are specified, this is just one.
+
+    *do j = 1, tallies(i) % n_nuclide_bins*
+
+        **integer(4) tallies(i) % nuclide_bins(j)**
+
+            Values of specified nuclide bins
+
+    **integer(4) tallies(i) % n_score_bins**
+
+        Number of scoring bins.
+
+    *do j = 1, tallies(i) % n_score_bins*
+
+        **integer(4) tallies(i) % score_bins(j)**
+
+            Values of specified scoring bins (e.g. SCORE_FLUX).
+            
+    *do j = 1, tallies(i) % n_score_bins*
+
+        **integer(4) tallies(i) % scatt_order(j)**
+
+            Scattering Order specified scoring bins.
+            
+    **integer(4) tallies(i) % n_score_bins**
+
+        Number of scoring bins without accounting for those added by
+        the scatter-pn command.
+
+**integer(4) n_realizations**
+
+    Number of realizations for global tallies.
+
+**integer(4) N_GLOBAL_TALLIES**
+
+    Number of global tally scores
+
+*do i = 1, N_GLOBAL_TALLIES*
+
+    **real(8) global_tallies(i) % sum**
+
+        Accumulated sum for the i-th global tally
+
+    **real(8) global_tallies(i) % sum_sq**
+
+        Accumulated sum of squares for the i-th global tally
+
+**integer(4) tallies_on**
+
+    Flag indicated if tallies are present in the file.
+
+if (tallies_on > 0)
+
+    *do i = 1, n_tallies*
+
+        *do k = 1, size(tallies(i) % scores, 2)*
+
+            *do j = 1, size(tallies(i) % scores, 1)*
+
+                **real(8) tallies(i) % scores(j,k) % sum**
+            
+                    Accumulated sum for the j-th score and k-th filter of the
+                    i-th tally
+
+                **real(8) tallies(i) % scores(j,k) % sum_sq**
+
+                    Accumulated sum of squares for the j-th score and k-th
+                    filter of the i-th tally
+
+if (run_mode == MODE_EIGENVALUE)
+
+    *do i = 1, n_particles*
+
+        **real(8) source_bank(i) % wgt**
+
+            Weight of the i-th source particle
+
+        **real(8) source_bank(i) % xyz(1:3)**
+
+            Coordinates of the i-th source particle.
+
+        **real(8) source_bank(i) % uvw(1:3)**
+
+            Direction of the i-th source particle
+
+        **real(8) source_bank(i) % E**
+
+            Energy of the i-th source particle.
+
+----------
+Revision 9
+----------
+
+**integer(4) FILETYPE_STATEPOINT**
+
+    Flags whether this file is a statepoint file or a particle restart file.
+
+**integer(4) REVISION_STATEPOINT**
+
+    Revision of the binary state point file. Any time a change is made in the
+    format of the state-point file, this integer is incremented.
+
+**integer(4) VERSION_MAJOR**
+
+    Major version number for OpenMC
+
+**integer(4) VERSION_MINOR**
+
+    Minor version number for OpenMC
+
+**integer(4) VERSION_RELEASE**
+
+    Release version number for OpenMC
+
+**character(19) time_stamp**
+
+    Date and time the state point was written.
+
+**character(255) path**
+
+    Absolute path to directory containing input files.
+
+**integer(8) seed**
+
+    Pseudo-random number generator seed.
+
+**integer(4) run_mode**
+
+    run mode used. The modes are described in constants.F90.
+
+**integer(8) n_particles**
+
+    Number of particles used per generation.
+
+**integer(4) n_batches**
+
+    Total number of batches (active + inactive).
+
+**integer(4) current_batch**
+
+    The number of batches already simulated.
+
+if (run_mode == MODE_EIGENVALUE)
+
+    **integer(4) n_inactive**
+
+        Number of inactive batches
+
+    **integer(4) gen_per_batch**
+
+        Number of generations per batch for criticality calculations
+
+    *do i = 1, current_batch \* gen_per_batch*
+
+        **real(8) k_generation(i)**
+
+             k-effective for the i-th total generation
+
+    *do i = 1, current_batch \* gen_per_batch*
+
+        **real(8) entropy(i)**
+
+            Shannon entropy for the i-th total generation
+
+    **real(8) k_col_abs**
+
+        Sum of product of collision/absorption estimates of k-effective
+
+    **real(8) k_col_tra**
+
+        Sum of product of collision/track-length estimates of k-effective
+
+    **real(8) k_abs_tra**
+
+        Sum of product of absorption/track-length estimates of k-effective
+
+    **real(8) k_combined(2)**
+
+        Mean and standard deviation of a combined estimate of k-effective
+
+**integer(4) n_meshes**
+
+    Number of meshes in tallies.xml file
+
+*do i = 1, n_meshes*
+
+    **integer(4) meshes(i) % id**
+
+        Unique ID of mesh.
+
+    **integer(4) meshes(i) % type**
+
+        Type of mesh.
+
+    **integer(4) meshes(i) % n_dimension**
+
+        Number of dimensions for mesh (2 or 3).
+
+    **integer(4) meshes(i) % dimension(:)**
+
+        Number of mesh cells in each dimension.
+
+    **real(8) meshes(i) % lower_left(:)**
+
+        Coordinates of lower-left corner of mesh.
+
+    **real(8) meshes(i) % upper_right(:)**
+
+        Coordinates of upper-right corner of mesh.
+
+    **real(8) meshes(i) % width(:)**
+
+        Width of each mesh cell in each dimension.
+
+**integer(4) n_tallies**
+
+*do i = 1, n_tallies*
+
+    **integer(4) tallies(i) % id**
+
+        Unique ID of tally.
+
+    **integer(4) tallies(i) % n_realizations**
+
+        Number of realizations for the i-th tally.
+
+    **integer(4) size(tallies(i) % scores, 1)**
+
+        Total number of score bins for the i-th tally
+
+    **integer(4) size(tallies(i) % scores, 2)**
+
+        Total number of filter bins for the i-th tally
+
+    **integer(4) tallies(i) % n_filters**
+
+    *do j = 1, tallies(i) % n_filters*
+
+        **integer(4) tallies(i) % filter(j) % type**
+
+            Type of tally filter.
+
+        **integer(4) tallies(i) % filter(j) % n_bins**
+
+            Number of bins for filter.
+
+        **integer(4)/real(8) tallies(i) % filter(j) % bins(:)**
+
+            Value for each filter bin of this type.
+
+    **integer(4) tallies(i) % n_nuclide_bins**
+
+        Number of nuclide bins. If none are specified, this is just one.
+
+    *do j = 1, tallies(i) % n_nuclide_bins*
+
+        **integer(4) tallies(i) % nuclide_bins(j)**
+
+            Values of specified nuclide bins
+
+    **integer(4) tallies(i) % n_score_bins**
+
+        Number of scoring bins.
+
+    *do j = 1, tallies(i) % n_score_bins*
+
+        **integer(4) tallies(i) % score_bins(j)**
+
+            Values of specified scoring bins (e.g. SCORE_FLUX).
+            
+    *do j = 1, tallies(i) % n_score_bins*
+
+        **integer(4) tallies(i) % scatt_order(j)**
+
+            Scattering Order specified scoring bins.
+            
+    **integer(4) tallies(i) % n_score_bins**
+
+        Number of scoring bins without accounting for those added by
+        the scatter-pn command.
+
+**integer(4) n_realizations**
+
+    Number of realizations for global tallies.
+
+**integer(4) N_GLOBAL_TALLIES**
+
+    Number of global tally scores
+
+*do i = 1, N_GLOBAL_TALLIES*
+
+    **real(8) global_tallies(i) % sum**
+
+        Accumulated sum for the i-th global tally
+
+    **real(8) global_tallies(i) % sum_sq**
+
+        Accumulated sum of squares for the i-th global tally
+
+**integer(4) tallies_on**
+
+    Flag indicated if tallies are present in the file.
+
+if (tallies_on > 0)
+
+    *do i = 1, n_tallies*
+
+        *do k = 1, size(tallies(i) % scores, 2)*
+
+            *do j = 1, size(tallies(i) % scores, 1)*
+
+                **real(8) tallies(i) % scores(j,k) % sum**
+            
+                    Accumulated sum for the j-th score and k-th filter of the
+                    i-th tally
+
+                **real(8) tallies(i) % scores(j,k) % sum_sq**
+
+                    Accumulated sum of squares for the j-th score and k-th
+                    filter of the i-th tally
+
+if (run_mode == MODE_EIGENVALUE)
+
+    *do i = 1, n_particles*
+
+        **real(8) source_bank(i) % wgt**
+
+            Weight of the i-th source particle
+
+        **real(8) source_bank(i) % xyz(1:3)**
+
+            Coordinates of the i-th source particle.
+
+        **real(8) source_bank(i) % uvw(1:3)**
+
+            Direction of the i-th source particle
+
+        **real(8) source_bank(i) % E**
+
+            Energy of the i-th source particle.
+
 ----------
 Revision 8
 ----------

From 0f95311f4766d0af59590f40af3f8ee80d359d57 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Sun, 29 Sep 2013 15:48:37 -0400
Subject: [PATCH 61/69] clarified HDF5 installation

---
 docs/source/usersguide/install.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/source/usersguide/install.rst b/docs/source/usersguide/install.rst
index aa257592b7..84f7633c2f 100644
--- a/docs/source/usersguide/install.rst
+++ b/docs/source/usersguide/install.rst
@@ -71,13 +71,15 @@ Prerequisites
       To compile with support for HDF5_ output (highly recommended), you will
       need to have HDF5 installed on your computer. The installed version will
       need to have been compiled with the same compiler you intend to compile
-      OpenMC with. HDF5_ must be built with parallel I/O features. An example
-      of configuring HDF5_ is listed below::
+      OpenMC with. HDF5_ must be built with parallel I/O features if you intend
+      to use HDF5_ with MPI. An example of configuring HDF5_ is listed below::
 
            FC=/opt/mpich/3.0.4-gnu/bin/mpif90 CC=/opt/mpich/3.0.4-gnu/bin/mpicc \
            ./configure --prefix=/opt/hdf5/1.8.11-gnu --enable-fortran \
                        --enable-fortran2003 --enable-parallel
 
+      You may omit '--enable-parallel' if you want to compile HDF5_ in serial.
+
     * PETSc_ for CMFD acceleration
 
       To enable CMFD acceleration, you will need to have PETSc_ (3.4.2 or higher)

From 99d9118ecaf04d083fea2fccdfda1ee3fc35fe72 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Sun, 29 Sep 2013 17:21:10 -0400
Subject: [PATCH 62/69] updated omp tests for all combinations

---
 tests/run_tests.py                 | 35 +++++++-----
 tests/test_compile/test_compile.py | 87 ++++++++++++++++++++++++++++--
 2 files changed, 107 insertions(+), 15 deletions(-)

diff --git a/tests/run_tests.py b/tests/run_tests.py
index 30d330e5d5..4edc59630b 100755
--- a/tests/run_tests.py
+++ b/tests/run_tests.py
@@ -67,12 +67,17 @@ sys.path.append(pwd)
 
 # Set list of tests, either default or from command line
 flags = []
-tests = ['compile', 'normal', 'debug', 'optimize',  'hdf5', 'hdf5-debug',
-         'hdf5-optimize', 'mpi', 'mpi-debug', 'mpi-optimize', 'omp',
-         'phdf5', 'phdf5-debug', 'phdf5-optimize',  'mpi-omp', 'omp-hdf5',
-         'petsc', 'petsc-debug', 'petsc-optimize', 'phdf5-petsc',
-         'phdf5-petsc-debug', 'phdf5-petsc-optimize',
-         'omp-phdf5-petsc-optimize']
+tests = ['compile', 'normal', 'debug', 'optimize',
+         'omp', 'omp-debug', 'omp-optimize',
+         'hdf5', 'hdf5-debug', 'hdf5-optimize',
+         'omp-hdf5', 'omp-hdf5-debug', 'omp-hdf5-optimize',
+         'mpi', 'mpi-debug', 'mpi-optimize',
+         'mpi-omp', 'mpi-omp-debug', 'mpi-omp-optimize',
+         'phdf5', 'phdf5-debug', 'phdf5-optimize',
+         'phdf5-omp', 'phdf5-omp-debug', 'phdf5-omp-optimize',
+         'petsc', 'petsc-debug', 'petsc-optimize',
+         'phdf5-petsc', 'phdf5-petsc-debug', 'phdf5-petsc-optimize',
+         'omp-phdf5-petsc', 'omp-phdf5-petsc-debug', 'omp-phdf5-petsc-optimize']
 if len(sys.argv) > 1:
     flags = [i for i in sys.argv[1:] if i.startswith('-')]
     tests_ = [i for i in sys.argv[1:] if not i.startswith('-')]
@@ -117,13 +122,19 @@ results = []
 for name in tests:
     if name == 'compile':
         run_compile()
-    elif name in ['normal', 'debug', 'optimize', 'omp'
-                  'hdf5', 'hdf5-debug', 'hdf5-optimize', 'omp-hdf5']:
+    elif name in ['normal', 'debug', 'optimize',
+                  'hdf5', 'hdf5-debug', 'hdf5-optimize',
+                  'omp', 'omp-debug', 'omp-optimize',
+                  'omp-hdf5', 'omp-hdf5-debug', 'omp-hdf5-optimize']:
         run_suite(name=name)
-    elif name in ['mpi', 'mpi-debug', 'mpi-optimize', 'mpi-omp', 'phdf5',
-                  'phdf5-debug', 'phdf5-optimize', 'petsc', 'petsc-debug',
-                  'petsc-optimize', 'phdf5-petsc', 'phdf5-petsc-debug',
-                  'phdf5-petsc-optimize', 'omp-phdf5-petsc-optimize']:
+    elif name in ['mpi', 'mpi-debug', 'mpi-optimize',
+                  'mpi-omp', 'mpi-omp-debug', 'mpi-omp-optimize',
+                  'phdf5', 'phdf5-debug', 'phdf5-optimize',
+                  'phdf5-omp', 'phdf5-omp-debug', 'phdf5-omp-optimize',
+                  'petsc', 'petsc-debug', 'petsc-optimize',
+                  'phdf5-petsc', 'phdf5-petsc-debug', 'phdf5-petsc-optimize',
+                  'omp-phdf5-petsc', 'omp-phdf5-petsc-debug',
+                   'omp-phdf5-petsc-optimize']:
         run_suite(name=name, mpi=True)
 
 # print out summary of results
diff --git a/tests/test_compile/test_compile.py b/tests/test_compile/test_compile.py
index 997f243d7b..ced2a86321 100644
--- a/tests/test_compile/test_compile.py
+++ b/tests/test_compile/test_compile.py
@@ -81,6 +81,20 @@ def test_omp():
     shutil.move('openmc', 'openmc-omp')
 
 
+def test_omp_debug():
+    returncode = run(['make', 'distclean'])
+    returncode = run(['make', compiler, 'OPENMP=yes', 'DEBUG=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-omp-debug')
+
+
+def test_omp_optimize():
+    returncode = run(['make', 'distclean'])
+    returncode = run(['make', compiler, 'OPENMP=yes', 'OPTIMIZE=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-omp-optimize')
+
+
 def test_hdf5():
     returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'HDF5=yes'])
@@ -102,6 +116,27 @@ def test_hdf5_optimize():
     shutil.move('openmc', 'openmc-hdf5-optimize')
 
 
+def test_omp_hdf5():
+    returncode = run(['make', 'distclean'])
+    returncode = run(['make', compiler, 'OPENMP=yes', 'HDF5=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-omp-hdf5')
+
+
+def test_omp_hdf5_debug():
+    returncode = run(['make', 'distclean'])
+    returncode = run(['make', compiler, 'OPENMP=yes', 'HDF5=yes', 'DEBUG=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-omp-hdf5-debug')
+
+
+def test_omp_hdf5_optimize():
+    returncode = run(['make', 'distclean'])
+    returncode = run(['make', compiler, 'OPENMP=yes', 'HDF5=yes', 'OPTIMIZE=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-omp-hdf5-optimize')
+
+
 def test_petsc():
     returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'PETSC=yes'])
@@ -131,11 +166,18 @@ def test_mpi_omp():
     shutil.move('openmc', 'openmc-mpi-omp')
 
 
-def test_omp_hdf5():
+def test_mpi_omp_debug():
     returncode = run(['make', 'distclean'])
-    returncode = run(['make', compiler, 'MPI=yes', 'OPENMP=yes', 'HDF5=yes'])
+    returncode = run(['make', compiler, 'MPI=yes', 'OPENMP=yes', 'DEBUG=yes'])
     assert returncode == 0
-    shutil.move('openmc', 'openmc-omp-hdf5')
+    shutil.move('openmc', 'openmc-mpi-omp-debug')
+
+
+def test_mpi_omp_optimize():
+    returncode = run(['make', 'distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'OPENMP=yes', 'OPTIMIZE=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-mpi-omp-optimize')
 
 
 def test_mpi_hdf5():
@@ -159,6 +201,29 @@ def test_mpi_hdf5_optimize():
     shutil.move('openmc', 'openmc-phdf5-optimize')
 
 
+def test_mpi_omp_hdf5():
+    returncode = run(['make', 'distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'OPENMP=yes', 'HDF5=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-phdf5-omp')
+
+
+def test_mpi_omp_hdf5_debug():
+    returncode = run(['make', 'distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'OPENMP=yes', 'HDF5=yes',
+                      'DEBUG=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-phdf5-omp-debug')
+
+
+def test_mpi_omp_hdf5_optimize():
+    returncode = run(['make', 'distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'OPENMP=yes', 'HDF5=yes',
+                      'OPTIMIZE=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-phdf5-omp-optimize')
+
+
 def test_mpi_hdf5_petsc():
     returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'HDF5=yes', 'PETSC=yes'])
@@ -182,6 +247,22 @@ def test_mpi_hdf5_petsc_optimize():
     shutil.move('openmc', 'openmc-phdf5-petsc-optimize')
 
 
+def test_mpi_omp_hdf5_petsc():
+    returncode = run(['make', 'distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'OMP=yes', 'HDF5=yes',
+                      'PETSC=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-omp-phdf5-petsc')
+
+
+def test_mpi_omp_hdf5_petsc_debug():
+    returncode = run(['make', 'distclean'])
+    returncode = run(['make', compiler, 'MPI=yes', 'OMP=yes', 'HDF5=yes',
+                      'PETSC=yes', 'DEBUG=yes'])
+    assert returncode == 0
+    shutil.move('openmc', 'openmc-omp-phdf5-petsc-debug')
+
+
 def test_mpi_omp_hdf5_petsc_optimize():
     returncode = run(['make', 'distclean'])
     returncode = run(['make', compiler, 'MPI=yes', 'OMP=yes', 'HDF5=yes',

From f1780faea2daf48744d356a41314e336d59ab214 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Sun, 29 Sep 2013 17:31:44 -0400
Subject: [PATCH 63/69] dont add compile test when specifying all-omp

---
 tests/run_tests.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/run_tests.py b/tests/run_tests.py
index 4edc59630b..cbb80d2686 100755
--- a/tests/run_tests.py
+++ b/tests/run_tests.py
@@ -112,6 +112,8 @@ if len(sys.argv) > 1:
             else:
                 for j in tests:
                     if j.rfind(suffix) != -1:
+                        if suffix == 'omp' and j == 'compile':
+                            continue
                         tests__.append(j)
         else:
             tests__.append(i)  # append specific test (e.g., mpi-debug)

From 260d051167ca60b896ab59c530a4d5f6f7f77c08 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Wed, 9 Oct 2013 16:13:59 -0400
Subject: [PATCH 64/69] changed dummy runtests to nosetests and fixed exception
 handling

---
 tests/run_tests.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/tests/run_tests.py b/tests/run_tests.py
index cbb80d2686..3da02c8998 100755
--- a/tests/run_tests.py
+++ b/tests/run_tests.py
@@ -22,7 +22,7 @@ def run_compile():
         os.remove(exe)
 
     # run compile test
-    result = nose.run(argv=['run_tests.py', 'test_compile'] + flags)
+    result = nose.run(argv=['nosetests', 'test_compile'] + flags)
     if not result:
         print('Did not pass compile tests.')
     results.append(('compile', result))
@@ -35,7 +35,7 @@ def run_suite(name=None, mpi=False):
 
     # Set arguments list. Note that the first argument is a dummy argument (the
     # script name). It's not actually recursively calling run_tests.py
-    argv = ['run_tests.py', '--exclude', 'test_compile'] + flags
+    argv = ['nosetests', '--exclude', 'test_compile'] + flags
 
     # Add MPI plugin if set
     if mpi:
@@ -48,11 +48,12 @@ def run_suite(name=None, mpi=False):
         os.chdir(pwd)
         os.rename(pwd + '/../src/openmc-' + name, pwd + '/../src/openmc')
         result = nose.run(argv=argv, addplugins=plugins)
-    finally:
         os.rename(pwd + '/../src/openmc', pwd + '/../src/openmc-' + name)
-        if not result:
-            print('Did not pass ' + name + ' tests')
-        results.append((name, result))
+    except OSError:
+        result = False
+    if not result:
+        print('Did not pass ' + name + ' tests')
+    results.append((name, result))
 
 # set mpiexec path
 if 'COMPILER' in os.environ:

From 3f47b31784748f9aa5bdf325275cfe69416c37b0 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Wed, 9 Oct 2013 16:20:19 -0400
Subject: [PATCH 65/69] added more info if compiler is not found

---
 tests/run_tests.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/run_tests.py b/tests/run_tests.py
index 3da02c8998..ec0091f162 100755
--- a/tests/run_tests.py
+++ b/tests/run_tests.py
@@ -51,6 +51,7 @@ def run_suite(name=None, mpi=False):
         os.rename(pwd + '/../src/openmc', pwd + '/../src/openmc-' + name)
     except OSError:
         result = False
+        print('No OpenMC executable found for ' + name + ' tests')
     if not result:
         print('Did not pass ' + name + ' tests')
     results.append((name, result))

From 1bfbde7cf07625ab153b6b44e44cd3e389b53c84 Mon Sep 17 00:00:00 2001
From: Paul Romano <paul.k.romano@gmail.com>
Date: Wed, 9 Oct 2013 19:42:36 -0400
Subject: [PATCH 66/69] PEP8 fixes on Python scripts.

---
 tests/run_tests.py                         |  5 +++--
 tests/test_cmfd_feed/results.py            | 26 +++++++++++-----------
 tests/test_cmfd_feed/test_cmfd_feed.py     | 11 ++++++---
 tests/test_cmfd_jfnk/results.py            | 26 +++++++++++-----------
 tests/test_cmfd_jfnk/test_cmfd_jfnk.py     | 11 ++++++---
 tests/test_cmfd_nofeed/results.py          | 26 +++++++++++-----------
 tests/test_cmfd_nofeed/test_cmfd_nofeed.py | 11 ++++++---
 7 files changed, 66 insertions(+), 50 deletions(-)

diff --git a/tests/run_tests.py b/tests/run_tests.py
index ec0091f162..49975c218a 100755
--- a/tests/run_tests.py
+++ b/tests/run_tests.py
@@ -79,7 +79,8 @@ tests = ['compile', 'normal', 'debug', 'optimize',
          'phdf5-omp', 'phdf5-omp-debug', 'phdf5-omp-optimize',
          'petsc', 'petsc-debug', 'petsc-optimize',
          'phdf5-petsc', 'phdf5-petsc-debug', 'phdf5-petsc-optimize',
-         'omp-phdf5-petsc', 'omp-phdf5-petsc-debug', 'omp-phdf5-petsc-optimize']
+         'omp-phdf5-petsc', 'omp-phdf5-petsc-debug',
+         'omp-phdf5-petsc-optimize']
 if len(sys.argv) > 1:
     flags = [i for i in sys.argv[1:] if i.startswith('-')]
     tests_ = [i for i in sys.argv[1:] if not i.startswith('-')]
@@ -138,7 +139,7 @@ for name in tests:
                   'petsc', 'petsc-debug', 'petsc-optimize',
                   'phdf5-petsc', 'phdf5-petsc-debug', 'phdf5-petsc-optimize',
                   'omp-phdf5-petsc', 'omp-phdf5-petsc-debug',
-                   'omp-phdf5-petsc-optimize']:
+                  'omp-phdf5-petsc-optimize']:
         run_suite(name=name, mpi=True)
 
 # print out summary of results
diff --git a/tests/test_cmfd_feed/results.py b/tests/test_cmfd_feed/results.py
index f86aaa53e3..26bcb36d9f 100644
--- a/tests/test_cmfd_feed/results.py
+++ b/tests/test_cmfd_feed/results.py
@@ -34,7 +34,7 @@ results4 = np.reshape(results4, size4)
 
 # set up output string
 outstr = ''
- 
+
 # write out k-combined
 outstr += 'k-combined:\n'
 outstr += "{0:12.6E} {1:12.6E}\n".format(sp.k_combined[0], sp.k_combined[1])
@@ -42,41 +42,41 @@ outstr += "{0:12.6E} {1:12.6E}\n".format(sp.k_combined[0], sp.k_combined[1])
 # write out tally results
 outstr += 'tally 1:\n'
 for item in results1:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'tally 2:\n'
 for item in results2:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'tally 3:\n'
 for item in results3:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'tally 4:\n'
 for item in results4:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 
 # write out cmfd answers
 outstr += 'cmfd indices\n'
 for item in sp.cmfd_indices:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'k cmfd\n'
 for item in sp.k_cmfd:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd entropy\n'
 for item in sp.cmfd_entropy:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd balance\n'
 for item in sp.cmfd_balance:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd dominance ratio\n'
 for item in sp.cmfd_dominance:
-  outstr += "{0:10.3E}\n".format(item)
+    outstr += "{0:10.3E}\n".format(item)
 outstr += 'cmfd openmc source comparison\n'
 for item in sp.cmfd_srccmp:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd source\n'
 cmfdsrc = np.reshape(sp.cmfd_src, np.product(sp.cmfd_indices), order='F')
 for item in cmfdsrc:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 
 # write results to file
-with open('results_test.dat','w') as fh:
+with open('results_test.dat', 'w') as fh:
     fh.write(outstr)
diff --git a/tests/test_cmfd_feed/test_cmfd_feed.py b/tests/test_cmfd_feed/test_cmfd_feed.py
index db36784888..59bcad866b 100644
--- a/tests/test_cmfd_feed/test_cmfd_feed.py
+++ b/tests/test_cmfd_feed/test_cmfd_feed.py
@@ -14,17 +14,19 @@ pwd = os.path.dirname(__file__)
 
 skipAll = False
 
+
 def setup():
     os.putenv('PWD', pwd)
     os.chdir(pwd)
     global skipAll
     skipAll = False
 
+
 def test_run():
     openmc_path = pwd + '/../../src/openmc'
     if int(NoseMPI.mpi_np) > 0:
         proc = Popen([NoseMPI.mpi_exec, '-np', NoseMPI.mpi_np, openmc_path],
-               stderr=STDOUT, stdout=PIPE)
+                     stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
     returncode = proc.wait()
@@ -36,6 +38,7 @@ def test_run():
         raise SkipTest
     assert returncode == 0
 
+
 def test_created_statepoint():
     if skipAll:
         raise SkipTest
@@ -43,11 +46,13 @@ def test_created_statepoint():
     assert len(statepoint) == 1
     assert statepoint[0].endswith('binary') or statepoint[0].endswith('h5')
 
+
 def test_output_exists():
     if skipAll:
         raise SkipTest
     assert os.path.exists(pwd + '/tallies.out')
 
+
 def test_results():
     if skipAll:
         raise SkipTest
@@ -55,9 +60,10 @@ def test_results():
     call(['python', 'results.py', statepoint[0]])
     compare = filecmp.cmp('results_test.dat', 'results_true.dat')
     if not compare:
-      os.rename('results_test.dat', 'results_error.dat')
+        os.rename('results_test.dat', 'results_error.dat')
     assert compare
 
+
 def teardown():
     output = glob.glob(pwd + '/statepoint.20.*')
     output.append(pwd + '/tallies.out')
@@ -65,4 +71,3 @@ def teardown():
     for f in output:
         if os.path.exists(f):
             os.remove(f)
-
diff --git a/tests/test_cmfd_jfnk/results.py b/tests/test_cmfd_jfnk/results.py
index f86aaa53e3..26bcb36d9f 100644
--- a/tests/test_cmfd_jfnk/results.py
+++ b/tests/test_cmfd_jfnk/results.py
@@ -34,7 +34,7 @@ results4 = np.reshape(results4, size4)
 
 # set up output string
 outstr = ''
- 
+
 # write out k-combined
 outstr += 'k-combined:\n'
 outstr += "{0:12.6E} {1:12.6E}\n".format(sp.k_combined[0], sp.k_combined[1])
@@ -42,41 +42,41 @@ outstr += "{0:12.6E} {1:12.6E}\n".format(sp.k_combined[0], sp.k_combined[1])
 # write out tally results
 outstr += 'tally 1:\n'
 for item in results1:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'tally 2:\n'
 for item in results2:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'tally 3:\n'
 for item in results3:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'tally 4:\n'
 for item in results4:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 
 # write out cmfd answers
 outstr += 'cmfd indices\n'
 for item in sp.cmfd_indices:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'k cmfd\n'
 for item in sp.k_cmfd:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd entropy\n'
 for item in sp.cmfd_entropy:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd balance\n'
 for item in sp.cmfd_balance:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd dominance ratio\n'
 for item in sp.cmfd_dominance:
-  outstr += "{0:10.3E}\n".format(item)
+    outstr += "{0:10.3E}\n".format(item)
 outstr += 'cmfd openmc source comparison\n'
 for item in sp.cmfd_srccmp:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd source\n'
 cmfdsrc = np.reshape(sp.cmfd_src, np.product(sp.cmfd_indices), order='F')
 for item in cmfdsrc:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 
 # write results to file
-with open('results_test.dat','w') as fh:
+with open('results_test.dat', 'w') as fh:
     fh.write(outstr)
diff --git a/tests/test_cmfd_jfnk/test_cmfd_jfnk.py b/tests/test_cmfd_jfnk/test_cmfd_jfnk.py
index db36784888..59bcad866b 100644
--- a/tests/test_cmfd_jfnk/test_cmfd_jfnk.py
+++ b/tests/test_cmfd_jfnk/test_cmfd_jfnk.py
@@ -14,17 +14,19 @@ pwd = os.path.dirname(__file__)
 
 skipAll = False
 
+
 def setup():
     os.putenv('PWD', pwd)
     os.chdir(pwd)
     global skipAll
     skipAll = False
 
+
 def test_run():
     openmc_path = pwd + '/../../src/openmc'
     if int(NoseMPI.mpi_np) > 0:
         proc = Popen([NoseMPI.mpi_exec, '-np', NoseMPI.mpi_np, openmc_path],
-               stderr=STDOUT, stdout=PIPE)
+                     stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
     returncode = proc.wait()
@@ -36,6 +38,7 @@ def test_run():
         raise SkipTest
     assert returncode == 0
 
+
 def test_created_statepoint():
     if skipAll:
         raise SkipTest
@@ -43,11 +46,13 @@ def test_created_statepoint():
     assert len(statepoint) == 1
     assert statepoint[0].endswith('binary') or statepoint[0].endswith('h5')
 
+
 def test_output_exists():
     if skipAll:
         raise SkipTest
     assert os.path.exists(pwd + '/tallies.out')
 
+
 def test_results():
     if skipAll:
         raise SkipTest
@@ -55,9 +60,10 @@ def test_results():
     call(['python', 'results.py', statepoint[0]])
     compare = filecmp.cmp('results_test.dat', 'results_true.dat')
     if not compare:
-      os.rename('results_test.dat', 'results_error.dat')
+        os.rename('results_test.dat', 'results_error.dat')
     assert compare
 
+
 def teardown():
     output = glob.glob(pwd + '/statepoint.20.*')
     output.append(pwd + '/tallies.out')
@@ -65,4 +71,3 @@ def teardown():
     for f in output:
         if os.path.exists(f):
             os.remove(f)
-
diff --git a/tests/test_cmfd_nofeed/results.py b/tests/test_cmfd_nofeed/results.py
index f86aaa53e3..26bcb36d9f 100644
--- a/tests/test_cmfd_nofeed/results.py
+++ b/tests/test_cmfd_nofeed/results.py
@@ -34,7 +34,7 @@ results4 = np.reshape(results4, size4)
 
 # set up output string
 outstr = ''
- 
+
 # write out k-combined
 outstr += 'k-combined:\n'
 outstr += "{0:12.6E} {1:12.6E}\n".format(sp.k_combined[0], sp.k_combined[1])
@@ -42,41 +42,41 @@ outstr += "{0:12.6E} {1:12.6E}\n".format(sp.k_combined[0], sp.k_combined[1])
 # write out tally results
 outstr += 'tally 1:\n'
 for item in results1:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'tally 2:\n'
 for item in results2:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'tally 3:\n'
 for item in results3:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'tally 4:\n'
 for item in results4:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 
 # write out cmfd answers
 outstr += 'cmfd indices\n'
 for item in sp.cmfd_indices:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'k cmfd\n'
 for item in sp.k_cmfd:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd entropy\n'
 for item in sp.cmfd_entropy:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd balance\n'
 for item in sp.cmfd_balance:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd dominance ratio\n'
 for item in sp.cmfd_dominance:
-  outstr += "{0:10.3E}\n".format(item)
+    outstr += "{0:10.3E}\n".format(item)
 outstr += 'cmfd openmc source comparison\n'
 for item in sp.cmfd_srccmp:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 outstr += 'cmfd source\n'
 cmfdsrc = np.reshape(sp.cmfd_src, np.product(sp.cmfd_indices), order='F')
 for item in cmfdsrc:
-  outstr += "{0:12.6E}\n".format(item)
+    outstr += "{0:12.6E}\n".format(item)
 
 # write results to file
-with open('results_test.dat','w') as fh:
+with open('results_test.dat', 'w') as fh:
     fh.write(outstr)
diff --git a/tests/test_cmfd_nofeed/test_cmfd_nofeed.py b/tests/test_cmfd_nofeed/test_cmfd_nofeed.py
index feb4a0657e..07176a8f2e 100644
--- a/tests/test_cmfd_nofeed/test_cmfd_nofeed.py
+++ b/tests/test_cmfd_nofeed/test_cmfd_nofeed.py
@@ -13,17 +13,19 @@ pwd = os.path.dirname(__file__)
 
 skipAll = False
 
+
 def setup():
     os.putenv('PWD', pwd)
     os.chdir(pwd)
     global skipAll
     skipAll = False
 
+
 def test_run():
     openmc_path = pwd + '/../../src/openmc'
     if int(NoseMPI.mpi_np) > 0:
         proc = Popen([NoseMPI.mpi_exec, '-np', NoseMPI.mpi_np, openmc_path],
-               stderr=STDOUT, stdout=PIPE)
+                     stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
     returncode = proc.wait()
@@ -35,6 +37,7 @@ def test_run():
         raise SkipTest
     assert returncode == 0
 
+
 def test_created_statepoint():
     if skipAll:
         raise SkipTest
@@ -42,11 +45,13 @@ def test_created_statepoint():
     assert len(statepoint) == 1
     assert statepoint[0].endswith('binary') or statepoint[0].endswith('h5')
 
+
 def test_output_exists():
     if skipAll:
         raise SkipTest
     assert os.path.exists(pwd + '/tallies.out')
 
+
 def test_results():
     if skipAll:
         raise SkipTest
@@ -54,9 +59,10 @@ def test_results():
     call(['python', 'results.py', statepoint[0]])
     compare = filecmp.cmp('results_test.dat', 'results_true.dat')
     if not compare:
-      os.rename('results_test.dat', 'results_error.dat')
+        os.rename('results_test.dat', 'results_error.dat')
     assert compare
 
+
 def teardown():
     output = glob.glob(pwd + '/statepoint.20.*')
     output.append(pwd + '/tallies.out')
@@ -64,4 +70,3 @@ def teardown():
     for f in output:
         if os.path.exists(f):
             os.remove(f)
-

From 241b6e567c4b920c30a96c321d3c4801255c9088 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Wed, 9 Oct 2013 21:20:55 -0400
Subject: [PATCH 67/69] residual vector should be intent inout

---
 src/cmfd_jfnk_solver.F90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cmfd_jfnk_solver.F90 b/src/cmfd_jfnk_solver.F90
index 958a4c3e56..7bd21a8758 100644
--- a/src/cmfd_jfnk_solver.F90
+++ b/src/cmfd_jfnk_solver.F90
@@ -267,8 +267,8 @@ contains
 
     use global,       only: cmfd_write_matrices
 
-    type(Vector), intent(in)  :: x   ! solution vector
-    type(Vector), intent(out) :: res ! residual vector
+    type(Vector), intent(in)    :: x   ! solution vector
+    type(Vector), intent(inout) :: res ! residual vector
 
     character(len=25) :: filename
     integer           :: n

From f7aacaf0a7054b0b2b63bf405f79650e269a31c1 Mon Sep 17 00:00:00 2001
From: Paul Romano <paul.k.romano@gmail.com>
Date: Wed, 9 Oct 2013 21:31:13 -0400
Subject: [PATCH 68/69] Change intent in res_interface to match previous
 commit.

---
 src/solver_interface.F90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/solver_interface.F90 b/src/solver_interface.F90
index 113e3119af..b629a0789f 100644
--- a/src/solver_interface.F90
+++ b/src/solver_interface.F90
@@ -56,8 +56,8 @@ module solver_interface
   abstract interface
     subroutine res_interface(x, res)
       import :: Vector
-      type(Vector), intent(in)  :: x   ! solution vector
-      type(Vector), intent(out) :: res ! residual vector
+      type(Vector), intent(in)    :: x   ! solution vector
+      type(Vector), intent(inout) :: res ! residual vector
     end subroutine res_interface
 
     subroutine jac_interface(x)

From 9025c729c720fef5511a96205f1af079b700be07 Mon Sep 17 00:00:00 2001
From: Bryan Herman <bherman@mit.edu>
Date: Wed, 16 Oct 2013 23:03:40 -0400
Subject: [PATCH 69/69] removed Popen wait and use communicate then get return
 code

---
 tests/test_basic/test_basic.py                            | 2 +-
 tests/test_cmfd_feed/test_cmfd_feed.py                    | 2 +-
 tests/test_cmfd_jfnk/test_cmfd_jfnk.py                    | 2 +-
 tests/test_cmfd_nofeed/test_cmfd_nofeed.py                | 2 +-
 tests/test_compile/test_compile.py                        | 2 +-
 .../test_confidence_intervals.py                          | 2 +-
 tests/test_density_atombcm/test_density_atombcm.py        | 2 +-
 tests/test_density_atomcm3/test_density_atomcm3.py        | 2 +-
 tests/test_density_kgm3/test_density_kgm3.py              | 2 +-
 tests/test_density_sum/test_density_sum.py                | 2 +-
 .../test_eigenvalue_genperbatch.py                        | 2 +-
 .../test_eigenvalue_no_inactive.py                        | 2 +-
 tests/test_energy_grid/test_energy_grid.py                | 2 +-
 tests/test_entropy/test_entropy.py                        | 2 +-
 tests/test_filter_cell/test_filter_cell.py                | 2 +-
 tests/test_filter_cellborn/test_filter_cellborn.py        | 2 +-
 tests/test_filter_energy/test_filter_energy.py            | 2 +-
 tests/test_filter_energyout/test_filter_energyout.py      | 2 +-
 .../test_filter_group_transfer.py                         | 2 +-
 tests/test_filter_material/test_filter_material.py        | 2 +-
 tests/test_filter_mesh_2d/test_filter_mesh_2d.py          | 2 +-
 tests/test_filter_mesh_3d/test_filter_mesh_3d.py          | 2 +-
 tests/test_filter_universe/test_filter_universe.py        | 2 +-
 tests/test_fixed_source/test_fixed_source.py              | 2 +-
 tests/test_lattice/test_lattice.py                        | 2 +-
 tests/test_lattice_multiple/test_lattice_multiple.py      | 2 +-
 tests/test_natural_element/test_natural_element.py        | 2 +-
 tests/test_output/test_output.py                          | 2 +-
 tests/test_plot_background/test_plot_background.py        | 2 +-
 tests/test_plot_basis/test_plot_basis.py                  | 2 +-
 tests/test_plot_colspec/test_plot_colspec.py              | 2 +-
 tests/test_plot_mask/test_plot_mask.py                    | 2 +-
 tests/test_ptables_off/test_ptables_off.py                | 2 +-
 tests/test_reflective_cone/test_reflective_cone.py        | 2 +-
 .../test_reflective_cylinder/test_reflective_cylinder.py  | 2 +-
 tests/test_reflective_plane/test_reflective_plane.py      | 2 +-
 tests/test_reflective_sphere/test_reflective_sphere.py    | 2 +-
 tests/test_rotation/test_rotation.py                      | 2 +-
 tests/test_salphabeta/test_salphabeta.py                  | 2 +-
 .../test_salphabeta_multiple/test_salphabeta_multiple.py  | 2 +-
 tests/test_score_MT/test_score_MT.py                      | 2 +-
 tests/test_score_absorption/test_score_absorption.py      | 2 +-
 tests/test_score_current/test_score_current.py            | 2 +-
 tests/test_score_events/test_score_events.py              | 2 +-
 tests/test_score_fission/test_score_fission.py            | 2 +-
 tests/test_score_flux/test_score_flux.py                  | 2 +-
 tests/test_score_kappafission/test_score_kappafission.py  | 2 +-
 tests/test_score_nufission/test_score_nufission.py        | 2 +-
 tests/test_score_nuscatter/test_score_nuscatter.py        | 2 +-
 tests/test_score_scatter/test_score_scatter.py            | 2 +-
 tests/test_score_scatter_n/test_score_scatter_n.py        | 2 +-
 tests/test_score_scatter_pn/test_score_scatter_pn.py      | 2 +-
 tests/test_score_total/test_score_total.py                | 2 +-
 tests/test_seed/test_seed.py                              | 2 +-
 tests/test_source_angle_mono/test_source_angle_mono.py    | 2 +-
 .../test_source_energy_maxwell.py                         | 2 +-
 tests/test_source_energy_mono/test_source_energy_mono.py  | 2 +-
 tests/test_source_point/test_source_point.py              | 2 +-
 tests/test_statepoint_batch/test_statepoint_batch.py      | 2 +-
 .../test_statepoint_interval/test_statepoint_interval.py  | 2 +-
 tests/test_statepoint_restart/test_statepoint_restart.py  | 8 ++++----
 .../test_statepoint_sourcesep.py                          | 2 +-
 tests/test_survival_biasing/test_survival_biasing.py      | 2 +-
 tests/test_tally_assumesep/test_tally_assumesep.py        | 2 +-
 tests/test_trace/test_trace.py                            | 2 +-
 tests/test_translation/test_translation.py                | 2 +-
 tests/test_uniform_fs/test_uniform_fs.py                  | 2 +-
 tests/test_universe/test_universe.py                      | 2 +-
 tests/test_void/test_void.py                              | 2 +-
 69 files changed, 72 insertions(+), 72 deletions(-)

diff --git a/tests/test_basic/test_basic.py b/tests/test_basic/test_basic.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_basic/test_basic.py
+++ b/tests/test_basic/test_basic.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_cmfd_feed/test_cmfd_feed.py b/tests/test_cmfd_feed/test_cmfd_feed.py
index 59bcad866b..1388ca85a5 100644
--- a/tests/test_cmfd_feed/test_cmfd_feed.py
+++ b/tests/test_cmfd_feed/test_cmfd_feed.py
@@ -29,9 +29,9 @@ def test_run():
                      stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     output = proc.communicate()[0]
     print(output)
+    returncode = proc.returncode
     if 'CMFD is not available' in output:
         global skipAll
         skipAll = True
diff --git a/tests/test_cmfd_jfnk/test_cmfd_jfnk.py b/tests/test_cmfd_jfnk/test_cmfd_jfnk.py
index 59bcad866b..1388ca85a5 100644
--- a/tests/test_cmfd_jfnk/test_cmfd_jfnk.py
+++ b/tests/test_cmfd_jfnk/test_cmfd_jfnk.py
@@ -29,9 +29,9 @@ def test_run():
                      stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     output = proc.communicate()[0]
     print(output)
+    returncode = proc.returncode
     if 'CMFD is not available' in output:
         global skipAll
         skipAll = True
diff --git a/tests/test_cmfd_nofeed/test_cmfd_nofeed.py b/tests/test_cmfd_nofeed/test_cmfd_nofeed.py
index 07176a8f2e..be7ec194a5 100644
--- a/tests/test_cmfd_nofeed/test_cmfd_nofeed.py
+++ b/tests/test_cmfd_nofeed/test_cmfd_nofeed.py
@@ -28,9 +28,9 @@ def test_run():
                      stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     output = proc.communicate()[0]
     print(output)
+    returncode = proc.returncode
     if 'CMFD is not available' in output:
         global skipAll
         skipAll = True
diff --git a/tests/test_compile/test_compile.py b/tests/test_compile/test_compile.py
index ced2a86321..8d411abd91 100644
--- a/tests/test_compile/test_compile.py
+++ b/tests/test_compile/test_compile.py
@@ -273,8 +273,8 @@ def test_mpi_omp_hdf5_petsc_optimize():
 
 def run(commands):
     proc = Popen(commands, stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     return returncode
 
 
diff --git a/tests/test_confidence_intervals/test_confidence_intervals.py b/tests/test_confidence_intervals/test_confidence_intervals.py
index a0dda8b468..a8372834f7 100644
--- a/tests/test_confidence_intervals/test_confidence_intervals.py
+++ b/tests/test_confidence_intervals/test_confidence_intervals.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_density_atombcm/test_density_atombcm.py b/tests/test_density_atombcm/test_density_atombcm.py
index d13f404722..2fde1e6312 100644
--- a/tests/test_density_atombcm/test_density_atombcm.py
+++ b/tests/test_density_atombcm/test_density_atombcm.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_density_atomcm3/test_density_atomcm3.py b/tests/test_density_atomcm3/test_density_atomcm3.py
index 860f858d34..74a644b59d 100644
--- a/tests/test_density_atomcm3/test_density_atomcm3.py
+++ b/tests/test_density_atomcm3/test_density_atomcm3.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_density_kgm3/test_density_kgm3.py b/tests/test_density_kgm3/test_density_kgm3.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_density_kgm3/test_density_kgm3.py
+++ b/tests/test_density_kgm3/test_density_kgm3.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_density_sum/test_density_sum.py b/tests/test_density_sum/test_density_sum.py
index d1bc7d3a69..fcafcf0f1f 100644
--- a/tests/test_density_sum/test_density_sum.py
+++ b/tests/test_density_sum/test_density_sum.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_eigenvalue_genperbatch/test_eigenvalue_genperbatch.py b/tests/test_eigenvalue_genperbatch/test_eigenvalue_genperbatch.py
index eb4e206c1b..24077bd591 100644
--- a/tests/test_eigenvalue_genperbatch/test_eigenvalue_genperbatch.py
+++ b/tests/test_eigenvalue_genperbatch/test_eigenvalue_genperbatch.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_eigenvalue_no_inactive/test_eigenvalue_no_inactive.py b/tests/test_eigenvalue_no_inactive/test_eigenvalue_no_inactive.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_eigenvalue_no_inactive/test_eigenvalue_no_inactive.py
+++ b/tests/test_eigenvalue_no_inactive/test_eigenvalue_no_inactive.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_energy_grid/test_energy_grid.py b/tests/test_energy_grid/test_energy_grid.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_energy_grid/test_energy_grid.py
+++ b/tests/test_energy_grid/test_energy_grid.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_entropy/test_entropy.py b/tests/test_entropy/test_entropy.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_entropy/test_entropy.py
+++ b/tests/test_entropy/test_entropy.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_filter_cell/test_filter_cell.py b/tests/test_filter_cell/test_filter_cell.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_filter_cell/test_filter_cell.py
+++ b/tests/test_filter_cell/test_filter_cell.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_filter_cellborn/test_filter_cellborn.py b/tests/test_filter_cellborn/test_filter_cellborn.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_filter_cellborn/test_filter_cellborn.py
+++ b/tests/test_filter_cellborn/test_filter_cellborn.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_filter_energy/test_filter_energy.py b/tests/test_filter_energy/test_filter_energy.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_filter_energy/test_filter_energy.py
+++ b/tests/test_filter_energy/test_filter_energy.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_filter_energyout/test_filter_energyout.py b/tests/test_filter_energyout/test_filter_energyout.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_filter_energyout/test_filter_energyout.py
+++ b/tests/test_filter_energyout/test_filter_energyout.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_filter_group_transfer/test_filter_group_transfer.py b/tests/test_filter_group_transfer/test_filter_group_transfer.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_filter_group_transfer/test_filter_group_transfer.py
+++ b/tests/test_filter_group_transfer/test_filter_group_transfer.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_filter_material/test_filter_material.py b/tests/test_filter_material/test_filter_material.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_filter_material/test_filter_material.py
+++ b/tests/test_filter_material/test_filter_material.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_filter_mesh_2d/test_filter_mesh_2d.py b/tests/test_filter_mesh_2d/test_filter_mesh_2d.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_filter_mesh_2d/test_filter_mesh_2d.py
+++ b/tests/test_filter_mesh_2d/test_filter_mesh_2d.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_filter_mesh_3d/test_filter_mesh_3d.py b/tests/test_filter_mesh_3d/test_filter_mesh_3d.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_filter_mesh_3d/test_filter_mesh_3d.py
+++ b/tests/test_filter_mesh_3d/test_filter_mesh_3d.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_filter_universe/test_filter_universe.py b/tests/test_filter_universe/test_filter_universe.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_filter_universe/test_filter_universe.py
+++ b/tests/test_filter_universe/test_filter_universe.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_fixed_source/test_fixed_source.py b/tests/test_fixed_source/test_fixed_source.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_fixed_source/test_fixed_source.py
+++ b/tests/test_fixed_source/test_fixed_source.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_lattice/test_lattice.py b/tests/test_lattice/test_lattice.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_lattice/test_lattice.py
+++ b/tests/test_lattice/test_lattice.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_lattice_multiple/test_lattice_multiple.py b/tests/test_lattice_multiple/test_lattice_multiple.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_lattice_multiple/test_lattice_multiple.py
+++ b/tests/test_lattice_multiple/test_lattice_multiple.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_natural_element/test_natural_element.py b/tests/test_natural_element/test_natural_element.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_natural_element/test_natural_element.py
+++ b/tests/test_natural_element/test_natural_element.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_output/test_output.py b/tests/test_output/test_output.py
index 9024d6fb19..dc48dc37b9 100644
--- a/tests/test_output/test_output.py
+++ b/tests/test_output/test_output.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_summary_exists():
diff --git a/tests/test_plot_background/test_plot_background.py b/tests/test_plot_background/test_plot_background.py
index a34b5bbfb7..d8fc18f755 100644
--- a/tests/test_plot_background/test_plot_background.py
+++ b/tests/test_plot_background/test_plot_background.py
@@ -17,8 +17,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path, '-p'], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_plot_exists():
diff --git a/tests/test_plot_basis/test_plot_basis.py b/tests/test_plot_basis/test_plot_basis.py
index fe0a13ccac..86a4159200 100644
--- a/tests/test_plot_basis/test_plot_basis.py
+++ b/tests/test_plot_basis/test_plot_basis.py
@@ -17,8 +17,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path, '-p'], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_plots_exists():
diff --git a/tests/test_plot_colspec/test_plot_colspec.py b/tests/test_plot_colspec/test_plot_colspec.py
index a34b5bbfb7..d8fc18f755 100644
--- a/tests/test_plot_colspec/test_plot_colspec.py
+++ b/tests/test_plot_colspec/test_plot_colspec.py
@@ -17,8 +17,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path, '-p'], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_plot_exists():
diff --git a/tests/test_plot_mask/test_plot_mask.py b/tests/test_plot_mask/test_plot_mask.py
index fe0a13ccac..86a4159200 100644
--- a/tests/test_plot_mask/test_plot_mask.py
+++ b/tests/test_plot_mask/test_plot_mask.py
@@ -17,8 +17,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path, '-p'], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_plots_exists():
diff --git a/tests/test_ptables_off/test_ptables_off.py b/tests/test_ptables_off/test_ptables_off.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_ptables_off/test_ptables_off.py
+++ b/tests/test_ptables_off/test_ptables_off.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_reflective_cone/test_reflective_cone.py b/tests/test_reflective_cone/test_reflective_cone.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_reflective_cone/test_reflective_cone.py
+++ b/tests/test_reflective_cone/test_reflective_cone.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_reflective_cylinder/test_reflective_cylinder.py b/tests/test_reflective_cylinder/test_reflective_cylinder.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_reflective_cylinder/test_reflective_cylinder.py
+++ b/tests/test_reflective_cylinder/test_reflective_cylinder.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_reflective_plane/test_reflective_plane.py b/tests/test_reflective_plane/test_reflective_plane.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_reflective_plane/test_reflective_plane.py
+++ b/tests/test_reflective_plane/test_reflective_plane.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_reflective_sphere/test_reflective_sphere.py b/tests/test_reflective_sphere/test_reflective_sphere.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_reflective_sphere/test_reflective_sphere.py
+++ b/tests/test_reflective_sphere/test_reflective_sphere.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_rotation/test_rotation.py b/tests/test_rotation/test_rotation.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_rotation/test_rotation.py
+++ b/tests/test_rotation/test_rotation.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_salphabeta/test_salphabeta.py b/tests/test_salphabeta/test_salphabeta.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_salphabeta/test_salphabeta.py
+++ b/tests/test_salphabeta/test_salphabeta.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_salphabeta_multiple/test_salphabeta_multiple.py b/tests/test_salphabeta_multiple/test_salphabeta_multiple.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_salphabeta_multiple/test_salphabeta_multiple.py
+++ b/tests/test_salphabeta_multiple/test_salphabeta_multiple.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_score_MT/test_score_MT.py b/tests/test_score_MT/test_score_MT.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_score_MT/test_score_MT.py
+++ b/tests/test_score_MT/test_score_MT.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_score_absorption/test_score_absorption.py b/tests/test_score_absorption/test_score_absorption.py
index b49faa8177..335d649dd7 100644
--- a/tests/test_score_absorption/test_score_absorption.py
+++ b/tests/test_score_absorption/test_score_absorption.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_score_current/test_score_current.py b/tests/test_score_current/test_score_current.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_score_current/test_score_current.py
+++ b/tests/test_score_current/test_score_current.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_score_events/test_score_events.py b/tests/test_score_events/test_score_events.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_score_events/test_score_events.py
+++ b/tests/test_score_events/test_score_events.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_score_fission/test_score_fission.py b/tests/test_score_fission/test_score_fission.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_score_fission/test_score_fission.py
+++ b/tests/test_score_fission/test_score_fission.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_score_flux/test_score_flux.py b/tests/test_score_flux/test_score_flux.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_score_flux/test_score_flux.py
+++ b/tests/test_score_flux/test_score_flux.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_score_kappafission/test_score_kappafission.py b/tests/test_score_kappafission/test_score_kappafission.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_score_kappafission/test_score_kappafission.py
+++ b/tests/test_score_kappafission/test_score_kappafission.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_score_nufission/test_score_nufission.py b/tests/test_score_nufission/test_score_nufission.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_score_nufission/test_score_nufission.py
+++ b/tests/test_score_nufission/test_score_nufission.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_score_nuscatter/test_score_nuscatter.py b/tests/test_score_nuscatter/test_score_nuscatter.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_score_nuscatter/test_score_nuscatter.py
+++ b/tests/test_score_nuscatter/test_score_nuscatter.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_score_scatter/test_score_scatter.py b/tests/test_score_scatter/test_score_scatter.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_score_scatter/test_score_scatter.py
+++ b/tests/test_score_scatter/test_score_scatter.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_score_scatter_n/test_score_scatter_n.py b/tests/test_score_scatter_n/test_score_scatter_n.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_score_scatter_n/test_score_scatter_n.py
+++ b/tests/test_score_scatter_n/test_score_scatter_n.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_score_scatter_pn/test_score_scatter_pn.py b/tests/test_score_scatter_pn/test_score_scatter_pn.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_score_scatter_pn/test_score_scatter_pn.py
+++ b/tests/test_score_scatter_pn/test_score_scatter_pn.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_score_total/test_score_total.py b/tests/test_score_total/test_score_total.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_score_total/test_score_total.py
+++ b/tests/test_score_total/test_score_total.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_seed/test_seed.py b/tests/test_seed/test_seed.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_seed/test_seed.py
+++ b/tests/test_seed/test_seed.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_source_angle_mono/test_source_angle_mono.py b/tests/test_source_angle_mono/test_source_angle_mono.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_source_angle_mono/test_source_angle_mono.py
+++ b/tests/test_source_angle_mono/test_source_angle_mono.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_source_energy_maxwell/test_source_energy_maxwell.py b/tests/test_source_energy_maxwell/test_source_energy_maxwell.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_source_energy_maxwell/test_source_energy_maxwell.py
+++ b/tests/test_source_energy_maxwell/test_source_energy_maxwell.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_source_energy_mono/test_source_energy_mono.py b/tests/test_source_energy_mono/test_source_energy_mono.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_source_energy_mono/test_source_energy_mono.py
+++ b/tests/test_source_energy_mono/test_source_energy_mono.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_source_point/test_source_point.py b/tests/test_source_point/test_source_point.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_source_point/test_source_point.py
+++ b/tests/test_source_point/test_source_point.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_statepoint_batch/test_statepoint_batch.py b/tests/test_statepoint_batch/test_statepoint_batch.py
index 57d00dbe8f..ed4f7b2a7a 100644
--- a/tests/test_statepoint_batch/test_statepoint_batch.py
+++ b/tests/test_statepoint_batch/test_statepoint_batch.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_statepoints_exist():
diff --git a/tests/test_statepoint_interval/test_statepoint_interval.py b/tests/test_statepoint_interval/test_statepoint_interval.py
index 5c3c87fbc5..263ab37907 100644
--- a/tests/test_statepoint_interval/test_statepoint_interval.py
+++ b/tests/test_statepoint_interval/test_statepoint_interval.py
@@ -20,8 +20,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_statepoints_exist():
diff --git a/tests/test_statepoint_restart/test_statepoint_restart.py b/tests/test_statepoint_restart/test_statepoint_restart.py
index 7a880d59b0..544c9197ec 100644
--- a/tests/test_statepoint_restart/test_statepoint_restart.py
+++ b/tests/test_statepoint_restart/test_statepoint_restart.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
@@ -45,8 +45,8 @@ def test_restart_form1():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path, '-r', statepoint[0]], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint_form1():
@@ -72,8 +72,8 @@ def test_restart_form2():
     else:
         proc = Popen([openmc_path, '--restart', statepoint[0]], 
                      stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint_form2():
@@ -94,8 +94,8 @@ def test_restart_serial():
     openmc_path = pwd + '/../../src/openmc'
     proc = Popen([openmc_path, '--restart', statepoint[0]], 
                  stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint_serial():
diff --git a/tests/test_statepoint_sourcesep/test_statepoint_sourcesep.py b/tests/test_statepoint_sourcesep/test_statepoint_sourcesep.py
index 61191c57ae..ca1c9966a4 100644
--- a/tests/test_statepoint_sourcesep/test_statepoint_sourcesep.py
+++ b/tests/test_statepoint_sourcesep/test_statepoint_sourcesep.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_statepoint_exists():
diff --git a/tests/test_survival_biasing/test_survival_biasing.py b/tests/test_survival_biasing/test_survival_biasing.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_survival_biasing/test_survival_biasing.py
+++ b/tests/test_survival_biasing/test_survival_biasing.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_tally_assumesep/test_tally_assumesep.py b/tests/test_tally_assumesep/test_tally_assumesep.py
index 62c4d7316a..151df366bd 100644
--- a/tests/test_tally_assumesep/test_tally_assumesep.py
+++ b/tests/test_tally_assumesep/test_tally_assumesep.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_trace/test_trace.py b/tests/test_trace/test_trace.py
index d618dfc5e8..a518ab7731 100644
--- a/tests/test_trace/test_trace.py
+++ b/tests/test_trace/test_trace.py
@@ -19,9 +19,9 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     stdout = proc.communicate()[0]
     print(stdout)
+    returncode = proc.returncode
     assert returncode == 0
     assert stdout.find('Simulating Particle 453') != -1
 
diff --git a/tests/test_translation/test_translation.py b/tests/test_translation/test_translation.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_translation/test_translation.py
+++ b/tests/test_translation/test_translation.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_uniform_fs/test_uniform_fs.py b/tests/test_uniform_fs/test_uniform_fs.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_uniform_fs/test_uniform_fs.py
+++ b/tests/test_uniform_fs/test_uniform_fs.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_universe/test_universe.py b/tests/test_universe/test_universe.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_universe/test_universe.py
+++ b/tests/test_universe/test_universe.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():
diff --git a/tests/test_void/test_void.py b/tests/test_void/test_void.py
index fe8344c15c..64d20b6115 100644
--- a/tests/test_void/test_void.py
+++ b/tests/test_void/test_void.py
@@ -19,8 +19,8 @@ def test_run():
                stderr=STDOUT, stdout=PIPE)
     else:
         proc = Popen([openmc_path], stderr=STDOUT, stdout=PIPE)
-    returncode = proc.wait()
     print(proc.communicate()[0])
+    returncode = proc.returncode
     assert returncode == 0
 
 def test_created_statepoint():