Merge pull request #1566 from paulromano/depletion-without-phdf5

Allow depletion with MPI and serial HDF5

docs/source/usersguide/install.rst

@@ -407,9 +407,7 @@ Prerequisites
 The Python API works with Python 3.5+. In addition to Python itself, the API
 relies on a number of third-party packages. All prerequisites can be installed
 using Conda_ (recommended), pip_, or through the package manager in most Linux
-distributions. To run simulations in parallel using MPI, it is recommended to
-build mpi4py, HDF5, h5py from source, in that order, using the same compilers
-as for OpenMC.
+distributions.
 
 .. admonition:: Required
    :class: error
@@ -461,6 +459,29 @@ as for OpenMC.
    `pytest <https://docs.pytest.org>`_
       The pytest framework is used for unit testing the Python API.
 
+If you are running simulations that require OpenMC's Python bindings to the C
+API (including depletion and CMFD), it is recommended to build ``h5py`` (and
+``mpi4py``, if you are using MPI) using the same compilers and HDF5 version as
+for OpenMC. Thus, the install process would proceed as follows:
+
+.. code-block:: sh
+
+    mkdir build && cd build
+    HDF5_ROOT=<path to HDF5> CXX=<path to mpicxx> cmake ..
+    make
+    make install
+
+    cd ..
+    MPICC=<path to mpicc> pip install mpi4py
+    HDF5_DIR=<path to HDF5> pip install --no-binary=h5py h5py
+
+If you are using parallel HDF5, you'll also need to make sure the right MPI
+wrapper is used when installing h5py:
+
+.. code-block:: sh
+
+    CC=<path to mpicc> HDF5_MPI=ON HDF5_DIR=<path to HDF5> pip install --no-binary=h5py h5py
+
 .. _usersguide_nxml:
 
 -----------------------------------------------------

openmc/deplete/__init__.py

@@ -7,28 +7,14 @@ A depletion front-end tool.
 import sys
 from unittest.mock import Mock
 
-from h5py import get_config
-
 from .dummy_comm import DummyCommunicator
 
 try:
     from mpi4py import MPI
-
     comm = MPI.COMM_WORLD
-    have_mpi = True
-    # check if running with MPI and if using parallel HDF5
-
-    if not get_config().mpi and comm.size > 1:
-        # Raise exception only on process 0
-        if comm.rank:
-            sys.exit()
-        raise RuntimeError(
-            "Need parallel HDF5 installed to perform depletion with MPI"
-        )
 except ImportError:
-    comm = DummyCommunicator()
-    have_mpi = False
     MPI = Mock()
+    comm = DummyCommunicator()
 
 from .nuclide import *
 from .chain import *

openmc/deplete/results.py

@@ -9,7 +9,7 @@ import copy
 import h5py
 import numpy as np
 
-from . import comm, have_mpi, MPI
+from . import comm, MPI
 from .reaction_rates import ReactionRates
 
 VERSION_RESULTS = (1, 0)
@@ -197,16 +197,24 @@ class Results:
             What step is this?
 
         """
-        if have_mpi and h5py.get_config().mpi:
-            kwargs = {'driver': 'mpio', 'comm': comm}
-        else:
-            kwargs = {}
-
         # Write new file if first time step, else add to existing file
-        kwargs['mode'] = "w" if step == 0 else "a"
+        kwargs = {'mode': "w" if step == 0 else "a"}
 
-        with h5py.File(filename, **kwargs) as handle:
-            self._to_hdf5(handle, step)
+        if h5py.get_config().mpi and comm.size > 1:
+            # Write results in parallel
+            kwargs['driver'] = 'mpio'
+            kwargs['comm'] = comm
+            with h5py.File(filename, **kwargs) as handle:
+                self._to_hdf5(handle, step, parallel=True)
+        else:
+            # Gather results at root process
+            all_results = comm.gather(self)
+
+            # Only root process writes results
+            if comm.rank == 0:
+                with h5py.File(filename, **kwargs) as handle:
+                    for res in all_results:
+                        res._to_hdf5(handle, step, parallel=False)
 
     def _write_hdf5_metadata(self, handle):
         """Writes result metadata in HDF5 file
@@ -286,7 +294,7 @@ class Results:
             "depletion time", (1,), maxshape=(None,),
             dtype="float64")
 
-    def _to_hdf5(self, handle, index):
+    def _to_hdf5(self, handle, index, parallel=False):
         """Converts results object into an hdf5 object.
 
         Parameters
@@ -295,13 +303,17 @@ class Results:
             An HDF5 file or group type to store this in.
         index : int
             What step is this?
+        parallel : bool
+            Being called with parallel HDF5?
 
         """
         if "/number" not in handle:
-            comm.barrier()
+            if parallel:
+                comm.barrier()
             self._write_hdf5_metadata(handle)
 
-        comm.barrier()
+        if parallel:
+            comm.barrier()
 
         # Grab handles
         number_dset = handle["/number"]
@@ -353,13 +365,13 @@ class Results:
         low = min(inds)
         high = max(inds)
         for i in range(n_stages):
-            number_dset[index, i, low:high+1, :] = self.data[i, :, :]
-            rxn_dset[index, i, low:high+1, :, :] = self.rates[i][:, :, :]
+            number_dset[index, i, low:high+1] = self.data[i]
+            rxn_dset[index, i, low:high+1] = self.rates[i]
             if comm.rank == 0:
                 eigenvalues_dset[index, i] = self.k[i]
         if comm.rank == 0:
-            time_dset[index, :] = self.time
-            power_dset[index, :] = self.power
+            time_dset[index] = self.time
+            power_dset[index] = self.power
             if self.proc_time is not None:
                 proc_time_dset[index] = (
                     self.proc_time / (comm.size * self.n_hdf5_mats)