From 73cbdb6324906070c68c4bf963087441006dd79c Mon Sep 17 00:00:00 2001 From: Paul Romano Date: Fri, 15 May 2020 14:23:35 -0500 Subject: [PATCH] Allow depletion with MPI and serial HDF5 --- docs/source/usersguide/install.rst | 27 ++++++++++++++++-- openmc/deplete/__init__.py | 16 +---------- openmc/deplete/results.py | 46 +++++++++++++++++++----------- 3 files changed, 55 insertions(+), 34 deletions(-) diff --git a/docs/source/usersguide/install.rst b/docs/source/usersguide/install.rst index ec10fea4fa..d1dece42e0 100644 --- a/docs/source/usersguide/install.rst +++ b/docs/source/usersguide/install.rst @@ -407,9 +407,7 @@ Prerequisites The Python API works with Python 3.5+. In addition to Python itself, the API relies on a number of third-party packages. All prerequisites can be installed using Conda_ (recommended), pip_, or through the package manager in most Linux -distributions. To run simulations in parallel using MPI, it is recommended to -build mpi4py, HDF5, h5py from source, in that order, using the same compilers -as for OpenMC. +distributions. .. admonition:: Required :class: error @@ -461,6 +459,29 @@ as for OpenMC. `pytest `_ The pytest framework is used for unit testing the Python API. +If you are running simulations that require OpenMC's Python bindings to the C +API (including depletion and CMFD), it is recommended to build ``h5py`` (and +``mpi4py``, if you are using MPI) using the same compilers and HDF5 version as +for OpenMC. Thus, the install process would proceed as follows: + +.. code-block:: sh + + mkdir build && cd build + HDF5_ROOT= CXX= cmake .. + make + make install + + cd .. + MPICC= pip install mpi4py + HDF5_DIR= pip install --no-binary=h5py h5py + +If you are using parallel HDF5, you'll also need to make sure the right MPI +wrapper is used when installing h5py: + +.. code-block:: + + CC= HDF5_MPI=ON HDF5_DIR= pip install --no-binary=h5py h5py + .. _usersguide_nxml: ----------------------------------------------------- diff --git a/openmc/deplete/__init__.py b/openmc/deplete/__init__.py index b54d7f11d3..00b29e3196 100644 --- a/openmc/deplete/__init__.py +++ b/openmc/deplete/__init__.py @@ -7,28 +7,14 @@ A depletion front-end tool. import sys from unittest.mock import Mock -from h5py import get_config - from .dummy_comm import DummyCommunicator try: from mpi4py import MPI - comm = MPI.COMM_WORLD - have_mpi = True - # check if running with MPI and if using parallel HDF5 - - if not get_config().mpi and comm.size > 1: - # Raise exception only on process 0 - if comm.rank: - sys.exit() - raise RuntimeError( - "Need parallel HDF5 installed to perform depletion with MPI" - ) except ImportError: - comm = DummyCommunicator() - have_mpi = False MPI = Mock() + comm = DummyCommunicator() from .nuclide import * from .chain import * diff --git a/openmc/deplete/results.py b/openmc/deplete/results.py index 19895b4d9c..52d3d0cc50 100644 --- a/openmc/deplete/results.py +++ b/openmc/deplete/results.py @@ -9,7 +9,7 @@ import copy import h5py import numpy as np -from . import comm, have_mpi, MPI +from . import comm, MPI from .reaction_rates import ReactionRates VERSION_RESULTS = (1, 0) @@ -197,16 +197,26 @@ class Results: What step is this? """ - if have_mpi and h5py.get_config().mpi: - kwargs = {'driver': 'mpio', 'comm': comm} - else: - kwargs = {} - # Write new file if first time step, else add to existing file - kwargs['mode'] = "w" if step == 0 else "a" + kwargs = {'mode': "w" if step == 0 else "a"} + + if h5py.get_config().mpi and comm.size > 1: + # Write results in parallel + kwargs['driver'] = 'mpio' + kwargs['comm'] = comm + with h5py.File(filename, **kwargs) as handle: + self._to_hdf5(handle, step, parallel=True) + else: + # Gather results at root process + all_results = comm.gather(self) + + # Only root process writes results + if comm.rank == 0: + with h5py.File(filename, **kwargs) as handle: + for res in all_results: + res._to_hdf5(handle, step, parallel=False) + - with h5py.File(filename, **kwargs) as handle: - self._to_hdf5(handle, step) def _write_hdf5_metadata(self, handle): """Writes result metadata in HDF5 file @@ -286,7 +296,7 @@ class Results: "depletion time", (1,), maxshape=(None,), dtype="float64") - def _to_hdf5(self, handle, index): + def _to_hdf5(self, handle, index, parallel=False): """Converts results object into an hdf5 object. Parameters @@ -295,13 +305,17 @@ class Results: An HDF5 file or group type to store this in. index : int What step is this? + parallel : bool + Being called with parallel HDF5? """ if "/number" not in handle: - comm.barrier() + if parallel: + comm.barrier() self._write_hdf5_metadata(handle) - comm.barrier() + if parallel: + comm.barrier() # Grab handles number_dset = handle["/number"] @@ -353,13 +367,13 @@ class Results: low = min(inds) high = max(inds) for i in range(n_stages): - number_dset[index, i, low:high+1, :] = self.data[i, :, :] - rxn_dset[index, i, low:high+1, :, :] = self.rates[i][:, :, :] + number_dset[index, i, low:high+1] = self.data[i] + rxn_dset[index, i, low:high+1] = self.rates[i] if comm.rank == 0: eigenvalues_dset[index, i] = self.k[i] if comm.rank == 0: - time_dset[index, :] = self.time - power_dset[index, :] = self.power + time_dset[index] = self.time + power_dset[index] = self.power if self.proc_time is not None: proc_time_dset[index] = ( self.proc_time / (comm.size * self.n_hdf5_mats)