From 01850016c6154841951b9de25ac153c79f845cbc Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Tue, 5 Apr 2022 15:32:43 -0500 Subject: [PATCH 01/15] use find_package(mpi) instead of relying on compilers set --- CMakeLists.txt | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e972c49378..511ce66b06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.3 FATAL_ERROR) +cmake_minimum_required(VERSION 3.10 FATAL_ERROR) project(openmc C CXX) # Set version numbers @@ -30,6 +30,7 @@ option(OPENMC_ENABLE_PROFILE "Compile with profiling flags" option(OPENMC_ENABLE_COVERAGE "Compile with coverage analysis flags" OFF) option(OPENMC_USE_DAGMC "Enable support for DAGMC (CAD) geometry" OFF) option(OPENMC_USE_LIBMESH "Enable support for libMesh unstructured mesh tallies" OFF) +option(OPENMC_USE_MPI "Enable MPI" OFF) #=============================================================================== # Set a default build configuration if not explicitly specified @@ -44,10 +45,9 @@ endif() # MPI for distributed-memory parallelism #=============================================================================== -set(MPI_ENABLED FALSE) -if(${CMAKE_CXX_COMPILER} MATCHES "(mpi[^/]*|CC)$") - message(STATUS "Detected MPI wrapper: ${CMAKE_CXX_COMPILER}") - set(MPI_ENABLED TRUE) +if(OPENMC_USE_MPI) + find_package(MPI REQUIRED) + include_directories(${MPI_CXX_INCLUDE_DIRS}) endif() #=============================================================================== @@ -108,7 +108,7 @@ endif() find_package(HDF5 REQUIRED COMPONENTS C HL) if(HDF5_IS_PARALLEL) - if(NOT MPI_ENABLED) + if(NOT OPENMC_USE_MPI) message(FATAL_ERROR "Parallel HDF5 was detected, but the detected compiler,\ ${CMAKE_CXX_COMPILER}, does not support MPI. An MPI-capable compiler must \ be used with parallel HDF5.") @@ -141,9 +141,15 @@ endif() set(CMAKE_POSITION_INDEPENDENT_CODE ON) +if(OPENMC_USE_MPI) + list(APPEND cxxflags ${MPI_CXX_COMPILE_FLAGS}) + list(APPEND ldflags ${MPI_CXX_LINK_FLAGS}) +endif() + if(OPENMC_ENABLE_PROFILE) list(APPEND cxxflags -g -fno-omit-frame-pointer) endif() + if(OPENMC_ENABLE_COVERAGE) list(APPEND cxxflags --coverage) list(APPEND ldflags --coverage) @@ -398,7 +404,7 @@ target_include_directories(libopenmc PRIVATE ${CMAKE_BINARY_DIR}/include) if (HDF5_IS_PARALLEL) target_compile_definitions(libopenmc PRIVATE -DPHDF5) endif() -if (MPI_ENABLED) +if (OPENMC_USE_MPI) target_compile_definitions(libopenmc PUBLIC -DOPENMC_MPI) endif() @@ -440,6 +446,10 @@ if (PNG_FOUND) target_link_libraries(libopenmc PNG::PNG) endif() +if (OPENMC_USE_MPI) + target_link_libraries(libopenmc MPI::MPI_CXX) +endif() + #=============================================================================== # openmc executable #=============================================================================== From 447e9cf02da322f40478420aab9d65b8450cb09c Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Tue, 5 Apr 2022 15:56:10 -0500 Subject: [PATCH 02/15] updated ci with new mpi flags --- tools/ci/gha-install.py | 2 +- tools/ci/gha-install.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/ci/gha-install.py b/tools/ci/gha-install.py index 4cc85671d9..4c69ba70b0 100644 --- a/tools/ci/gha-install.py +++ b/tools/ci/gha-install.py @@ -34,7 +34,7 @@ def install(omp=False, mpi=False, phdf5=False, dagmc=False, libmesh=False): # Use MPI wrappers when building in parallel if mpi: - os.environ['CXX'] = 'mpicxx' + cmake_cmd.append('-DOPENMC_USE_MPI=on') # Tell CMake to prefer parallel HDF5 if specified if phdf5: diff --git a/tools/ci/gha-install.sh b/tools/ci/gha-install.sh index aa40eb90b1..52e71b6936 100755 --- a/tools/ci/gha-install.sh +++ b/tools/ci/gha-install.sh @@ -32,7 +32,6 @@ fi if [[ $MPI == 'y' ]]; then pip install --no-binary=mpi4py mpi4py - export CC=mpicc export HDF5_MPI=ON export HDF5_DIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich pip install --no-binary=h5py h5py From 837ccc73af47e433287e24ec7601d763fb6de001 Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Tue, 5 Apr 2022 15:59:09 -0500 Subject: [PATCH 03/15] updated documentation with new compile flag for mpi --- docs/source/usersguide/install.rst | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/docs/source/usersguide/install.rst b/docs/source/usersguide/install.rst index 42258b0864..d7173e8c04 100644 --- a/docs/source/usersguide/install.rst +++ b/docs/source/usersguide/install.rst @@ -387,20 +387,11 @@ Example of configuring for Debug mode: Compiling with MPI ++++++++++++++++++ -To compile with MPI, set the :envvar:`CXX` environment variable to the path to -the MPI C++ wrapper. For example, in a bash shell: +To compile with MPI, use the `-DOPENMC_USE_MPI=on` cmake flag. For example, in a bash shell: .. code-block:: sh - export CXX=mpicxx - cmake /path/to/openmc - -Note that in many shells, environment variables can be set for a single command, -i.e. - -.. code-block:: sh - - CXX=mpicxx cmake /path/to/openmc + cmake -DOPENMC_USE_MPI=on /path/to/openmc Selecting HDF5 Installation +++++++++++++++++++++++++++ From 40294834939750dd67d48cf1538c4c7700d13f63 Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Tue, 5 Apr 2022 15:59:41 -0500 Subject: [PATCH 04/15] wording --- docs/source/usersguide/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/usersguide/install.rst b/docs/source/usersguide/install.rst index d7173e8c04..53c06ab0ad 100644 --- a/docs/source/usersguide/install.rst +++ b/docs/source/usersguide/install.rst @@ -387,7 +387,7 @@ Example of configuring for Debug mode: Compiling with MPI ++++++++++++++++++ -To compile with MPI, use the `-DOPENMC_USE_MPI=on` cmake flag. For example, in a bash shell: +To compile with MPI, use the `-DOPENMC_USE_MPI=on` cmake option. For example, in a bash shell: .. code-block:: sh From 20a18398bffcabf273df6b0f3c9cb2a68926bdd9 Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Tue, 5 Apr 2022 16:19:32 -0500 Subject: [PATCH 05/15] export cc=mpicc apparently necessary for h5py --- tools/ci/gha-install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci/gha-install.sh b/tools/ci/gha-install.sh index 52e71b6936..aa40eb90b1 100755 --- a/tools/ci/gha-install.sh +++ b/tools/ci/gha-install.sh @@ -32,6 +32,7 @@ fi if [[ $MPI == 'y' ]]; then pip install --no-binary=mpi4py mpi4py + export CC=mpicc export HDF5_MPI=ON export HDF5_DIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich pip install --no-binary=h5py h5py From 4f6001d574903fe728afda1f0a7b1fc181056253 Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Wed, 6 Apr 2022 09:28:02 -0500 Subject: [PATCH 06/15] updated method for setting mpi for regression tests --- tests/regression_tests/cpp_driver/test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/regression_tests/cpp_driver/test.py b/tests/regression_tests/cpp_driver/test.py index 0726e4c6ec..6870cfe7d7 100644 --- a/tests/regression_tests/cpp_driver/test.py +++ b/tests/regression_tests/cpp_driver/test.py @@ -33,12 +33,14 @@ def cpp_driver(request): os.chdir(str(local_builddir)) if config['mpi']: - os.environ['CXX'] = 'mpicxx' + mpi_arg = "On" + else: + mpi_arg = "Off" try: print("Building driver") # Run cmake/make to build the shared libary - subprocess.run(['cmake', os.path.pardir], check=True) + subprocess.run(['cmake', os.path.pardir, '-DOPENMC_USE_MPI=' + mpi_arg], check=True) subprocess.run(['make'], check=True) os.chdir(os.path.pardir) From 4010b32cffe11e4cf20be99ca9edb7861f49de3a Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Wed, 6 Apr 2022 09:32:38 -0500 Subject: [PATCH 07/15] update mpi setting in another regression test --- tests/regression_tests/external_moab/test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/regression_tests/external_moab/test.py b/tests/regression_tests/external_moab/test.py index caff6b5dbd..da55c735a9 100644 --- a/tests/regression_tests/external_moab/test.py +++ b/tests/regression_tests/external_moab/test.py @@ -49,12 +49,14 @@ def cpp_driver(request): os.chdir(str(local_builddir)) if config['mpi']: - os.environ['CXX'] = 'mpicxx' + mpi_arg = "On" + else: + mpi_arg = "Off" try: print("Building driver") # Run cmake/make to build the shared libary - subprocess.run(['cmake', os.path.pardir], check=True) + subprocess.run(['cmake', os.path.pardir, '-DOPENMC_USE_MPI=' + mpi_arg], check=True) subprocess.run(['make'], check=True) os.chdir(os.path.pardir) From ac607a83f94e6bca761eb62f635440467bb739c6 Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Wed, 6 Apr 2022 09:33:49 -0500 Subject: [PATCH 08/15] updated more mpi instances in install documentation --- docs/source/usersguide/install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/usersguide/install.rst b/docs/source/usersguide/install.rst index 53c06ab0ad..1cb00ae559 100644 --- a/docs/source/usersguide/install.rst +++ b/docs/source/usersguide/install.rst @@ -203,7 +203,7 @@ Prerequisites respectively. To link against a parallel HDF5 library, make sure to set the HDF5_PREFER_PARALLEL CMake option, e.g.:: - CXX=mpicxx.mpich cmake -DHDF5_PREFER_PARALLEL=on .. + cmake -DHDF5_PREFER_PARALLEL=on -DOPENMC_USE_MPI=on .. Note that the exact package names may vary depending on your particular distribution and version. @@ -263,7 +263,7 @@ Prerequisites installation should be specified as part of the ``CMAKE_PREFIX_PATH`` variable.:: - CXX=mpicxx cmake -DOPENMC_USE_LIBMESH=on -DCMAKE_PREFIX_PATH=/path/to/libmesh/installation + cmake -DOPENMC_USE_LIBMESH=on -DOPENMC_USE_MPI=on -DCMAKE_PREFIX_PATH=/path/to/libmesh/installation Note that libMesh is most commonly compiled with MPI support. If that is the case, then OpenMC should be compiled with MPI support as well. From 0c4488c52e0b6aca1ba6ef319a4055b62741e4aa Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Wed, 6 Apr 2022 09:36:31 -0500 Subject: [PATCH 09/15] update mpi setting in dockerfile --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6a9b890091..755d9f2a06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,8 +59,7 @@ ENV LIBMESH_INSTALL_DIR=$HOME/LIBMESH ENV NJOY_REPO='https://github.com/njoy/NJOY2016' # Setup environment variables for Docker image -ENV CC=/usr/bin/mpicc CXX=/usr/bin/mpicxx \ - LD_LIBRARY_PATH=${DAGMC_INSTALL_DIR}/lib:$LD_LIBRARY_PATH \ +ENV LD_LIBRARY_PATH=${DAGMC_INSTALL_DIR}/lib:$LD_LIBRARY_PATH \ OPENMC_CROSS_SECTIONS=/root/nndc_hdf5/cross_sections.xml \ OPENMC_ENDF_DATA=/root/endf-b-vii.1 \ DEBIAN_FRONTEND=noninteractive @@ -179,6 +178,7 @@ RUN mkdir -p ${HOME}/OpenMC && cd ${HOME}/OpenMC \ && mkdir build && cd build ; \ if [ ${build_dagmc} = "on" ] && [ ${build_libmesh} = "on" ]; then \ cmake ../openmc \ + -DOPENMC_USE_MPI=on \ -DHDF5_PREFER_PARALLEL=on \ -DOPENMC_USE_DAGMC=on \ -DOPENMC_USE_LIBMESH=on \ @@ -186,18 +186,21 @@ RUN mkdir -p ${HOME}/OpenMC && cd ${HOME}/OpenMC \ fi ; \ if [ ${build_dagmc} = "on" ] && [ ${build_libmesh} = "off" ]; then \ cmake ../openmc \ + -DOPENMC_USE_MPI=on \ -DHDF5_PREFER_PARALLEL=on \ -DOPENMC_USE_DAGMC=ON \ -DCMAKE_PREFIX_PATH=${DAGMC_INSTALL_DIR} ; \ fi ; \ if [ ${build_dagmc} = "off" ] && [ ${build_libmesh} = "on" ]; then \ cmake ../openmc \ + -DOPENMC_USE_MPI=on \ -DHDF5_PREFER_PARALLEL=on \ -DOPENMC_USE_LIBMESH=on \ -DCMAKE_PREFIX_PATH=${LIBMESH_INSTALL_DIR} ; \ fi ; \ if [ ${build_dagmc} = "off" ] && [ ${build_libmesh} = "off" ]; then \ cmake ../openmc \ + -DOPENMC_USE_MPI=on \ -DHDF5_PREFER_PARALLEL=on ; \ fi ; \ make 2>/dev/null -j${compile_cores} install \ From eedbfdc2b1e80e60dc524e0d3283c214f19943f1 Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Wed, 6 Apr 2022 13:29:34 -0500 Subject: [PATCH 10/15] removed unneeded lines --- CMakeLists.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 511ce66b06..d5018163f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,7 +47,6 @@ endif() if(OPENMC_USE_MPI) find_package(MPI REQUIRED) - include_directories(${MPI_CXX_INCLUDE_DIRS}) endif() #=============================================================================== @@ -141,11 +140,6 @@ endif() set(CMAKE_POSITION_INDEPENDENT_CODE ON) -if(OPENMC_USE_MPI) - list(APPEND cxxflags ${MPI_CXX_COMPILE_FLAGS}) - list(APPEND ldflags ${MPI_CXX_LINK_FLAGS}) -endif() - if(OPENMC_ENABLE_PROFILE) list(APPEND cxxflags -g -fno-omit-frame-pointer) endif() From 3bcc45c9bd2f41e571728f8a7d39957d32b61fb1 Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Wed, 6 Apr 2022 13:29:48 -0500 Subject: [PATCH 11/15] find mpi in cmake config --- cmake/OpenMCConfig.cmake.in | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/OpenMCConfig.cmake.in b/cmake/OpenMCConfig.cmake.in index 8f7ae71936..3279e5ba80 100644 --- a/cmake/OpenMCConfig.cmake.in +++ b/cmake/OpenMCConfig.cmake.in @@ -21,3 +21,7 @@ find_package(PNG) if(NOT TARGET OpenMC::libopenmc) include("${OpenMC_CMAKE_DIR}/OpenMCTargets.cmake") endif() + +if(@OPENMC_USE_MPI@) + find_package(MPI REQUIRED) +endif() From 3ba511c0566e5e26ab83a2c5c60ea261b3bdaf2e Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Wed, 6 Apr 2022 13:41:44 -0500 Subject: [PATCH 12/15] update mpi options --- docs/source/usersguide/install.rst | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/docs/source/usersguide/install.rst b/docs/source/usersguide/install.rst index 1cb00ae559..9a705b5fad 100644 --- a/docs/source/usersguide/install.rst +++ b/docs/source/usersguide/install.rst @@ -352,8 +352,12 @@ OPENMC_ENABLE_COVERAGE Compile and link code instrumented for coverage analysis. This is typically used in conjunction with gcov_. (Default: off) +OPENMC_USE_MPI + Turns on compiling with MPI (default: off). For further information on MPI options, + please see the `FindMPI.cmake documentation `_. + To set any of these options (e.g., turning on profiling), the following form -should be used: +should be used:ß .. code-block:: sh @@ -382,16 +386,7 @@ Example of configuring for Debug mode: .. code-block:: sh - cmake -DCMAKE_BUILD_TYPE=Debug /path/to/openmc - -Compiling with MPI -++++++++++++++++++ - -To compile with MPI, use the `-DOPENMC_USE_MPI=on` cmake option. For example, in a bash shell: - -.. code-block:: sh - - cmake -DOPENMC_USE_MPI=on /path/to/openmc + cmake -DCMAKE_BUILD_TYPE=Debug /path/to/openmcß Selecting HDF5 Installation +++++++++++++++++++++++++++ From f960553bedc3a82712474d1fa5b07b6f21017ea2 Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Wed, 6 Apr 2022 13:53:32 -0500 Subject: [PATCH 13/15] mpi_args formatting Co-authored-by: Paul Romano --- tests/regression_tests/cpp_driver/test.py | 2 +- tests/regression_tests/external_moab/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/regression_tests/cpp_driver/test.py b/tests/regression_tests/cpp_driver/test.py index 6870cfe7d7..95912b3f7b 100644 --- a/tests/regression_tests/cpp_driver/test.py +++ b/tests/regression_tests/cpp_driver/test.py @@ -40,7 +40,7 @@ def cpp_driver(request): try: print("Building driver") # Run cmake/make to build the shared libary - subprocess.run(['cmake', os.path.pardir, '-DOPENMC_USE_MPI=' + mpi_arg], check=True) + subprocess.run(['cmake', os.path.pardir, f'-DOPENMC_USE_MPI={mpi_arg}'], check=True) subprocess.run(['make'], check=True) os.chdir(os.path.pardir) diff --git a/tests/regression_tests/external_moab/test.py b/tests/regression_tests/external_moab/test.py index da55c735a9..47e3ef80ed 100644 --- a/tests/regression_tests/external_moab/test.py +++ b/tests/regression_tests/external_moab/test.py @@ -56,7 +56,7 @@ def cpp_driver(request): try: print("Building driver") # Run cmake/make to build the shared libary - subprocess.run(['cmake', os.path.pardir, '-DOPENMC_USE_MPI=' + mpi_arg], check=True) + subprocess.run(['cmake', os.path.pardir, f'-DOPENMC_USE_MPI={mpi_arg}'], check=True) subprocess.run(['make'], check=True) os.chdir(os.path.pardir) From 6208b4077021415249c7bf56d8a01df4108661ae Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Wed, 6 Apr 2022 14:09:47 -0500 Subject: [PATCH 14/15] fixing a typo because I mix up control and command keys on macs still --- docs/source/usersguide/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/usersguide/install.rst b/docs/source/usersguide/install.rst index 9a705b5fad..3277fc6e47 100644 --- a/docs/source/usersguide/install.rst +++ b/docs/source/usersguide/install.rst @@ -386,7 +386,7 @@ Example of configuring for Debug mode: .. code-block:: sh - cmake -DCMAKE_BUILD_TYPE=Debug /path/to/openmcß + cmake -DCMAKE_BUILD_TYPE=Debug /path/to/openmc Selecting HDF5 Installation +++++++++++++++++++++++++++ From ec4f1b71b54fabf9373e6d8b4e7a0d2add087865 Mon Sep 17 00:00:00 2001 From: Kalin Kiesling Date: Wed, 6 Apr 2022 14:14:12 -0500 Subject: [PATCH 15/15] fixing a typo because I mix up control and command keys on macs still - again --- docs/source/usersguide/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/usersguide/install.rst b/docs/source/usersguide/install.rst index 3277fc6e47..3e3f7754a6 100644 --- a/docs/source/usersguide/install.rst +++ b/docs/source/usersguide/install.rst @@ -357,7 +357,7 @@ OPENMC_USE_MPI please see the `FindMPI.cmake documentation `_. To set any of these options (e.g., turning on profiling), the following form -should be used:ß +should be used: .. code-block:: sh