From 309a913b06ccbe542c4fe62f0543771d5be74968 Mon Sep 17 00:00:00 2001 From: Sterling Harper Date: Tue, 7 Aug 2018 16:51:39 -0400 Subject: [PATCH 1/7] Add statepoint with source_bank to C-API Also simplify logic for write_source_point --- openmc/capi/core.py | 8 ++-- src/simulation.F90 | 26 ++++++++++--- src/state_point.F90 | 89 ++++++++++++++++++++------------------------- 3 files changed, 66 insertions(+), 57 deletions(-) diff --git a/openmc/capi/core.py b/openmc/capi/core.py index a1f908c1c..903999628 100644 --- a/openmc/capi/core.py +++ b/openmc/capi/core.py @@ -52,7 +52,7 @@ _dll.openmc_simulation_init.restype = c_int _dll.openmc_simulation_init.errcheck = _error_handler _dll.openmc_simulation_finalize.restype = c_int _dll.openmc_simulation_finalize.errcheck = _error_handler -_dll.openmc_statepoint_write.argtypes = [POINTER(c_char_p)] +_dll.openmc_statepoint_write.argtypes = [POINTER(c_char_p), POINTER(c_int)] _dll.openmc_statepoint_write.restype = c_int _dll.openmc_statepoint_write.errcheck = _error_handler @@ -269,7 +269,7 @@ def source_bank(): return as_array(ptr, (n.value,)).view(bank_dtype) -def statepoint_write(filename=None): +def statepoint_write(filename=None, write_source=False): """Write a statepoint file. Parameters @@ -277,11 +277,13 @@ def statepoint_write(filename=None): filename : str or None Path to the statepoint to write. If None is passed, a default name that contains the current batch will be written. + write_source : bool + Whether or not to include the source bank in the statepoint. """ if filename is not None: filename = c_char_p(filename.encode()) - _dll.openmc_statepoint_write(filename) + _dll.openmc_statepoint_write(filename, c_int(write_source)) @contextmanager diff --git a/src/simulation.F90 b/src/simulation.F90 index c8c9dfcf8..cd7e68da4 100644 --- a/src/simulation.F90 +++ b/src/simulation.F90 @@ -317,6 +317,7 @@ contains #ifdef OPENMC_MPI integer :: mpi_err ! MPI error code #endif + character(MAX_FILE_LEN) :: filename ! Reduce tallies onto master process and accumulate call time_tallies % start() @@ -349,14 +350,29 @@ contains ! Write out state point if it's been specified for this batch if (statepoint_batch % contains(current_batch)) then - err = openmc_statepoint_write() + if (sourcepoint_batch % contains(current_batch) .and. source_write & + .and. .not. source_separate) then + err = openmc_statepoint_write(write_source=1) + else + err = openmc_statepoint_write(write_source=0) + end if + end if + + ! Write out a separate source point if it's been specified for this batch + if (sourcepoint_batch % contains(current_batch) .and. source_write & + .and. source_separate) call write_source_point() + + ! Write a continously-overwritten source point if requested. + if (source_latest) then + filename = trim(path_output) // 'source' // '.h5' + call write_source_point(filename) end if ! Write out source point if it's been specified for this batch - if ((sourcepoint_batch % contains(current_batch) .or. source_latest) .and. & - source_write) then - call write_source_point() - end if + !if ((sourcepoint_batch % contains(current_batch) .or. source_latest) .and. & + ! source_write) then + ! call write_source_point() + !end if end subroutine finalize_batch diff --git a/src/state_point.F90 b/src/state_point.F90 index dbc55be7a..4d66a37aa 100644 --- a/src/state_point.F90 +++ b/src/state_point.F90 @@ -58,8 +58,9 @@ contains ! OPENMC_STATEPOINT_WRITE writes an HDF5 statepoint file to disk !=============================================================================== - function openmc_statepoint_write(filename) result(err) bind(C) - type(C_PTR), intent(in), optional :: filename + function openmc_statepoint_write(filename, write_source) result(err) bind(C) + type(C_PTR), intent(in), optional :: filename + integer(C_INT), intent(in), optional :: write_source integer(C_INT) :: err integer :: i, j, k @@ -74,6 +75,7 @@ contains character(C_CHAR), pointer :: string(:) character(len=:, kind=C_CHAR), allocatable :: filename_ character(MAX_WORD_LEN, kind=C_CHAR) :: temp_name + logical :: parallel err = 0 if (present(filename)) then @@ -137,13 +139,6 @@ contains ! Write out current batch number call write_dataset(file_id, "current_batch", current_batch) - ! Indicate whether source bank is stored in statepoint - if (source_separate) then - call write_attribute(file_id, "source_present", 0) - else - call write_attribute(file_id, "source_present", 1) - end if - ! Write out information for eigenvalue run if (run_mode == MODE_EIGENVALUE) then call write_dataset(file_id, "n_inactive", n_inactive) @@ -432,17 +427,40 @@ contains call file_close(file_id) end if + +#ifdef PHDF5 + parallel = .true. +#else + parallel = .false. +#endif + + ! Write the source bank if desired + if (present(write_source)) then + if (write_source /= 0) then + if (master .or. parallel) then + file_id = file_open(filename_, 'a', parallel=.true.) + end if + call write_source_bank(file_id, work_index, source_bank) + call write_attribute(file_id, "source_present", 1) + if (master .or. parallel) call file_close(file_id) + else + call write_attribute(file_id, "source_present", 0) + end if + else + call write_attribute(file_id, "source_present", 0) + end if end function openmc_statepoint_write !=============================================================================== ! WRITE_SOURCE_POINT !=============================================================================== - subroutine write_source_point() + subroutine write_source_point(filename) + character(MAX_FILE_LEN), intent(in), optional :: filename logical :: parallel integer(HID_T) :: file_id - character(MAX_FILE_LEN) :: filename + character(MAX_FILE_LEN) :: filename_ ! When using parallel HDF5, the file is written to collectively by all ! processes. With MPI-only, the file is opened and written by the master @@ -454,47 +472,20 @@ contains parallel = .false. #endif - ! Check to write out source for a specified batch - if (sourcepoint_batch%contains(current_batch)) then - if (source_separate) then - filename = trim(path_output) // 'source.' // & - & zero_padded(current_batch, count_digits(n_max_batches)) - filename = trim(filename) // '.h5' - call write_message("Creating source file " // trim(filename) & - // "...", 5) - - ! Create separate source file - if (master .or. parallel) then - file_id = file_open(filename, 'w', parallel=.true.) - call write_attribute(file_id, "filetype", 'source') - end if - else - filename = trim(path_output) // 'statepoint.' // & - zero_padded(current_batch, count_digits(n_max_batches)) - filename = trim(filename) // '.h5' - - if (master .or. parallel) then - file_id = file_open(filename, 'a', parallel=.true.) - end if - end if - - call write_source_bank(file_id, work_index, source_bank) - if (master .or. parallel) call file_close(file_id) + if (present(filename)) then + filename_ = filename + else + filename_ = trim(path_output) // 'source.' // & + & zero_padded(current_batch, count_digits(n_max_batches)) + filename_ = trim(filename_) // '.h5' end if - ! Also check to write source separately in overwritten file - if (source_latest) then - filename = trim(path_output) // 'source' // '.h5' - call write_message("Creating source file " // trim(filename) // "...", 5) - if (master .or. parallel) then - file_id = file_open(filename, 'w', parallel=.true.) - call write_attribute(file_id, "filetype", 'source') - end if - - call write_source_bank(file_id, work_index, source_bank) - - if (master .or. parallel) call file_close(file_id) + if (master .or. parallel) then + file_id = file_open(filename_, 'w', parallel=.true.) + call write_attribute(file_id, "filetype", 'source') end if + call write_source_bank(file_id, work_index, source_bank) + if (master .or. parallel) call file_close(file_id) end subroutine write_source_point From 2ff8eb4696f315f6f47a7d5b7a7e200a6647ea48 Mon Sep 17 00:00:00 2001 From: Sterling Harper Date: Tue, 7 Aug 2018 17:03:38 -0400 Subject: [PATCH 2/7] Add a C-API based restart test --- src/state_point.F90 | 3 ++- tests/unit_tests/test_capi.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/state_point.F90 b/src/state_point.F90 index 4d66a37aa..3f1d5b397 100644 --- a/src/state_point.F90 +++ b/src/state_point.F90 @@ -70,6 +70,7 @@ contains integer(HID_T) :: cmfd_group, tallies_group, tally_group, meshes_group, & filters_group, filter_group, derivs_group, & deriv_group, runtime_group + integer(C_INT) :: ignored_err real(C_DOUBLE) :: k_combined(2) character(MAX_WORD_LEN), allocatable :: str_array(:) character(C_CHAR), pointer :: string(:) @@ -151,7 +152,7 @@ contains call write_dataset(file_id, "k_col_abs", k_col_abs) call write_dataset(file_id, "k_col_tra", k_col_tra) call write_dataset(file_id, "k_abs_tra", k_abs_tra) - err = openmc_get_keff(k_combined) + ignored_err = openmc_get_keff(k_combined) call write_dataset(file_id, "k_combined", k_combined) ! Write out CMFD info diff --git a/tests/unit_tests/test_capi.py b/tests/unit_tests/test_capi.py index af013cbb5..288676ff7 100644 --- a/tests/unit_tests/test_capi.py +++ b/tests/unit_tests/test_capi.py @@ -311,3 +311,31 @@ def test_mesh(capi_init): msf = openmc.capi.MeshSurfaceFilter(mesh) assert msf.mesh == mesh + + +def test_restart(pincell_model): + # Run for 7 batches then write a statepoint. + openmc.capi.init() + openmc.capi.simulation_init() + for i in range(7): + openmc.capi.next_batch() + openmc.capi.statepoint_write('restart_test.h5', True) + + # Run 3 more batches and copy the keff. + for i in range(3): + openmc.capi.next_batch() + keff0 = openmc.capi.keff() + openmc.capi.simulation_finalize() + openmc.capi.finalize() + + # Restart the simulation and run all 10 batches. + openmc.capi.init(args=('-r', 'restart_test.h5')) + openmc.capi.simulation_init() + for i in range(10): + openmc.capi.next_batch() + keff1 = openmc.capi.keff() + openmc.capi.simulation_finalize() + openmc.capi.finalize() + + # Compare the keff values. + assert keff0 == pytest.approx(keff1) From 0f4159ca9738f131170f88ce7538f68a4ca719ca Mon Sep 17 00:00:00 2001 From: Sterling Harper Date: Tue, 7 Aug 2018 17:06:53 -0400 Subject: [PATCH 3/7] Remove commented-out code --- src/simulation.F90 | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/simulation.F90 b/src/simulation.F90 index cd7e68da4..ecc31dbd0 100644 --- a/src/simulation.F90 +++ b/src/simulation.F90 @@ -368,12 +368,6 @@ contains call write_source_point(filename) end if - ! Write out source point if it's been specified for this batch - !if ((sourcepoint_batch % contains(current_batch) .or. source_latest) .and. & - ! source_write) then - ! call write_source_point() - !end if - end subroutine finalize_batch !=============================================================================== From b5dca6baff9c1433c0f71963250406e4e376c996 Mon Sep 17 00:00:00 2001 From: Sterling Harper Date: Tue, 7 Aug 2018 18:45:10 -0400 Subject: [PATCH 4/7] Fix writing of "source_present" to statepoint --- src/state_point.F90 | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/state_point.F90 b/src/state_point.F90 index 3f1d5b397..efe999f8c 100644 --- a/src/state_point.F90 +++ b/src/state_point.F90 @@ -140,6 +140,17 @@ contains ! Write out current batch number call write_dataset(file_id, "current_batch", current_batch) + ! Indicate whether source bank is stored in statepoint + if (present(write_source)) then + if (write_source /= 0) then + call write_attribute(file_id, "source_present", 1) + else + call write_attribute(file_id, "source_present", 0) + end if + else + call write_attribute(file_id, "source_present", 0) + end if + ! Write out information for eigenvalue run if (run_mode == MODE_EIGENVALUE) then call write_dataset(file_id, "n_inactive", n_inactive) @@ -442,13 +453,8 @@ contains file_id = file_open(filename_, 'a', parallel=.true.) end if call write_source_bank(file_id, work_index, source_bank) - call write_attribute(file_id, "source_present", 1) if (master .or. parallel) call file_close(file_id) - else - call write_attribute(file_id, "source_present", 0) end if - else - call write_attribute(file_id, "source_present", 0) end if end function openmc_statepoint_write From 4b62006d23f679aa0ce7a089c4df1e7a674c619f Mon Sep 17 00:00:00 2001 From: Sterling Harper Date: Wed, 8 Aug 2018 10:54:53 -0400 Subject: [PATCH 5/7] Fix capi restart unit test --- tests/unit_tests/test_capi.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/unit_tests/test_capi.py b/tests/unit_tests/test_capi.py index 288676ff7..4b80ba7eb 100644 --- a/tests/unit_tests/test_capi.py +++ b/tests/unit_tests/test_capi.py @@ -313,10 +313,14 @@ def test_mesh(capi_init): assert msf.mesh == mesh -def test_restart(pincell_model): - # Run for 7 batches then write a statepoint. +def test_restart(capi_init): + # Finalize and re-init to make internal state consistent with XML. + openmc.capi.hard_reset() + openmc.capi.finalize() openmc.capi.init() openmc.capi.simulation_init() + + # Run for 7 batches then write a statepoint. for i in range(7): openmc.capi.next_batch() openmc.capi.statepoint_write('restart_test.h5', True) @@ -325,17 +329,17 @@ def test_restart(pincell_model): for i in range(3): openmc.capi.next_batch() keff0 = openmc.capi.keff() - openmc.capi.simulation_finalize() - openmc.capi.finalize() - # Restart the simulation and run all 10 batches. + # Restart the simulation from the statepoint and run all 10 batches. + openmc.capi.simulation_finalize() + openmc.capi.hard_reset() + openmc.capi.finalize() openmc.capi.init(args=('-r', 'restart_test.h5')) openmc.capi.simulation_init() for i in range(10): openmc.capi.next_batch() keff1 = openmc.capi.keff() openmc.capi.simulation_finalize() - openmc.capi.finalize() # Compare the keff values. assert keff0 == pytest.approx(keff1) From d5a85164e23d9ca396afbaf6e9d636e860342885 Mon Sep 17 00:00:00 2001 From: Sterling Harper Date: Wed, 8 Aug 2018 12:19:59 -0400 Subject: [PATCH 6/7] Include source bank in statepoint by default --- openmc/capi/core.py | 10 +++++----- src/simulation.F90 | 4 ++-- src/state_point.F90 | 34 +++++++++++++++++++--------------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/openmc/capi/core.py b/openmc/capi/core.py index 903999628..b5ada7956 100644 --- a/openmc/capi/core.py +++ b/openmc/capi/core.py @@ -1,6 +1,6 @@ from contextlib import contextmanager -from ctypes import (CDLL, c_int, c_int32, c_int64, c_double, c_char_p, c_char, - POINTER, Structure, c_void_p, create_string_buffer) +from ctypes import (CDLL, c_bool, c_int, c_int32, c_int64, c_double, c_char_p, + c_char, POINTER, Structure, c_void_p, create_string_buffer) from warnings import warn import numpy as np @@ -52,7 +52,7 @@ _dll.openmc_simulation_init.restype = c_int _dll.openmc_simulation_init.errcheck = _error_handler _dll.openmc_simulation_finalize.restype = c_int _dll.openmc_simulation_finalize.errcheck = _error_handler -_dll.openmc_statepoint_write.argtypes = [POINTER(c_char_p), POINTER(c_int)] +_dll.openmc_statepoint_write.argtypes = [POINTER(c_char_p), POINTER(c_bool)] _dll.openmc_statepoint_write.restype = c_int _dll.openmc_statepoint_write.errcheck = _error_handler @@ -269,7 +269,7 @@ def source_bank(): return as_array(ptr, (n.value,)).view(bank_dtype) -def statepoint_write(filename=None, write_source=False): +def statepoint_write(filename=None, write_source=True): """Write a statepoint file. Parameters @@ -283,7 +283,7 @@ def statepoint_write(filename=None, write_source=False): """ if filename is not None: filename = c_char_p(filename.encode()) - _dll.openmc_statepoint_write(filename, c_int(write_source)) + _dll.openmc_statepoint_write(filename, c_bool(write_source)) @contextmanager diff --git a/src/simulation.F90 b/src/simulation.F90 index ecc31dbd0..4ddf54169 100644 --- a/src/simulation.F90 +++ b/src/simulation.F90 @@ -352,9 +352,9 @@ contains if (statepoint_batch % contains(current_batch)) then if (sourcepoint_batch % contains(current_batch) .and. source_write & .and. .not. source_separate) then - err = openmc_statepoint_write(write_source=1) + err = openmc_statepoint_write(write_source=.true._C_BOOL) else - err = openmc_statepoint_write(write_source=0) + err = openmc_statepoint_write(write_source=.false._C_BOOL) end if end if diff --git a/src/state_point.F90 b/src/state_point.F90 index efe999f8c..b679e43fd 100644 --- a/src/state_point.F90 +++ b/src/state_point.F90 @@ -59,10 +59,11 @@ contains !=============================================================================== function openmc_statepoint_write(filename, write_source) result(err) bind(C) - type(C_PTR), intent(in), optional :: filename - integer(C_INT), intent(in), optional :: write_source + type(C_PTR), intent(in), optional :: filename + logical(C_BOOL), intent(in), optional :: write_source integer(C_INT) :: err + logical :: write_source_ integer :: i, j, k integer :: i_xs integer, allocatable :: id_array(:) @@ -79,6 +80,8 @@ contains logical :: parallel err = 0 + + ! Set the filename if (present(filename)) then call c_f_pointer(filename, string, [MAX_FILE_LEN]) filename_ = to_f_string(string) @@ -89,6 +92,13 @@ contains filename_ = trim(filename_) // '.h5' end if + ! Determine whether or not to write the source bank + if (present(write_source)) then + write_source_ = write_source + else + write_source_ = .true. + end if + ! Write message call write_message("Creating state point " // trim(filename_) // "...", 5) @@ -141,12 +151,8 @@ contains call write_dataset(file_id, "current_batch", current_batch) ! Indicate whether source bank is stored in statepoint - if (present(write_source)) then - if (write_source /= 0) then - call write_attribute(file_id, "source_present", 1) - else - call write_attribute(file_id, "source_present", 0) - end if + if (write_source_) then + call write_attribute(file_id, "source_present", 1) else call write_attribute(file_id, "source_present", 0) end if @@ -447,14 +453,12 @@ contains #endif ! Write the source bank if desired - if (present(write_source)) then - if (write_source /= 0) then - if (master .or. parallel) then - file_id = file_open(filename_, 'a', parallel=.true.) - end if - call write_source_bank(file_id, work_index, source_bank) - if (master .or. parallel) call file_close(file_id) + if (write_source_) then + if (master .or. parallel) then + file_id = file_open(filename_, 'a', parallel=.true.) end if + call write_source_bank(file_id, work_index, source_bank) + if (master .or. parallel) call file_close(file_id) end if end function openmc_statepoint_write From 7994af72fa22b466ff8c27fcc93e9cc5064ddc61 Mon Sep 17 00:00:00 2001 From: Sterling Harper Date: Tue, 14 Aug 2018 11:08:43 -0400 Subject: [PATCH 7/7] Fix capi restart test --- tests/unit_tests/test_capi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit_tests/test_capi.py b/tests/unit_tests/test_capi.py index 4b80ba7eb..629dbf7d7 100644 --- a/tests/unit_tests/test_capi.py +++ b/tests/unit_tests/test_capi.py @@ -330,13 +330,13 @@ def test_restart(capi_init): openmc.capi.next_batch() keff0 = openmc.capi.keff() - # Restart the simulation from the statepoint and run all 10 batches. + # Restart the simulation from the statepoint and the 5 active batches. openmc.capi.simulation_finalize() openmc.capi.hard_reset() openmc.capi.finalize() openmc.capi.init(args=('-r', 'restart_test.h5')) openmc.capi.simulation_init() - for i in range(10): + for i in range(5): openmc.capi.next_batch() keff1 = openmc.capi.keff() openmc.capi.simulation_finalize()