diff --git a/tests/regression_tests/model_xml/test.py b/tests/regression_tests/model_xml/test.py
index 5bfe912a44..0dc2cc2898 100644
--- a/tests/regression_tests/model_xml/test.py
+++ b/tests/regression_tests/model_xml/test.py
@@ -93,5 +93,6 @@ def test_input_arg(run_in_tmpdir):
 
     # now ensure we get an error for an incorrect filename,
     # even in the presence of other, valid XML files
-    with pytest.raises(RuntimeError):
-        openmc.run(input_file='ex-em-ell.xml')
\ No newline at end of file
+    with pytest.raises(RuntimeError) as execinfo:
+        openmc.run(input_file='ex-em-ell.xml')
+        assert 'ex-em-ell.xml' in execinfo.value
\ No newline at end of file