comp-chem/cp2k
1
0
Fork 0
mirror of https://github.com/cp2k/cp2k.git synced 2026-07-28 06:05:29 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions
cp2k/tools/regtesting/do_regtest

1218 lines
44 KiB
Bash
Executable file
Raw Permalink Blame History

#!/bin/bash
###################################################################################
# do_regtest some info [Joost VandeVondele, 2005-02-25]
# Major revision of the script [Marko Misic, PRACE, July - August 2013]
###################################################################################
###################################################################################
#
# command line argument passing
#
###################################################################################
# ensure error messages are in English
export LC_ALL=C
# init
dir_base=$PWD
full_time_t1=$(date +%s)
date_stamp=$(date '+%F_%H-%M-%S')
ndirstoskip=0
skip_dirs[1]=""
ndirstorestrict=0
restrict_dirs[1]=""
default_err_tolerance="1.0E-14"
mpiexec="mpiexec --bind-to none"
ishard=1
nshards=1
# parse options
while [ $# -ge 1 ]; do
case $1 in
# load system-specific configuration file, which can directly change all variables in this script.. powerful ;-)
-c|-config)
[[ -f $2 ]] || { echo "ERROR: Configuration file $2 not found"; exit 1; }
source $2 || { echo "ERROR: Sourcing of the configuration file $2 failed" ; exit 1; }
shift;;
# make VERSION=...
-version)
cp2k_version=$2
shift ;;
# make ARCH=...
-arch)
dir_triplet=$2
shift ;;
# location of cp2k relative the the current working directory (i.e. ${dir_base}/${cp2k_dir}/exe can be found)
-cp2kdir)
cp2k_dir=$2
shift ;;
# location for the arch files
-archdir)
arch_dir=$2
shift ;;
# root path of output folder (default is ${dir_base})
-dirout)
dir_out=$2
shift ;;
# MPI executable
-mpiexec)
mpiexec=$2
shift;;
# MPI ranks
-mpiranks)
numprocs=$2
shift;;
# OMP threads
-ompthreads)
numthreads=$2
shift;;
# maximum total number of tasks (processes)
-maxtasks)
maxtasks=$2
shift;;
-maxbuildtasks)
maxbuildtasks=$2
shift;;
# maximum time for a job [in s]
-jobmaxtime)
job_max_time=$2
shift;;
# sharding
-shard)
let ishard=$2
let nshards=$3
(( ishard < 1 )) && { echo "ERROR: Shard number too low."; exit 1; }
(( nshards < 1 )) && { echo "ERROR: Number of shards too low."; exit 1; }
(( ishard > nshards )) && { echo "ERROR: Shard number too high."; exit 1; }
shift 2;;
# build a list of directories to skip
-skipdir)
let ndirstoskip=ndirstoskip+1
skip_dirs[ndirstoskip]=$2
shift;;
-skipunittest)
do_unit_test="no" ;;
# build a list of directories to restrict, i.e. only matching dirs will be run
-restrictdir)
let ndirstorestrict=ndirstorestrict+1
restrict_dirs[ndirstorestrict]=$2
shift;;
# enable partial testing of only those directories that contain failed tests
-retest)
doretest="yes";;
# do not do a realclean before building
-quick)
quick="quick";;
# Overwrite internal date stamp
-date_stamp)
date_stamp=$2
shift;;
-farming)
farming="yes";;
# do not check if code has changed
-nobuild)
quick="quick"; nobuild="nobuild";;
# do not test, actually.
-skiptest)
skiptest="skiptest";;
# print help
-help|-h|--help)
echo "Usage: do_regtest [OPTION]"
echo "Run the CP2K regression test suite"
echo "Example: do_regtest -c my_regtest.conf"
echo ""
echo "General:"
echo " -h, -help, --help print this help screen."
echo " -c, -config FILE read any of the following configuration switches from FILE."
echo " -cp2kdir PATH location of cp2k source tree relative to current working directory."
echo " -dirout PATH root path of output folder (default: current working directory)."
echo " -mpiexec EXE name of the executable to run mpi-ranks. Default: mpiexec."
echo ""
echo "Build:"
echo " -version VERSION VERSION passed to make. Default: sopt."
echo " -arch ARCH ARCH passed to make. Default: Linux-x86-64-gfortran."
echo " -quick rebuild if needed, but without realclean. Default: off."
echo " -nobuild do not build cp2k, rely on user's build. Default: off."
echo ""
echo "Runtime:"
echo " -mpiranks NRANKS number of mpi-ranks. Default: 2 for parallel versions, 1 for serial."
echo " -ompthreads NTHREADS number of OpenMP threads. Default: 2 for smp versions otherwise 1."
echo " -maxtasks NPROCS total number of processor to use. Default: \`nproc\`."
echo " -maxbuildtasks NPROCS total number of processor to use for compilation. Default taken from -maxtasks."
echo " -jobmaxtime SECONDS maximum execution time of a single test. Default 600."
echo " -farming test via a single farming run (only for parallel cp2k)."
echo ""
echo "Testing:"
echo " -shard ISHARD NSHARDS do sharding: Divide the test dirs into NSHARD subsets and only run the ISHARD-th set."
echo " -skiptest do not run test, only build. Default: off."
echo " -skipunittest do not run unit tests. Default: off."
echo " -skipdir TESTDIR do not run tests in TESTDIR. This switch can repeated."
echo " -restrictdir TESTDIR run only tests in TESTDIR. This switch can repeated."
echo " -retest run only tests in directories, which had failing tests in previous run."
echo ""
echo "Exit codes:"
echo " 0 clean exit with testing"
echo " 3 problem with realclean"
echo " 4 build errors"
echo " 5 problem with retest option - no TEST directory with latest test results found"
echo " 6 problem with retest option - no error summary exists in the last TEST directory"
echo " 7 reference directory is locked"
echo " 8 ctrl-C (SIGINT) and various other signals trapped"
echo ""
echo "For more information visit: <http://cp2k.org/dev:regtesting>"
exit 0;;
# stop on invalid flag
-*)
echo "ERROR: Invalid command line flag $1 found"
exit 1;;
# Default case
*)
echo "ERROR: Unknown command line string $1 found"
exit 1;;
esac
shift
done
###################################################################################
#
# default settings for various flags, if not defined by flags or conf files
#
###################################################################################
cp2k_version=${cp2k_version:-sopt}
dir_out=${dir_out:-${dir_base}}
dir_triplet=${dir_triplet:-Linux-x86-64-gfortran}
leakcheck=${leakcheck:-"NO"}
doretest=${doretest:-"no"}
nobuild=${nobuild:-"build"}
cp2k_run_postfix=${cp2k_run_postfix:-""}
make=${make:-make}
awk=${awk:-awk}
quick=${quick:-"noquick"}
skiptest=${skiptest:-"noskiptest"}
do_unit_test=${do_unit_test:-"yes"}
farming=${farming:-"no"}
#
# guessing the number of tasks / threads / ranks
#
# by default we use all available processors, obeys e.g. docker --cpuset-cpus
maxtasksdefault=$(nproc)
maxtasks=${maxtasks:-${maxtasksdefault}}
maxbuildtasks=${maxbuildtasks:-${maxtasks}}
if [[ $dir_triplet == *valgrind* ]] ; then
valgrindstring="valgrind $VALGRIND_OPTIONS"
job_max_time=${job_max_time:-7200}
else
valgrindstring=""
job_max_time=${job_max_time:-400}
fi
if [[ $cp2k_version == p* ]] ; then
numprocs=${numprocs:-2}
else
numprocs=${numprocs:-1}
fi
mem_limit=unlimited
if [[ $numprocs -gt 1 ]] ; then
if [[ "$farming" == "yes" ]]; then
cp2k_run_prefix=${cp2k_run_prefix:-"${mpiexec} -np $(($maxtasks + 1)) ${valgrindstring}"}
else
cp2k_run_prefix=${cp2k_run_prefix:-"${mpiexec} -np ${numprocs} ${valgrindstring}"}
fi
else
cp2k_run_prefix=${cp2k_run_prefix:-"${valgrindstring}"}
fi
if [[ $cp2k_version == *smp* ]] ; then
numthreads=${numthreads:-2}
else
numthreads=${numthreads:-1}
fi
export OMP_NUM_THREADS=${numthreads}
export CP2K_DATA_DIR=${cp2k_data_dir:-"${dir_base}/${cp2k_dir}/data"}
#
# these should not be needed
#
export ARCH=${dir_triplet}
ARCH_SPEC="ARCH=${ARCH}${arch_dir:+ ARCHDIR=$arch_dir} "
cp2k_prefix=${cp2k_prefix:-"${cp2k_run_prefix} ${dir_base}/${cp2k_dir}/exe/${dir_triplet}/cp2k.${cp2k_version}"}
cp2k_postfix=${cp2k_postfix:-"${cp2k_run_postfix}"}
###################################################################################
#
# the rest should be internals.
#
###################################################################################
###################################################################################
#
# set up the initial directory structures
#
###################################################################################
repo_dir=${dir_base}/${cp2k_dir}
test_types_file=${repo_dir}/tests/TEST_TYPES
lastdir=LAST-${dir_triplet}-${cp2k_version}
dir_last=${dir_base}/${lastdir}
testdir=TEST-${dir_triplet}-${cp2k_version}-${date_stamp}
dir_test=${dir_out}/${testdir}
changelog=${dir_last}/ChangeLog
changelog_diff=${changelog}.diff
changelog_new=${changelog}.new
changelog_old=${changelog}.old
changelog_tests=${changelog}-tests
changelog_tests_diff=${changelog_tests}.diff
changelog_tests_new=${changelog_tests}.new
changelog_tests_old=${changelog_tests}.old
mkdir -p ${dir_test}
mkdir -p ${dir_last}
make_out=${dir_test}/make.out
error_description_file=${dir_test}/error_summary
>${error_description_file}
memory_description_file=${dir_test}/memory_summary
>${memory_description_file}
summary=${dir_test}/summary.txt
timings=${dir_test}/timings.txt
printf "Summary of the regression tester run from ${date_stamp} using ${dir_triplet} ${cp2k_version} \n" >${summary}
###################################################################################
#
# function to grep for changes in the output. Takes five arguments
#
###################################################################################
function do_test_grep(){
output_new=$1
output_old=$2
error_file=$3
grep_string=$4
grep_field=$5
error_tolerance=$6
ref_value=$7
if [ -n "${ref_value}" ]; then
e1=$ref_value
label="ref"
else
e1=$(grep -a "${grep_string}" ${output_old} | tail -1 | ${awk} -v f=${grep_field} '{print $f}')
label="old"
fi
e2=$(grep -a "${grep_string}" ${output_new} | tail -1 | ${awk} -v f=${grep_field} '{print $f}')
big=$(echo "${e1} ${e2} ${error_tolerance}" | ${awk} '{if($2==0){v=sqrt(($1-$2)^2)}else{v=sqrt((($1-$2)/$2)^2)}; if (v>$3) printf("%16.8e",v); else printf("0") ;}')
case ${big} in
0)
# ok, same energy
return 0 ;;
*)
# nope too large
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >>${error_file}
echo "${output_new} : " >> ${error_file}
echo " ${grep_string} : ${label} = ${e1} new = ${e2} " >> ${error_file}
echo " relative error : ${big} > numerical tolerance = ${error_tolerance} " >>${error_file}
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >>${error_file}
return 1 ;;
esac
}
###################################################################################
#
# function to select which test to run
#
###################################################################################
function do_test() {
which_test=$1
num_tolerance=$2
ref_value=$3
output_new=$4
output_old=$5
error_file=$6
case ${which_test} in
0)
#just be happy you executed
return 0;;
*)
do_test_grep ${output_new} ${output_old} ${error_file} "${test_grep[which_test]}" "${test_col[which_test]}" "${num_tolerance}" "${ref_value}"
return $? ;;
esac
}
###################################################################################
#
# function to run an entire regtest dir
#
###################################################################################
function run_regtest_dir() {
dir=$1
task=${dir//\//-}
touch ${dir_test}/status/REGTEST_RUNNING-$task
n_runtime_error=0
n_wrong_results=0
n_correct=0
n_tests=0
cd ${dir_test}/tests/${dir}
mkdir -p ${dir_test}/${dir}
mkdir -p ${dir_last}/${dir}
#
# run the tests now
#
echo "Starting regression tests in ${dir_test}/tests/${dir} (${idir} of ${ndir})"
echo ">>> ${dir_test}/tests/${dir}" > ${dir_test}/status/REGTEST_TASK_TESTS-$task
ntest=$(grep '^\s*\w' TEST_FILES | wc -l)
t1=$(date +%s)
for ((itest=1;itest<=ntest;itest++));
do
touch ${lockfile}
n_tests=$((n_tests+1))
this_test=""
input_file=$(grep '^\s*\w' TEST_FILES | ${awk} -v itest=$itest '{c=c+1;if (c==itest) print $1}')
test_type=$(grep '^\s*\w' TEST_FILES | ${awk} -v itest=$itest '{c=c+1;if (c==itest) print $2}')
# third field allows numerical tolerances to be read from the TEST_FILES
# if value does not exist set to the default of 1.0E-14
test_tolerance=$(grep '^\s*\w' TEST_FILES | ${awk} -v itest=$itest -v def_err_tol=$default_err_tolerance '{c=c+1;if (c==itest) if (NF >= 3) { print $3 } else { print def_err_tol } }')
test_ref_value=$(grep '^\s*\w' TEST_FILES | ${awk} -v itest=$itest '{c=c+1;if (c==itest) if (NF == 4) { print $4 } }')
output_file=${dir_test}/${dir}/${input_file}.out
output_last=${dir_last}/${dir}/${input_file}.out
#
# look here for the actual place where cp2k runs....
#
if [[ "$farming" == "yes" ]]; then
if [ -f ${output_file} ]; then
cp2k_exit_status=0
else
cp2k_exit_status=43
fi
else
( ulimit -t ${job_max_time} -v ${mem_limit} ; ${cp2k_prefix} ${input_file} ${cp2k_postfix} &> ${output_file} ) >& /dev/null
(( cp2k_exit_status = $? ))
fi
# check if this was a valgrinded run, and adjust exit status as needed
grep -a "^==[0-9][0-9]*== ERROR SUMMARY" ${output_file} &> /dev/null
if (( $? == 0 )); then
grep -a "^==[0-9][0-9]*== ERROR SUMMARY: 0 errors from 0 contexts" ${output_file} &> /dev/null
# nonzero error count
if (( $? )); then
cp2k_exit_status=42
fi
fi
if (( cp2k_exit_status )); then
# CP2K failed obviously
if (( cp2k_exit_status == 137 )); then
# SIGKILL = 9 ... exit code 9+128
# usually caused by time-out
this_test="KILLED"
else
if (( cp2k_exit_status == 43 )); then
this_test="FAILED START"
else
this_test="RUNTIME FAIL"
fi
fi
# failed starts (farming) are not interesting
if (( cp2k_exit_status != 43 )); then
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >>${error_description_file}
echo ${output_file} >>${error_description_file}
tail -n 100 ${output_file} >>${error_description_file}
# append an eventual valgrind report to the file
grep "^==[0-9][0-9]*== " ${output_file} | head -n 20 >> ${error_description_file}
echo "EXIT CODE: " $cp2k_exit_status " MEANING: " $this_test >>${error_description_file}
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >>${error_description_file}
n_runtime_error=$((n_runtime_error+1))
failed_tests="${failed_tests} ${output_file}"
fi
else
# ran but did not end !?
grep -a "ENDED" ${output_file} &> /dev/null
if (( $? )); then
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >>${error_description_file}
echo ${output_file} >>${error_description_file}
tail -n 100 ${output_file} >>${error_description_file}
# append an eventual valgrind report to the file
grep "^==[0-9][0-9]*== " ${output_file} | head -n 20 >> ${error_description_file}
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >>${error_description_file}
this_test="RUNTIME FAIL"
n_runtime_error=$((n_runtime_error+1))
failed_tests="${failed_tests} ${output_file}"
else
# Still running, you must be joking...
# see if we manage to pass the testing
# but only if we can compare (or don't need to)
if [ -f ${output_last} ] || [ -n "${test_ref_value}" ] || [ "${test_type}" == "0" ]; then
do_test "${test_type}" "${test_tolerance}" "${test_ref_value}" "${output_file}" "${output_last}" "${error_description_file}"
if (( $? )); then
this_test="WRONG RESULT TEST ${test_type}"
n_wrong_results=$((n_wrong_results+1))
else
this_test="OK"
n_correct=$((n_correct+1))
fi
else
this_test="MISSING REFERENCE"
n_wrong_results=$((n_wrong_results+1))
fi
fi
fi
# Keep the output up-to-date
timing=0
if [[ ${this_test} == "OK" ]]; then
timing=$(grep -a "CP2K " ${output_file} | tail -n 1 | ${awk} '{printf("%7.2f",$NF)}')
this_test="${this_test} (${timing} sec)"
fi
leak_marker=""
if [[ ${leakcheck} == "YES" ]]; then
if [[ -n $(echo ${cp2k_prefix} | grep "valgrind") ]]; then
leak_error_string="ERROR SUMMARY:"
nerror=$(grep -i "${leak_error_string}" ${output_file} | awk '{print $4}')
dum=""
(( nerror > 0 )) && dum=${nerror}
else
case ${FORT_C_NAME} in
gfortran*) leak_error_string="SUMMARY: LeakSanitizer:";;
esac
dum=$(grep -l "${leak_error_string}" ${output_file})
fi
if [[ ${dum} != "" ]]; then
leak_marker="!"
echo "XXXXXXXX ${output_file} XXXXXXX" >>${memory_description_file}
grep -i "${leak_error_string}" ${output_file} >>${memory_description_file}
fi
fi
# grep numeric test value from output again
test_value="-"
if [[ ${test_type} != 0 ]]; then
grep_string="${test_grep[test_type]}"
grep_field="${test_col[test_type]}"
if [ -f ${output_file} ]; then
test_value=$(grep -a "${grep_string}" ${output_file} | tail -1 | ${awk} -v f=${grep_field} '{print $f}')
fi
fi
printf " %-50s %25s %20s %s\n" "${input_file}" "${test_value}" "${this_test}" "${leak_marker}" >> ${dir_test}/status/REGTEST_TASK_TESTS-$task
printf "%7.2f %s\n" "${timing}" "${dir}/${input_file}" >> "${timings}"
done
t2=$(date +%s)
timing_all=$((1+t2-t1))
printf "%s %0.2f %s\n" "<<< ${dir_test}/tests/${dir} (${idir} of ${ndir}) done in" ${timing_all} "sec" >> ${dir_test}/status/REGTEST_TASK_TESTS-$task
echo "${n_runtime_error} ${n_wrong_results} ${n_correct} ${n_tests}" > ${dir_test}/status/REGTEST_TASK_RESULT-$task
cat ${dir_test}/status/REGTEST_TASK_TESTS-$task
rm -f ${dir_test}/status/REGTEST_TASK_TESTS-$task ${dir_test}/status/REGTEST_RUNNING-$task
}
###################################################################################
#
# function to run a unit test
#
###################################################################################
function run_unittest() {
dir=$1
task=${dir//\//-}
touch ${dir_test}/status/REGTEST_RUNNING-$task
n_runtime_error=0
n_correct=0
mkdir -p ${dir_test}/UNIT
mkdir -p ${dir_test}/tests/${dir}
cd ${dir_test}/tests/${dir}
echo "Starting unit test in ${dir_test}/test/${dir} (${idir} of ${ndir})"
echo ">>> ${dir_test}/${dir}" > ${dir_test}/status/REGTEST_TASK_TESTS-$task
t1=$(date +%s)
output_file=${dir_test}/${dir}.out
unittest=${dir##*/}
if [[ "${cp2k_version}" == "sopt" ]]; then
ext="ssmp"
elif [[ "${cp2k_version}" == "popt" ]]; then
ext="psmp"
else
ext="${cp2k_version}"
fi
testcmd="${cp2k_run_prefix} ${dir_base}/${cp2k_dir}/exe/${dir_triplet}/${unittest}.${ext} ${dir_base}/${cp2k_dir}"
( ulimit -t ${job_max_time} -v ${mem_limit} ; ${testcmd} &> ${output_file} ) >& /dev/null
(( cp2k_exit_status = $? ))
t2=$(date +%s)
timing=$((1+t2-t1))
# check if this was a valgrinded run, and adjust exit status as needed
grep -a "^==[0-9][0-9]*== ERROR SUMMARY" ${output_file} &> /dev/null
if (( $? == 0 )); then
grep -a "^==[0-9][0-9]*== ERROR SUMMARY: 0 errors from 0 contexts" ${output_file} &> /dev/null
# nonzero error count
if (( $? )); then
cp2k_exit_status=42
fi
fi
if (( cp2k_exit_status == 0)); then
this_test=$(printf "OK (%7.2f sec)" ${timing})
n_correct=$((n_correct+1))
else
# CP2K failed obviously
if (( cp2k_exit_status == 137 )); then
# SIGKILL = 9 ... exit code 9+128
# usually caused by time-out
this_test="KILLED"
else
this_test="RUNTIME FAIL"
fi
#
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >>${error_description_file}
echo ${output_file} >>${error_description_file}
tail -40 ${output_file} >>${error_description_file}
# append an eventual valgrind report to the file
grep "^==[0-9][0-9]*== " ${output_file} | head -n 20 >> ${error_description_file}
echo "EXIT CODE: " $cp2k_exit_status " MEANING: " $this_test >>${error_description_file}
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >>${error_description_file}
n_runtime_error=$((n_runtime_error+1))
failed_tests="${failed_tests} ${output_file}"
fi
leak_marker=""
if [[ ${leakcheck} == "YES" ]]; then
if [[ -n $(echo ${cp2k_prefix} | grep "valgrind") ]]; then
leak_error_string="ERROR SUMMARY:"
nerror=$(grep -i "${leak_error_string}" ${output_file} | awk '{print $4}')
dum=""
(( nerror > 0 )) && dum=${nerror}
else
case ${FORT_C_NAME} in
gfortran*) leak_error_string="SUMMARY: LeakSanitizer:";;
esac
dum=$(grep -l "${leak_error_string}" ${output_file})
fi
if [[ ${dum} != "" ]]; then
leak_marker="!"
echo "XXXXXXXX ${output_file} XXXXXXX" >>${memory_description_file}
grep -i "${leak_error_string}" ${output_file} >>${memory_description_file}
fi
fi
printf " %-50s %25s %20s %s\n" "${dir##*/}" "-" "${this_test}" "${leak_marker}" >> ${dir_test}/status/REGTEST_TASK_TESTS-$task
printf "%s %0.2f %s\n" "<<< ${dir_test}/tests/${dir} (${idir} of ${ndir}) done in" ${timing} "sec" >> ${dir_test}/status/REGTEST_TASK_TESTS-$task
echo "${n_runtime_error} 0 ${n_correct} 1" > ${dir_test}/status/REGTEST_TASK_RESULT-$task
cat ${dir_test}/status/REGTEST_TASK_TESTS-$task
rm -f ${dir_test}/status/REGTEST_TASK_TESTS-$task ${dir_test}/status/REGTEST_RUNNING-$task
printf "%7.2f %s\n" "${timing}" "UNIT/${dir##*/}" >> "${timings}"
}
###################################################################################
###################################################################################
# Check for lockfile
lockfile=${dir_last}/LOCKFILE
# install a trap handler for ctrl-C and other ways to die, doing IO is no good in case of SIGPIPE
# kills all processes that have a PID of ppid (good for running jobs), removes the lockfile, and exits with code 8
pgid=$(ps -o "%r" $$ | grep "[0-9]")
trapcommand="rm -f ${lockfile} ; kill -- -$pgid >& out.kill ; exit 8"
trap "$trapcommand" SIGINT SIGTERM SIGPIPE SIGHUP SIGQUIT
if [[ -f ${lockfile} ]]; then
# the lockfile is touched before the start of any new regtest. If it hasn't been touched
# recently (i.e. less than the time a single test is allowed to run), the lockfile is considered stale.
job_max_time_stale=$((job_max_time+10))
if [[ $(date +%s -r ${lockfile}) -gt $(date +%s --date="${job_max_time_stale} sec ago") ]]; then
echo "ERROR: Directory ${dir_last} is locked"
echo " Check if a regtest is already running for the same directory and if not then"
echo " remove the lockfile ${lockfile}"
echo " first and retry"
rm -rf ${dir_test}
echo -e "\nSummary: Test directory is locked."
echo -e "Status: UNKNOWN\n"
exit 7
else
echo "WARNING: stale lockfile in directory ${dir_last} "
echo " continuing regtest"
touch ${lockfile}
fi
else
touch ${lockfile}
fi
if (( maxtasks < numprocs*OMP_NUM_THREADS )); then
echo "ERROR: maxtasks should be greater or equal numprocs*OMP_NUM_THREADS"
echo " maxtasks = ${maxtasks}"
echo " numprocs = ${numprocs}"
echo " OMP_NUM_THREADS = ${OMP_NUM_THREADS}"
rm -rf ${dir_test} ${lockfile}
exit 8
fi
if [[ ${leakcheck} == "YES" ]]; then
if [[ ! -f ${dir_base}/suppr.txt ]]; then
echo "NOTE: Suppression file ${dir_base}/suppr.txt is missing"
touch ${dir_base}/suppr.txt
echo "NOTE: An empty suppression file has been created"
fi
fi
function file_info() {
echo " regtesting location summary file: $summary"
echo " regtesting location error_summary file: $error_description_file"
echo " regtesting location memory_summary file: $memory_description_file"
echo " regtesting location output dir: $dir_test"
echo " regtesting location last dir: $dir_last"
}
# Start testing
echo "*************************** testing started ******************************"
echo " started on " $(date)
echo " configuration: ${dir_triplet}-${cp2k_version} "
file_info
#
# should include all flags, ideally
#
echo "---------------------------- Settings ------------------------------------"
echo "maxtasks = ${maxtasks}"
echo "numprocs = ${numprocs}"
echo "OMP_NUM_THREADS = ${OMP_NUM_THREADS:-(default)}"
echo "OMP_STACKSIZE = ${OMP_STACKSIZE:-(default)}"
echo "KMP_STACKSIZE = ${KMP_STACKSIZE:-(not set)}"
echo "cp2k_run_prefix = ${cp2k_run_prefix}"
echo "cp2k_run_postfix = ${cp2k_run_postfix}"
echo "cp2k_prefix = ${cp2k_prefix}"
echo "cp2k_postfix = ${cp2k_postfix}"
echo "cp2k_version = ${cp2k_version}"
echo "dir_triplet = ${dir_triplet}"
echo "job_max_time = ${job_max_time}"
echo "leakcheck = ${leakcheck}"
echo "doretest = ${doretest}"
echo "nobuild = ${nobuild}"
echo "quick = ${quick}"
echo "shard = ${ishard} / ${nshards}"
echo "skiptest = ${skiptest}"
echo "do_unit_test = ${do_unit_test}"
echo "farming = ${farming}"
echo "maxbuildtasks = ${maxbuildtasks}"
echo "mem_limit = ${mem_limit}"
t=1
while [ $t -le ${ndirstoskip} ]; do
echo "skip_dirs = ${skip_dirs[t]}"
let t=t+1
done
t=1
while [ $t -le ${ndirstorestrict} ]; do
echo "restrict_dirs = ${restrict_dirs[t]}"
let t=t+1
done
echo "--------------------------- GIT ------------------------------------------"
git_sha="<N/A>"
submodule_sha=()
if [[ -d "${dir_base}/${cp2k_dir}/.git" ]] ; then
head_ref=$(<"${dir_base}/${cp2k_dir}/.git/HEAD")
if [[ ${head_ref} =~ ^ref:\ (.*) ]] ; then
ref_file="${dir_base}/${cp2k_dir}/.git/${BASH_REMATCH[1]}"
if [[ -f "${ref_file}" ]] ; then
git_sha=$(<"${ref_file}")
fi
else
# this is a detached HEAD, no further deref needed
git_sha="${head_ref}"
fi
modules_git_dir="${dir_base}/${cp2k_dir}/.git/modules/"
# find submodule indexes, if any
while IFS= read -r -d $'\0'; do
submodule_basedir="${REPLY%/*}"
submodule_hash=$(<"${submodule_basedir}/HEAD")
submodule_dir="${submodule_basedir#${modules_git_dir}}"
submodule_sha+=("${submodule_dir}:${submodule_hash}")
done < <(find "${modules_git_dir}" -name index -print0)
fi
echo "CommitSHA: ${git_sha}"
for submodule in "${submodule_sha[@]}" ; do
echo "Commmit SHA of submodule in ${submodule%%:*}: ${submodule##*:}"
done
# add some more information about that last commit if `git` is available
if [[ "${git_sha}" != "<N/A>" ]] && command -v git >/dev/null 2>&1 ; then
GIT_WORK_TREE="${dir_base}/${cp2k_dir}" GIT_DIR="${dir_base}/${cp2k_dir}/.git" \
git --no-pager \
log -n 1 \
--format="CommitTime: %ci%nCommitAuthor: %an%nCommitAuthorEmail: %ae%nCommitSubject: %s%n" \
"${git_sha}"
fi
echo "--------------------------- Resource limits ------------------------------"
prlimit
echo "--------------------------- SELinux --------------------------------------"
if [[ -f /usr/sbin/getenforce ]]; then
echo "SELinux is installed and is $(/usr/sbin/getenforce)"
else
echo "No SELinux installation found"
fi
echo "--------------------------- Preparations ---------------------------------"
# make realclean
if [[ ${quick} != "quick" ]]; then
cd ${dir_base}/${cp2k_dir}
if [[ "${cp2k_version}" == "sopt" ]]; then
ext="sopt ssmp"
elif [[ "${cp2k_version}" == "popt" ]]; then
ext="popt psmp"
else
ext="${cp2k_version}"
fi
${make} realclean ${ARCH_SPEC} VERSION="${ext}" >>${make_out} 2>&1
if (( $? )); then
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >>${error_description_file}
cat ${make_out} >>${error_description_file}
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >>${error_description_file}
echo "ERROR: ${make} realclean ${ARCH_SPEC} VERSION=\"${ext}\" failed ... bailing out" >>${error_description_file}
cat "${error_description_file}"
rm ${lockfile}
exit 3
else
echo "${make} realclean ${ARCH_SPEC} VERSION=\"${ext}\" went fine"
fi
else
echo "Quick testing, no realclean"
fi
# from here failures are likely to be bugs in cp2k
if [[ ${nobuild} != "nobuild" ]]; then
echo "--------------------------- ARCH-file ------------------------------------"
cat "${arch_dir:-${dir_base}/${cp2k_dir}/arch}/${ARCH}.${cp2k_version}"
echo "-------------------------- Build-Tools -----------------------------------"
cd ${dir_base}/${cp2k_dir}
${make} toolversions ${ARCH_SPEC} VERSION=${cp2k_version}
echo "----------------------- External Modules ---------------------------------"
cd ${dir_base}/${cp2k_dir}
${make} extversions ${ARCH_SPEC} VERSION=${cp2k_version}
echo "---------------------------- Modules -------------------------------------"
if [ "$(type -t module)" = 'function' ]; then
module list 2>&1
else
echo "Module system not installed."
fi
echo "------------------------ compiling cp2k ----------------------------------"
echo "${make} -j $((maxbuildtasks)) ${ARCH_SPEC} VERSION=${cp2k_version}"
echo "(make output is written to ${make_out})"
${make} -j ${maxbuildtasks} ${ARCH_SPEC} VERSION=${cp2k_version} >>${make_out} 2>&1
if (( $? )); then
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >>${error_description_file}
cat ${make_out} >>${error_description_file}
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >>${error_description_file}
echo "ERROR: ${make} -j $((maxbuildtasks)) ${ARCH_SPEC} VERSION=${cp2k_version} failed" >>${error_description_file}
cat "${error_description_file}"
echo -e "\nSummary: Compilation failed."
echo -e "Status: FAILED\n"
rm ${lockfile}
exit 4
else
compile_warnings=$(grep -i "Warning" ${make_out} | wc -l)
echo "${make} -j $((maxbuildtasks)) ${ARCH_SPEC} VERSION=${cp2k_version} went fine (${compile_warnings} warnings)"
fi
else
echo "No build, continue regression testing"
fi
if [[ ${skiptest} != "skiptest" ]]; then
echo "------------------------ regtesting cp2k ---------------------------------"
###################################################################################
#
# parse the TEST_TYPES file to do different kinds of test
#
# tests grep for the last line in the file where a string matches (test_grep)
# and compares a numeric field at a given column (test_col)
#
# the format of the TEST_TYPES file is (notice the '!' as a field separator, to allow
# for spaces in the test_grep)
#
# Ntest_types
# test_grep_1 ! test_col_1
# test_grep_2 ! test_col_2
# ....
# followed by comment lines
#
###################################################################################
Ntest_types=$(awk -v l=1 -v c=1 'BEGIN{FS="!"}{lr=lr+1;if (lr==l) print $c}' ${test_types_file})
test_grep[0]=""
test_col[0]=1
t=1
while [ $t -le ${Ntest_types} ]; do
test_grep[t]=$(${awk} -v l=$t -v c=1 'BEGIN{FS="!"}{lr=lr+1;if (lr==l+1) print $c}' ${test_types_file})
test_col[t]=$(${awk} -v l=$t -v c=2 'BEGIN{FS="!"}{lr=lr+1;if (lr==l+1) print $c}' ${test_types_file})
let t=t+1
done
###################################################################################
#
# *** now start testing
# *** for a given directory we do a run on all files in TEST_FILES and
# *** do the test as indicated by the number
# *** files are run in order so that they can e.g. restart
#
###################################################################################
n_runtime_error=0
n_wrong_results=0
n_correct=0
n_tests=0
echo "------------------------- dynamic libraries linked -----------------------"
echo "ldd ${dir_base}/${cp2k_dir}/exe/${dir_triplet}/cp2k.${cp2k_version} : "
ldd ${dir_base}/${cp2k_dir}/exe/${dir_triplet}/cp2k.${cp2k_version}
echo "--------------------------------------------------------------------------"
printf "Copying tests into working directory ... "
# Copy the tests into the working regtest directory.
if df --output=target >/dev/null ; then
# GNU coreutils version for Linux etc.
source_device=$(df --output=target ${dir_base}/${cp2k_dir}/tests | tail -1)
target_device=$(df --output=target ${dir_test} | tail -1)
if [[ -f /.dockerenv ]]; then
printf "Inside Docker, using normal copy ... "
cp -a ${dir_base}/${cp2k_dir}/tests ${dir_test}
elif [[ ${source_device} == "/afs" || ${target_device} == "/afs" ]]; then
printf "cannot use hard links with AFS.\n"
printf "Using normal copy instead ... "
cp -a ${dir_base}/${cp2k_dir}/tests ${dir_test}
elif [[ ${source_device} == ${target_device} ]]; then
cp -al ${dir_base}/${cp2k_dir}/tests ${dir_test}/tests || \
cp -rpl ${dir_base}/${cp2k_dir}/tests ${dir_test}/tests
else
echo "cannot use hard links to copy the tests folder."
echo "Source device is ${source_device} for ${dir_base}/${cp2k_dir}/tests"
echo "Target device is ${target_device} for ${dir_test}"
printf "Using normal copy instead ... "
cp -a ${dir_base}/${cp2k_dir}/tests ${dir_test}
fi
else
# Probably running on a mac, just use normal cp
cp -a ${dir_base}/${cp2k_dir}/tests ${dir_test}
fi
printf "done!\n"
#
# preprocess the restrict directories if the retest is enabled
#
last_test_dir=""
retest_dirs=""
if [[ $doretest == "yes" ]]; then
last_test_dir=$(ls ${dir_out} | grep ^TEST-${dir_triplet}-${cp2k_version}- | tail -2 | head -1)
if [[ "${testdir}" == "${last_test_dir}" ]]; then
echo "No TEST directory with latest test results found. Nothing to test!"
rm ${lockfile}
rm -rf ${dir_test}
exit 5
elif [[ ! -f "${dir_out}/${last_test_dir}/error_summary" ]]; then
echo "No error summary exists in the last TEST directory. Nothing to test!"
rm ${lockfile}
rm -rf ${dir_test}
exit 6
else
retest_dirs=$(grep .inp.out ${dir_out}/${last_test_dir}/error_summary | awk -F/ '{ test = 0; for (i = 1; i < NF; i++) { if (test == 1) { printf "%s/", $i } if ($i ~ /TEST/) test = 1; } printf "\n" }' | sort | sed 's/\/$//' | awk '!x[$0]++')
for t in ${retest_dirs}; do
let ndirstorestrict=ndirstorestrict+1;
restrict_dirs[ndirstorestrict]=$t;
done
if (( ndirstorestrict == 0 )); then
echo "No error occurred during the last run. Nothing to retest!"
rm ${lockfile}
rm -rf ${dir_test}
exit 0
fi
fi
fi
#
# get the cp2k flags for supported features
#
cp2kflags=$(${cp2k_prefix} --version ${cp2k_postfix} < /dev/null 2>&1 | grep cp2kflags )
echo "CP2K supports: $cp2kflags"
# get a list of regtest directories
dirs=$(cat ${dir_test}/tests/TEST_DIRS | grep -v "#" | awk '{print $1}')
# prepend list of unit-tests to dirs, use a special "UNIT/" prefix
if [[ "$do_unit_test" == "yes" ]]; then
if [[ "${cp2k_version}" == "sopt" ]]; then
ext="ssmp"
elif [[ "${cp2k_version}" == "popt" ]]; then
ext="psmp"
else
ext="${cp2k_version}"
fi
for unittest in ${dir_base}/${cp2k_dir}/exe/${dir_triplet}/*_unittest.${ext}
do
t=${unittest##*/}
dirs="UNIT/${t%.*} $dirs "
done
fi
# sharding
new_dirs=""
t=1
for dir in ${dirs} ; do
if ((t % nshards + 1 == ishard )) ; then
new_dirs="$new_dirs $dir"
fi
let t=t+1
done
dirs=$new_dirs
# filter out test-dir that should be skipped
new_dirs=""
for dir in ${dirs}
do
dir=${dir%/}
match="no"
t=1
# Match to exclusion list
while [ $t -le ${ndirstoskip} ]; do
if [[ "${skip_dirs[t]}" == "${dir}" ]]; then
match="yes"
fi
let t=t+1
done
# Match to the restrict list, if no restrict list is found, all dirs match
if [ ${ndirstorestrict} -gt 0 ]; then
restrictmatch="no"
t=1
while [ $t -le ${ndirstorestrict} ]; do
if [[ "${restrict_dirs[t]%/}" == "${dir}" ]]; then
restrictmatch="yes"
fi
let t=t+1
done
else
restrictmatch="yes"
fi
# does this dir require features we do not have ?
featurematch="yes"
features_required=$(cat ${dir_test}/tests/TEST_DIRS | grep -v "#" | awk -v dir="$dir" '{if ($1==dir) for(i=2;i<=NF;i++) printf("%s ",$i)}')
for fr in $features_required
do
# allow a specification on the mpiranks needed e.g. fr="mpiranks>2&&mpiranks<6"
echo $fr | grep mpiranks >& /dev/null
if [[ "$?" == "1" ]]; then
echo $cp2kflags | grep $fr >& /dev/null
res=$?
else
res=$(echo "" | awk -v mpiranks=$numprocs "{print !($fr)}")
fi
if [[ "$res" == "1" ]]; then
featurematch="no"
echo "Skipping $dir : missing required feature : $fr"
fi
done
# If not excluded add to list of dirs
if [[ "${match}" == "no" && "${restrictmatch}" == "yes" && "${featurematch}" == "yes" ]]; then
new_dirs="$new_dirs $dir"
fi
done
dirs=$new_dirs
# Just to be sure, clean possible existing status files.
cd ${dir_test}
mkdir ${dir_test}/status
rm -f ${dir_test}/status/REGTEST_RUNNING-* \
${dir_test}/status/REGTEST_TASK_RESULT-* \
${dir_test}/status/REGTEST_TASK_TESTS-*
#
# generate farming input
#
if [[ "$farming" == "yes" ]]; then
echo "generating farming input "
ijob=0
cat << EOF > ${dir_test}/farming.inp
&GLOBAL
PROJECT farming
PROGRAM FARMING
RUN_TYPE NONE
&END GLOBAL
&FARMING
GROUP_SIZE $numprocs
MASTER_SLAVE
EOF
idir=0
for dir in ${dirs}; do
if [[ $dir != UNIT/* ]]; then
# generate output dir
mkdir -p ${dir_test}/${dir}
#
# generate the equivalent farming input
#
idir=$((idir+1))
ntest=$(grep '^\s*\w' ${dir_test}/tests/${dir}/TEST_FILES | ${awk} '{c=c+1}END{print c}')
for ((itest=1;itest<=ntest;itest++));
do
input_file=$(grep '^\s*\w' ${dir_test}/tests/${dir}/TEST_FILES | ${awk} -v itest=$itest '{c=c+1;if (c==itest) print $1}')
output_file=${dir_test}/${dir}/${input_file}.out
jobid=$((idir*100000+itest))
if [[ $itest != 1 ]]; then
jobid_prev=$((idir*100000+itest-1))
depstr="DEPENDENCIES $jobid_prev"
else
depstr=""
fi
ijob=$((ijob+1))
printf "&JOB \n DIRECTORY ${dir_test}/tests/${dir} \n INPUT_FILE_NAME ${input_file} \n OUTPUT_FILE_NAME ${output_file} \n JOB_ID $jobid \n $depstr \n&END JOB\n" >> ${dir_test}/farming.inp
done
fi
done
printf "&END FARMING\n" >> ${dir_test}/farming.inp
echo "Running farming for $ijob jobs"
${cp2k_prefix} ${dir_base}/${cp2k_dir}/exe/${dir_triplet}/cp2k.${cp2k_version} farming.inp ${cp2k_postfix} < /dev/null >& farming.out
if (( $? )); then # on failed farming, provide info
echo "==================== Farming terminated with an error, dumping output ========================"
cat farming.out
echo "==============================================================================================="
else
echo "farming went fine"
fi
fi
#
# execute all regtests
#
(( ndir = $(echo ${dirs} | wc -w) ))
(( idir = 0 ))
for dir in ${dirs}; do
(( idir++ ))
# tests in different dirs can run in parallel. We spawn processes up to a given maximum
(
if [[ $dir == UNIT/* ]]; then
run_unittest $dir
else
run_regtest_dir $dir
fi
)&
#
# Here we allow only a given maximum of tasks
#
runningtasks=10000
while (( runningtasks >= maxtasks/(numprocs*OMP_NUM_THREADS) )); do
sleep 0.1
runningtasks=$(ls -1 ${dir_test}/status/REGTEST_RUNNING-* 2>/dev/null | wc -l)
done
done
#
# wait for all tasks to finish
#
wait
#
# generate results
#
for dir in ${dirs};
do
task=${dir//\//-}
file=${dir_test}/status/REGTEST_TASK_RESULT-$task
tmp=$(awk '{print $1}' $file)
n_runtime_error=$((n_runtime_error+tmp))
tmp=$(awk '{print $2}' $file)
n_wrong_results=$((n_wrong_results+tmp))
tmp=$(awk '{print $3}' $file)
n_correct=$((n_correct+tmp))
tmp=$(awk '{print $4}' $file)
n_tests=$((n_tests+tmp))
rm -f $file
done
echo "--------------------------------------------------------------------------"
cat "${error_description_file}"
echo "--------------------------------------------------------------------------"
file_info
echo ""
echo "--------------------------------- Timings --------------------------------"
${dir_base}/${cp2k_dir}/tools/regtesting/timings.py "${timings}"
echo ""
echo "--------------------------------- Summary --------------------------------"
if [[ ${quick} != "quick" ]]; then
printf "Number of COMPILE warns %d\n" ${compile_warnings} | tee -a ${summary}
fi
printf "Number of FAILED tests %d\n" ${n_runtime_error} | tee -a ${summary}
printf "Number of WRONG tests %d\n" ${n_wrong_results} | tee -a ${summary}
printf "Number of CORRECT tests %d\n" ${n_correct} | tee -a ${summary}
printf "Total number of tests %d\n" ${n_tests} | tee -a ${summary}
if [[ ${leakcheck} == "YES" ]]; then
echo "--------------------------------------------------------------------------" | tee -a ${summary}
case ${FORT_C_NAME} in
gfortran*)
n_leaking_tests=$(grep "XXXXXXXX" ${memory_description_file} | wc | ${awk} '{print $1}')
n_leaks=$(grep "SUMMARY" ${memory_description_file} | ${awk} 'BEGIN{n=0}{n=n+$7}END{print n}')
printf "Number of LEAKING tests %d\n" ${n_leaking_tests} | tee -a ${summary}
;;
esac
(( n_leaks == 0 )) && echo "No memory leaks detected" >>${memory_description_file}
printf "Number of memory leaks %d\n" ${n_leaks} | tee -a ${summary}
if [[ -s ${dir_base}/suppr.txt ]]; then
echo "--------------------------------------------------------------------------" | tee -a ${summary}
printf "Note, that the following leak suppressions are currently active:\n" | tee -a ${summary}
cat ${dir_base}/suppr.txt | tee -a ${summary}
fi
echo "--------------------------------------------------------------------------" | tee -a ${summary}
echo "GREPME ${n_runtime_error} ${n_wrong_results} ${n_correct} 0 ${n_tests} ${n_leaks}"
else
echo "No memory leak check was performed" >>${memory_description_file}
echo "GREPME ${n_runtime_error} ${n_wrong_results} ${n_correct} 0 ${n_tests} X"
fi
full_time_t2=$(date +%s)
full_timing_all=$((1+full_time_t2-full_time_t1))
REPORT_SUMMARY=$(printf "correct: %d / %d" ${n_correct} ${n_tests})
if (( n_wrong_results > 0 )); then
REPORT_SUMMARY+=$(printf "; wrong: %d" ${n_wrong_results})
fi
if (( n_runtime_error > 0 )); then
REPORT_SUMMARY+=$(printf "; failed: %d" ${n_runtime_error})
fi
if (( n_leaks > 0 )); then
REPORT_SUMMARY+=$(printf "; memleaks: %d" ${n_leaks})
fi
REPORT_SUMMARY+=$(python3 -c "print('; %.0fmin'%(${full_timing_all}/60.0))")
echo -e "\nSummary: ${REPORT_SUMMARY}"
if ((n_wrong_results > 0)) || ((n_runtime_error > 0)) || ((n_leaks > 0)); then
echo -e "Status: FAILED\n"
else
echo -e "Status: OK\n"
fi
echo "--------------------------------------------------------------------------"
printf "Regtest took %0.2f seconds.\n" ${full_timing_all}
echo "--------------------------------------------------------------------------"
date
echo "*************************** testing ended ********************************"
fi
rm ${lockfile}
exit 0
Reference in a new issue View git blame Copy permalink