From 73b1c92bcdb78aaa1403e15afb6da3bb6902908b Mon Sep 17 00:00:00 2001 From: edoapra Date: Tue, 5 Jan 2021 16:48:52 -0800 Subject: [PATCH] added docker github actions executed only with [do_extra_archs] label --- .github/workflows/github_actions_archs.yml | 125 +++++++++++++++++++++ travis/build_env.sh | 17 ++- travis/compile_nwchem.sh | 10 +- travis/nwchem.bashrc | 35 +++++- travis/run_qas.sh | 26 ++++- 5 files changed, 196 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/github_actions_archs.yml diff --git a/.github/workflows/github_actions_archs.yml b/.github/workflows/github_actions_archs.yml new file mode 100644 index 0000000000..593f61d4b3 --- /dev/null +++ b/.github/workflows/github_actions_archs.yml @@ -0,0 +1,125 @@ +name: NWChem_CI_archs + +on: [push, pull_request] + +jobs: + build_archs: + if: "!contains(github.event.head_commit.message, 'ci skip') && contains(github.event.head_commit.message, 'do_extra_archs')" + runs-on: ubuntu-18.04 + strategy: + matrix: + include: + - arch: aarch64 + distro_short: ubuntu + distro: ubuntu18.04 + ftarget: " TARGET=ARMV8" + blas: "build_openblas" + scalapack: "off" + nwchem_modules: "tinyqmpw" + - arch: ppc64le + distro_short: ubuntu + distro: ubuntu18.04 + blas: "build_openblas" + scalapack: "on" + ftarget: " TARGET=POWER8" + nwchem_modules: "tinyqmpw" + + fail-fast: false + steps: + - name: Setup cache + id: setup-cache + uses: actions/cache@v2 + with: + path: | + ~/cache + key: ${{ runner.os }}-${{ matrix.mpi_impl}}-${{ matrix.distro}}-${{ matrix.arch}}-nwchem-v003 + - uses: actions/checkout@v2 + name: Checkout + with: + fetch-depth: 40 + - name: tools_checkout + run: | + ls -l + cd src/tools + ./get-tools-github + cd .. + mkdir -p ../bin/LINUX64 + gcc -o ../bin/LINUX64/depend.x config/depend.c + make USE_INTERNALBLAS=y nwchem_config NWCHEM_MODULES="${{ matrix.nwchem_modules }}" + make USE_INTERNALBLAS=y deps_stamp + rm -f ../bin/LINUX64/depend.x + - name: reconfig when needed + if: steps.setup-cache.outputs.cache-hit != 'true' + run: | + cd src + make USE_INTERNALBLAS=y nwchem_config NWCHEM_MODULES="nwdft driver solvation" + - uses: uraimo/run-on-arch-action@v2.0.8 + name: Build and Test NWChem + id: build + with: + arch: ${{ matrix.arch }} + distro: ${{ matrix.distro }} + # Not required, but speeds up builds + githubToken: ${{ github.token }} + # Create cached/volume directories on host + setup: | + mkdir -p ~/cache + # Mount the ~/cache directory as ..nwchem/cache in the container + dockerRunArgs: | + --volume "${HOME}/cache:/home/runner/work/nwchem/nwchem/cache" + #environment + env: | + FORCETARGET: ${{ matrix.ftarget }} + COMEX_MAX_NB_OUTSTANDING: 4 + SIMINT_MAXAM: 3 + MPI_IMPL: openmpi + ARMCI_NETWORK: MPI-TS + FC: gfortran + DISTR: ${{ matrix.distro_short }} + BLAS_ENV: ${{ matrix.blas }} + SCALAPACK_ENV: ${{ matrix.scalapack }} + shell: /bin/sh + install: | + case "${{ matrix.distro }}" in + ubuntu*|jessie|stretch|buster) + apt-get update -q -y + apt-get install -q -y sudo git gfortran libopenmpi-dev curl wget unzip perl rsync python3-dev python-dev cmake + ;; + fedora*) + dnf -y update + dnf -y install git which sudo curl wget unzip openblas-serial64 openmpi-devel perl python3-devel gcc-gfortran unzip cmake patch time + ;; + esac + run: | + pwd + df + ls -lrt + uname -a + lscpu + ./travis/build_env.sh + echo ' done build_env ' + mkdir -p cache/libext/lib || true + mkdir -p src/libext/lib || true + ls -Rl src/libext || true + ls -lRrt cache + df cache + rsync -av cache/libext/lib/* src/libext/lib/. || true + echo "cache fetched" + ls -Rl cache/libext/lib/ ||true + ./travis/compile_nwchem.sh + echo ' done compiling' + rsync -av src/libext/lib/* cache/libext/lib/. || true + echo "cache stored" + ls -l cache/libext/lib/ + ./travis/run_qas.sh + + - name: Cache check + run: | + pwd + echo HOME is $HOME || true + ls -l ~/ || true + ls -Rl ~/cache || true + ls -l ~/ || true + ls -l ~/work/ || true + ls -l ~/work/nwchem || true + diff --git a/travis/build_env.sh b/travis/build_env.sh index c29db046a9..1dee2b3553 100755 --- a/travis/build_env.sh +++ b/travis/build_env.sh @@ -1,6 +1,12 @@ #!/bin/bash os=`uname` arch=`uname -m` +dist="ubuntu" +if test -f " /usr/lib/fedora-release"; then + dist="fedora" +fi +echo dist is "$dist" +echo DISTR is "$DISTR" if [[ "$os" == "Darwin" ]]; then # HOMEBREW_NO_AUTO_UPDATE=1 brew cask uninstall oclint || true # HOMEBREW_NO_INSTALL_CLEANUP=1 HOMEBREW_NO_AUTO_UPDATE=1 brew install gcc "$MPI_IMPL" openblas python3 ||true @@ -13,7 +19,15 @@ arch=`uname -m` # HOMEBREW_NO_INSTALL_CLEANUP=1 HOMEBREW_NO_AUTO_UPDATE=1 brew install scalapack # fi fi -if [[ "$os" == "Linux" ]]; then + if [[ "$os" == "Linux" ]]; then + if [[ "$DISTR" == "fedora" ]];then + sudo dnf udate; sudo dnf -y install perl perl python3-devel time patch openblas-serial64 openmpi-devel cmake gcc-gfortran unzip + # module load mpi + export PATH=/usr/lib64/openmpi/bin:$PATH + export LD_LIBRARY_PATH=/usr/lib64/openmpi/lib:$LD_LIBRARY_PATH + which mpif90 + mpif90 -show + else if [[ "$MPI_IMPL" == "openmpi" ]]; then mpi_bin="openmpi-bin" ; mpi_libdev="libopenmpi-dev" scalapack_libdev="libscalapack-openmpi-dev" fi @@ -24,4 +38,5 @@ if [[ "$os" == "Linux" ]]; then sudo add-apt-repository universe && sudo apt update # sudo apt-get -y install gfortran python3-dev python-dev cmake "$mpi_libdev" "$mpi_bin" "$scalapack_libdev" make perl libopenblas-dev python3 rsync sudo apt-get -y install gfortran python3-dev python-dev cmake "$mpi_libdev" "$mpi_bin" make perl python3 rsync + fi fi diff --git a/travis/compile_nwchem.sh b/travis/compile_nwchem.sh index 8c10e3592a..ba1a8f8643 100755 --- a/travis/compile_nwchem.sh +++ b/travis/compile_nwchem.sh @@ -40,7 +40,7 @@ fi # export LDFLAGS="-L/usr/local/opt/python@3.7/lib:$LDFLAGS" fi if [[ -z "$TRAVIS_HOME" ]]; then - make V=1 FOPTIMIZE="$FOPT" FDEBUG="$FDOPT" -j3 + make V=0 FOPTIMIZE="$FOPT" FDEBUG="$FDOPT" -j3 else ../travis/sleep_loop.sh make V=1 FOPTIMIZE="$FOPT" FDEBUG="$FDOPT" -j3 fi @@ -53,14 +53,10 @@ fi # ls -lrt $DYLD_LIBRARY_PATH # tail -120 make.log elif [[ "$os" == "Linux" ]]; then - if [[ "$arch" == "aarch64" ]]; then - export MAKEFLAGS=-j8 - else - export MAKEFLAGS=-j3 - fi + export MAKEFLAGS=-j3 echo "$FOPT$FDOPT" if [[ -z "$TRAVIS_HOME" ]]; then - make V=1 FOPTIMIZE="$FOPT" FDEBUG="$FDOPT" -j3 + make V=0 FOPTIMIZE="$FOPT" FDEBUG="$FDOPT" -j3 else ../travis/sleep_loop.sh make V=1 FOPTIMIZE="$FOPT" FDEBUG="$FDOPT" -j3 fi diff --git a/travis/nwchem.bashrc b/travis/nwchem.bashrc index 47232c685a..9515f22c5b 100644 --- a/travis/nwchem.bashrc +++ b/travis/nwchem.bashrc @@ -1,6 +1,7 @@ #- env: == default == os=`uname` arch=`uname -m` +echo DISTR is "$DISTR" if [[ -z "$TRAVIS_BUILD_DIR" ]] ; then TRAVIS_BUILD_DIR=$(pwd) else @@ -30,10 +31,36 @@ export USE_NOIO=1 if [[ "$BLAS_SIZE" == "4" ]]; then export USE_64TO32=y fi + +if [[ "$DISTR" == "fedora" ]]; then + export PATH=/usr/lib64/"$MPI_IMPL"/bin:$PATH + export LD_LIBRARY_PATH=/usr/lib64/"$MPI_IMPL"/lib:$LD_LIBRARY_PATH +fi +if [[ "$BLAS_ENV" == "internal" ]]; then + export USE_INTERNALBLAS=1 + export BLAS_SIZE=8 +elif [[ "$BLAS_ENV" == "build_openblas" ]]; then + export BUILD_OPENBLAS="y" + export BLAS_SIZE=8 +fi if [[ -z "$USE_INTERNALBLAS" ]]; then - export BUILD_OPENBLAS="y" - export BUILD_SCALAPACK="y" - export BLAS_SIZE=8 - export SCALAPACK_SIZE=8 + if [[ -z "$BLASOPT" ]] ; then + export BUILD_OPENBLAS="y" + export BLAS_SIZE=8 + else + unset BUILD_OPENBLAS + fi + if [[ "$SCALAPACK_ENV" == "off" ]]; then + unset BUILD_SCALAPACK + unset SCALAPACK + unset SCALAPACK_SIZE + else + if [[ -z "$SCALAPACK" ]] ; then + export BUILD_SCALAPACK="y" + export SCALAPACK_SIZE=8 + else + unset BUILD_SCALAPACK + fi + fi fi export NWCHEM_EXECUTABLE=$TRAVIS_BUILD_DIR/.cachedir/binaries/$NWCHEM_TARGET/nwchem_"$arch"_`echo $NWCHEM_MODULES|sed 's/ /-/g'`_"$MPI_IMPL" diff --git a/travis/run_qas.sh b/travis/run_qas.sh index 97f599c8bf..38606a8d87 100755 --- a/travis/run_qas.sh +++ b/travis/run_qas.sh @@ -23,7 +23,7 @@ export NWCHEM_NWPW_LIBRARY=$TRAVIS_BUILD_DIR/.cachedir/files/libraryps/ nprocs=2 do_largeqas=1 - if [[ "$EXTRA_BUILD" == "1" ]] || [[ ! -z "$USE_SIMINT" ]]; then + if [[ "$EXTRA_BUILD" == "1" ]] || [[ ! -z "$USE_SIMINT" ]] || [[ "$arch" == "aarch64" ]] || [[ "$arch" == "ppc64le" ]]; then do_largeqas=0 fi @@ -64,6 +64,9 @@ export NWCHEM_NWPW_LIBRARY=$TRAVIS_BUILD_DIR/.cachedir/files/libraryps/ esac ;; esac +if [[ "$MPI_IMPL" == "openmpi" ]]; then +export MPIRUN_NPOPT=" --allow-run-as-root -mca mpi_yield_when_idle 0 --oversubscribe -np " +fi echo === ls binaries cache === ls -lrt $TRAVIS_BUILD_DIR/.cachedir/binaries/$NWCHEM_TARGET/ || true echo ========================= @@ -79,13 +82,26 @@ export NWCHEM_NWPW_LIBRARY=$TRAVIS_BUILD_DIR/.cachedir/files/libraryps/ cd $TRAVIS_BUILD_DIR/QA && ./runtests.mpi.unix procs $nprocs tce_ipccsd_f2 tce_eaccsd_ozone fi else - cd $TRAVIS_BUILD_DIR/QA && ./runtests.mpi.unix procs $nprocs dft_he2+ prop_mep_gcube - cd $TRAVIS_BUILD_DIR/QA && ./runtests.mpi.unix procs $nprocs cosmo_h2o_dft +# check if dft is among modules + if [[ ! $(grep -i dft $TRAVIS_BUILD_DIR/src/stubs.F| awk '/dft_input/') ]]; then + cd $TRAVIS_BUILD_DIR/QA && ./runtests.mpi.unix procs $nprocs dft_he2+ + if [[ ! $(grep -i prop $TRAVIS_BUILD_DIR/src/stubs.F| awk '/prop_input/') ]]; then + cd $TRAVIS_BUILD_DIR/QA && ./runtests.mpi.unix procs $nprocs prop_mep_gcube + fi + if [[ ! $(grep -i cosmo $TRAVIS_BUILD_DIR/src/stubs.F| awk '/cosmo_input/') ]]; then + cd $TRAVIS_BUILD_DIR/QA && ./runtests.mpi.unix procs $nprocs cosmo_h2o_dft + fi + else + echo ' dft_input stubbed' + fi if [[ "$USE_SIMINT" != "1" ]] ; then - cd $TRAVIS_BUILD_DIR/QA && ./runtests.mpi.unix procs $nprocs pspw +# check if pspw is among modules + if [[ ! $(grep -i pspw $TRAVIS_BUILD_DIR/src/stubs.F| awk '/pspw_input/') ]]; then + cd $TRAVIS_BUILD_DIR/QA && ./runtests.mpi.unix procs $nprocs pspw + fi fi # check if python is among modules - if [[ $(echo ${NWCHEM_MODULES}| awk ' /python/') ]]; then + if [[ ! $(grep -i python $TRAVIS_BUILD_DIR/src/stubs.F| awk '/python_input/') ]]; then cd $TRAVIS_BUILD_DIR/QA && ./runtests.mpi.unix procs $nprocs pyqa3 fi if [[ "$do_largeqas" == 1 ]]; then