From 165051c36de7b4e8badd7046e504bb395a462f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ole=20Sch=C3=BCtt?= Date: Thu, 25 Apr 2019 15:00:52 +0200 Subject: [PATCH] Docker: Make build more quiet and resilient --- tools/docker/scripts/install_aiida.sh | 13 +++++++++---- tools/docker/scripts/install_ase.sh | 15 ++++++++++----- tools/docker/scripts/install_basics.sh | 7 ++++--- tools/docker/scripts/install_conventions.sh | 8 +++++++- tools/docker/scripts/install_formatting.sh | 11 ++++++++--- tools/docker/scripts/install_i-pi.sh | 15 ++++++++++----- tools/docker/scripts/install_manual.sh | 13 +++++++++---- tools/docker/scripts/install_python.sh | 16 +++++++++------- tools/docker/scripts/install_regtest.sh | 2 +- tools/docker/scripts/install_scaling.sh | 17 ++++++++++++----- .../docker/scripts/install_ubuntu_toolchain.sh | 4 ++-- tools/docker/scripts/test_aiida.sh | 6 +++--- tools/docker/scripts/test_i-pi.sh | 4 ++-- 13 files changed, 86 insertions(+), 45 deletions(-) diff --git a/tools/docker/scripts/install_aiida.sh b/tools/docker/scripts/install_aiida.sh index c154bc031e..ce0b56a2b4 100755 --- a/tools/docker/scripts/install_aiida.sh +++ b/tools/docker/scripts/install_aiida.sh @@ -3,10 +3,10 @@ # author: Ole Schuett # install Ubuntu packages -apt-get update +apt-get update -qq export DEBIAN_FRONTEND=noninteractive export DEBCONF_NONINTERACTIVE_SEEN=true -apt-get install -y --no-install-recommends \ +apt-get install -qq --no-install-recommends \ build-essential \ python-setuptools \ python-wheel \ @@ -19,7 +19,7 @@ apt-get install -y --no-install-recommends \ rm -rf /var/lib/apt/lists/* # install python packages -pip install flake8 aiida ase +pip install --quiet flake8 aiida ase # create ubuntu user with sudo powers adduser --disabled-password --gecos "" ubuntu @@ -34,7 +34,12 @@ ln -vs /opt/cp2k-toolchain/install/arch/local* . # pre-build cp2k cd /workspace/cp2k -make -j VERSION=pdbg +echo -n "Warming cache by trying to compile... " +if make -j VERSION=pdbg &> /dev/null ; then + echo "done." +else + echo "failed." +fi rm -rf lib exe #EOF diff --git a/tools/docker/scripts/install_ase.sh b/tools/docker/scripts/install_ase.sh index 0036fbf785..c3dc52f6a5 100755 --- a/tools/docker/scripts/install_ase.sh +++ b/tools/docker/scripts/install_ase.sh @@ -3,8 +3,8 @@ # author: Ole Schuett # install python packages -apt-get update -apt-get install -y --no-install-recommends \ +apt-get update -qq +apt-get install -qq --no-install-recommends \ python3 \ python3-dev \ python3-pip \ @@ -14,10 +14,10 @@ apt-get install -y --no-install-recommends \ rm -rf /var/lib/apt/lists/* # install python packages -pip3 install numpy scipy matplotlib flask +pip3 install --quiet numpy scipy matplotlib flask # clone ase reprository -git clone --depth=1 --single-branch -b master https://gitlab.com/ase/ase.git /opt/ase +git clone --quiet --depth=1 --single-branch -b master https://gitlab.com/ase/ase.git /opt/ase # shellcheck disable=SC1091 source /opt/cp2k-toolchain/install/setup @@ -28,7 +28,12 @@ ln -vs /opt/cp2k-toolchain/install/arch/local* . # pre-build cp2k cd /workspace/cp2k -make -j VERSION=pdbg +echo -n "Warming cache by trying to compile... " +if make -j VERSION=pdbg &> /dev/null ; then + echo "done." +else + echo "failed." +fi rm -rf lib exe #EOF diff --git a/tools/docker/scripts/install_basics.sh b/tools/docker/scripts/install_basics.sh index 42756d51ae..73dcd373b7 100755 --- a/tools/docker/scripts/install_basics.sh +++ b/tools/docker/scripts/install_basics.sh @@ -3,8 +3,8 @@ # author: Ole Schuett # install Ubuntu packages -apt-get update -apt-get install -y --no-install-recommends \ +apt-get update -qq +apt-get install -qq --no-install-recommends \ autoconf \ autogen \ automake \ @@ -21,9 +21,10 @@ apt-get install -y --no-install-recommends \ rsync \ unzip \ wget + rm -rf /var/lib/apt/lists/* # clone cp2k repository -git clone --recursive --depth=1 --single-branch -b master https://github.com/cp2k/cp2k.git /workspace/cp2k +git clone --quiet --recursive --depth=1 --single-branch -b master https://github.com/cp2k/cp2k.git /workspace/cp2k #EOF diff --git a/tools/docker/scripts/install_conventions.sh b/tools/docker/scripts/install_conventions.sh index a621ca6868..209c3b935d 100755 --- a/tools/docker/scripts/install_conventions.sh +++ b/tools/docker/scripts/install_conventions.sh @@ -11,7 +11,13 @@ ln -vs /opt/cp2k-toolchain/install/arch/local* . # pre-build cp2k cd /workspace/cp2k/tools/conventions -./test_conventions.sh +echo -n "Warming cache by trying run test_conventions.sh... " +if ./test_conventions.sh &> /dev/null ; then + echo "done." +else + echo "failed." +fi + rm -rf ../lib/ ../exe/ #EOF diff --git a/tools/docker/scripts/install_formatting.sh b/tools/docker/scripts/install_formatting.sh index c9b4b39f38..005a13e531 100755 --- a/tools/docker/scripts/install_formatting.sh +++ b/tools/docker/scripts/install_formatting.sh @@ -3,12 +3,17 @@ # author: Ole Schuett # install Ubuntu packages -apt-get update -apt-get install -y --no-install-recommends libfindbin-libs-perl +apt-get update -qq +apt-get install -qq --no-install-recommends libfindbin-libs-perl rm -rf /var/lib/apt/lists/* # pre-run prettify cd /workspace/cp2k -make -j 16 pretty +echo -n "Warming cache by trying to run make pretty... " +if make -j 16 pretty &> /dev/null ; then + echo "done." +else + echo "failed." +fi #EOF diff --git a/tools/docker/scripts/install_i-pi.sh b/tools/docker/scripts/install_i-pi.sh index 1accc09e2a..a35b2fc91c 100755 --- a/tools/docker/scripts/install_i-pi.sh +++ b/tools/docker/scripts/install_i-pi.sh @@ -3,8 +3,8 @@ # author: Ole Schuett # install Ubuntu packages -apt-get update -apt-get install -y --no-install-recommends \ +apt-get update -qq +apt-get install -qq --no-install-recommends \ python \ python-pip \ python-wheel \ @@ -12,10 +12,10 @@ apt-get install -y --no-install-recommends \ rm -rf /var/lib/apt/lists/* # install python packages -pip install numpy +pip install --quiet numpy # clone i-pi repository -git clone --depth=1 --single-branch -b master https://github.com/i-pi/i-pi.git /opt/i-pi +git clone --quiet --depth=1 --single-branch -b master https://github.com/i-pi/i-pi.git /opt/i-pi # shellcheck disable=SC1091 source /opt/cp2k-toolchain/install/setup @@ -26,7 +26,12 @@ ln -vs /opt/cp2k-toolchain/install/arch/local* . # pre-build cp2k cd /workspace/cp2k -make -j VERSION=pdbg +echo -n "Warming cache by trying to compile... " +if make -j VERSION=pdbg &> /dev/null ; then + echo "done." +else + echo "failed." +fi rm -rf lib exe #EOF diff --git a/tools/docker/scripts/install_manual.sh b/tools/docker/scripts/install_manual.sh index c99884041b..bdbde6ec0c 100755 --- a/tools/docker/scripts/install_manual.sh +++ b/tools/docker/scripts/install_manual.sh @@ -3,9 +3,9 @@ # author: Ole Schuett # install Ubuntu packages -apt-get update -apt-get install -y --no-install-recommends \ - default-jre-headless \ +apt-get update -qq +apt-get install -qq --no-install-recommends \ + default-jre-headless \ libsaxonhe-java rm -rf /var/lib/apt/lists/* @@ -18,7 +18,12 @@ ln -vs /opt/cp2k-toolchain/install/arch/local* . # pre-build cp2k cd /workspace/cp2k -make -j VERSION=psmp +echo -n "Warming cache by trying to compile... " +if make -j VERSION=psmp &> /dev/null ; then + echo "done." +else + echo "failed." +fi rm -rf lib exe #EOF diff --git a/tools/docker/scripts/install_python.sh b/tools/docker/scripts/install_python.sh index 7968857d7a..419ea0c851 100755 --- a/tools/docker/scripts/install_python.sh +++ b/tools/docker/scripts/install_python.sh @@ -3,8 +3,8 @@ # author: Ole Schuett # install Ubuntu packages -apt-get update -apt-get install -y --no-install-recommends \ +apt-get update -qq +apt-get install -qq --no-install-recommends \ python \ python3 \ python-pip \ @@ -19,17 +19,19 @@ apt-get install -y --no-install-recommends \ rm -rf /var/lib/apt/lists/* # install python packages -pip install numpy matplotlib requests -pip3 install numpy matplotlib requests +pip install --quiet numpy matplotlib requests +pip3 install --quiet numpy matplotlib requests # install python2.6 +echo "Building Python-2.6.9... " cd /tmp wget -q https://www.python.org/ftp/python/2.6.9/Python-2.6.9.tgz +echo "7277b1285d8a82f374ef6ebaac85b003266f7939b3f2a24a3af52f9523ac94db Python-2.6.9.tgz" | sha256sum --check tar -xzf Python-2.6.9.tgz pushd Python-2.6.9 -./configure -make -j -make install +./configure > /tmp/python2.6.9_configure.log +make -j > /tmp/python2.6.9_make.log +make install > /tmp/python2.6.9_install.log popd rm -rf Python-2.6.9* diff --git a/tools/docker/scripts/install_regtest.sh b/tools/docker/scripts/install_regtest.sh index ff2278f309..24ece85387 100755 --- a/tools/docker/scripts/install_regtest.sh +++ b/tools/docker/scripts/install_regtest.sh @@ -19,7 +19,7 @@ source /opt/cp2k-toolchain/install/setup # pre-build cp2k cd /workspace/cp2k -echo -n "Trying to compile... " +echo -n "Warming cache by trying to compile... " if make -j ARCH="${ARCH}" VERSION="${VERSION}" &> /dev/null ; then echo "done." else diff --git a/tools/docker/scripts/install_scaling.sh b/tools/docker/scripts/install_scaling.sh index ddd4ed6a7e..c276f007e2 100755 --- a/tools/docker/scripts/install_scaling.sh +++ b/tools/docker/scripts/install_scaling.sh @@ -3,8 +3,8 @@ # author: Ole Schuett # install numpy -apt-get update -apt-get install -y --no-install-recommends python-numpy +apt-get update -qq +apt-get install -qq --no-install-recommends python-numpy rm -rf /var/lib/apt/lists/* # setup arch files @@ -16,9 +16,16 @@ source /opt/cp2k-toolchain/install/setup # pre-build cp2k cd /workspace/cp2k -make -j VERSION="popt" cp2k -make -j VERSION="psmp" cp2k -make -j VERSION="ssmp" cp2k +echo -n "Warming cache by trying to compile... " +if make -j VERSION="popt" cp2k &> /dev/null && \ + make -j VERSION="psmp" cp2k &> /dev/null && \ + make -j VERSION="ssmp" cp2k &> /dev/null ; then + echo "done." +else + echo "failed." +fi + + rm -rf lib exe #EOF diff --git a/tools/docker/scripts/install_ubuntu_toolchain.sh b/tools/docker/scripts/install_ubuntu_toolchain.sh index b72f53a022..f7c7f9f080 100755 --- a/tools/docker/scripts/install_ubuntu_toolchain.sh +++ b/tools/docker/scripts/install_ubuntu_toolchain.sh @@ -10,8 +10,8 @@ fi GCC_VERSION=$1 # install Ubuntu packages -apt-get update -apt-get install -y --no-install-recommends \ +apt-get update -qq +apt-get install -qq --no-install-recommends \ gcc-${GCC_VERSION} \ g++-${GCC_VERSION} \ gfortran-${GCC_VERSION} \ diff --git a/tools/docker/scripts/test_aiida.sh b/tools/docker/scripts/test_aiida.sh index 41a164f180..0b9fbf4407 100755 --- a/tools/docker/scripts/test_aiida.sh +++ b/tools/docker/scripts/test_aiida.sh @@ -19,11 +19,11 @@ chmod +x /usr/bin/cp2k echo -e "\n========== Installing AiiDA-CP2K plugin ==========" cd /opt/ -git clone https://github.com/cp2k/aiida-cp2k.git -pip install ./aiida-cp2k/[pre-commit] +git clone --quiet https://github.com/cp2k/aiida-cp2k.git +pip install --quiet ./aiida-cp2k/[pre-commit] # workaround for dependency chain in 1.0.0b1 -pip install 'topika==0.1.3' +pip install --quiet 'topika==0.1.3' echo -e "\n========== Configuring AiiDA ==========" for i in $(dirname "$(which mpirun)")/* ; do ln -sf "$i" /usr/bin/; done diff --git a/tools/docker/scripts/test_i-pi.sh b/tools/docker/scripts/test_i-pi.sh index d1de70477e..01bcc22d63 100755 --- a/tools/docker/scripts/test_i-pi.sh +++ b/tools/docker/scripts/test_i-pi.sh @@ -11,8 +11,8 @@ make -j VERSION=pdbg cp2k echo -e "\n========== Installing i-Pi ==========" cd /opt/i-pi -git pull -pip install . +git pull --quiet +pip install --quiet . echo -e "\n========== Running i-Pi Tests =========="