From 067e01a6732c64a193f73fcdfac5658fc19303bc Mon Sep 17 00:00:00 2001 From: Matthias Krack Date: Thu, 16 Dec 2021 18:47:45 +0100 Subject: [PATCH] Switch to new CRAY-XC50 arch file --- arch/CRAY-XC50-gnu.psmp | 244 ++++++++++++++++++++++++++++++++++++++++ arch/CRAY-XC50_gnu.psmp | 213 ----------------------------------- 2 files changed, 244 insertions(+), 213 deletions(-) create mode 100644 arch/CRAY-XC50-gnu.psmp delete mode 100644 arch/CRAY-XC50_gnu.psmp diff --git a/arch/CRAY-XC50-gnu.psmp b/arch/CRAY-XC50-gnu.psmp new file mode 100644 index 0000000000..6307b00267 --- /dev/null +++ b/arch/CRAY-XC50-gnu.psmp @@ -0,0 +1,244 @@ +#!/bin/bash +# +# CP2K arch file for Cray-XC50 (Piz Daint, CSCS, GPU partition) +# +# Tested with: GNU 8.3.0, Cray-MPICH 7.7.16, Cray-libsci 20.09.1, Cray-FFTW 3.3.8, +# LIBINT 2.6.0, LIBXC 5.1.7, LIBVORI 210412, LIBXSMM 1.16.2, +# ELPA 2021.11.001.rc1, PLUMED 2.7.3, SPGLIB 1.16.2, GSL 2.7, +# COSMA 2.5.1, SIRIUS 7.3.0 +# +# Usage: Source this arch file and then run make as instructed. +# The compiler version can be specified as argument. +# Otherwise a toolchain installation is expected as default. +# +# Author: Matthias Krack (matthias.krack@psi.ch, PSI, December 2021) +# +# \ + if [[ "${0}" == "${BASH_SOURCE}" ]]; then \ + echo "ERROR: Script ${0##*/} must be sourced"; \ + echo "Usage: source ${0##*/}"; \ + exit 1; \ + fi; \ + this_file=${BASH_SOURCE##*/}; \ + if [[ -n "${1}" ]]; then \ + module add daint-gpu; \ + module rm PrgEnv-cray; \ + module add PrgEnv-gnu; \ + module rm gcc; \ + module add gcc/${1}; \ + module add cray-fftw; \ + module add cudatoolkit; \ + echo "Expected setup:"; \ + echo " cray-mpich/7.7.16"; \ + echo " craype-haswell"; \ + echo " daint-gpu/20.11"; \ + echo " craype/2.7.3"; \ + echo " cray-libsci/20.09.1"; \ + echo " PrgEnv-gnu/6.0.9"; \ + echo " gcc/${1}"; \ + echo " cray-fftw/3.3.8.8"; \ + echo " cudatoolkit/11.0.2_3.38-8.1__g5b73779"; \ + module list; \ + module -f save cp2k_gpu_gnu_psmp; \ + echo "To load the required modules in your batch job script, use:"; \ + echo " module restore cp2k_gpu_gnu_psmp"; \ + else \ + printf "Sourcing ${PWD}/tools/toolchain/install/setup ... "; \ + source ${PWD}/tools/toolchain/install/setup; \ + printf "done\n"; \ + fi; \ + echo "Check the output above for error messages and consistency!"; \ + echo "If everything is OK, you can build a CP2K production binary with"; \ + echo " make -j ARCH=${this_file%%.*} VERSION=${this_file##*.}"; \ + echo "Alternatively, you can add further checks, e.g. for regression testing, with"; \ + echo " make -j ARCH=${this_file%%.*} VERSION=${this_file##*.} DO_CHECKS=yes"; \ + echo "or build CP2K as a library with"; \ + echo " make -j ARCH=${this_file%%.*} VERSION=${this_file##*.} libcp2k"; \ + return + +# Set options +DO_CHECKS := no +USE_ACC := yes +USE_COSMA := 2.5.1 +USE_ELPA := 2021.11.001.rc1 +USE_LIBINT := 2.6.0 +USE_LIBVORI := 210412 +USE_LIBXC := 5.1.7 +USE_LIBXSMM := 1.17 +USE_PLUMED := 2.7.3 +USE_SIRIUS := 7.3.0 +USE_SPGLIB := 1.16.2 +# Only needed for SIRIUS +LIBVDWXC_VER := 0.4.0 +SPFFT_VER := 1.0.5 +SPLA_VER := 1.5.2 +HDF5_VER := 1.12.0 + +LMAX := 5 +MAX_CONTR := 4 + +GPUVER := P100 +OFFLOAD_TARGET := cuda + +CC := cc +CXX := CC +OFFLOAD_CC := nvcc +FC := ftn +LD := ftn +AR := ar -r + +# cc, CC, and ftn include already the proper -march flag +CFLAGS := -O2 -fopenmp -fopenmp-simd -ftree-vectorize -funroll-loops -g + +DFLAGS := -D__parallel +DFLAGS += -D__SCALAPACK +DFLAGS += -D__FFTW3 +DFLAGS += -D__MPI_VERSION=3 +DFLAGS += -D__MAX_CONTR=$(strip $(MAX_CONTR)) + +INSTALL_PATH := $(PWD)/tools/toolchain/install + +ifeq ($(DO_CHECKS), yes) + DFLAGS += -D__CHECK_DIAG +endif + +ifeq ($(USE_ACC), yes) + DFLAGS += -D__DBCSR_ACC + DFLAGS += -D__GRID_CUDA +# No performance gain with PW_CUDA currently +# DFLAGS += -D__PW_CUDA +endif + +ifneq ($(USE_PLUMED),) + USE_PLUMED := $(strip $(USE_PLUMED)) + PLUMED_LIB := $(INSTALL_PATH)/plumed-$(USE_PLUMED)/lib + DFLAGS += -D__PLUMED2 + USE_GSL := 2.7 + LIBS += $(PLUMED_LIB)/libplumed.a +endif + +ifneq ($(USE_ELPA),) + USE_ELPA := $(strip $(USE_ELPA)) + ELPA_INC := $(INSTALL_PATH)/elpa-$(USE_ELPA)/gpu/include/elpa-$(USE_ELPA) + ELPA_LIB := $(INSTALL_PATH)/elpa-$(USE_ELPA)/gpu/lib + CFLAGS += -I$(ELPA_INC)/elpa -I$(ELPA_INC)/modules + DFLAGS += -D__ELPA + DFLAGS += -D__ELPA_NVIDIA_GPU + LIBS += $(ELPA_LIB)/libelpa.a +endif + +ifneq ($(USE_LIBVORI),) + USE_LIBVORI := $(strip $(USE_LIBVORI)) + LIBVORI_LIB := $(INSTALL_PATH)/libvori-$(USE_LIBVORI)/lib + DFLAGS += -D__LIBVORI + LIBS += $(LIBVORI_LIB)/libvori.a +endif + +ifneq ($(USE_LIBXC),) + USE_LIBXC := $(strip $(USE_LIBXC)) + LIBXC_INC := $(INSTALL_PATH)/libxc-$(USE_LIBXC)/include + LIBXC_LIB := $(INSTALL_PATH)/libxc-$(USE_LIBXC)/lib + CFLAGS += -I$(LIBXC_INC) + DFLAGS += -D__LIBXC + LIBS += $(LIBXC_LIB)/libxcf03.a + LIBS += $(LIBXC_LIB)/libxc.a +endif + +ifneq ($(USE_LIBINT),) + USE_LIBINT := $(strip $(USE_LIBINT)) + LMAX := $(strip $(LMAX)) + LIBINT_INC := $(INSTALL_PATH)/libint-v$(USE_LIBINT)-cp2k-lmax-$(LMAX)/include + LIBINT_LIB := $(INSTALL_PATH)/libint-v$(USE_LIBINT)-cp2k-lmax-$(LMAX)/lib + CFLAGS += -I$(LIBINT_INC) + DFLAGS += -D__LIBINT + LIBS += $(LIBINT_LIB)/libint2.a +endif + +ifneq ($(USE_SPGLIB),) + USE_SPGLIB := $(strip $(USE_SPGLIB)) + SPGLIB_INC := $(INSTALL_PATH)/spglib-$(USE_SPGLIB)/include + SPGLIB_LIB := $(INSTALL_PATH)/spglib-$(USE_SPGLIB)/lib + CFLAGS += -I$(SPGLIB_INC) + DFLAGS += -D__SPGLIB + LIBS += $(SPGLIB_LIB)/libsymspg.a +endif + +ifneq ($(USE_LIBXSMM),) + USE_LIBXSMM := $(strip $(USE_LIBXSMM)) + LIBXSMM_INC := $(INSTALL_PATH)/libxsmm-$(USE_LIBXSMM)/include + LIBXSMM_LIB := $(INSTALL_PATH)/libxsmm-$(USE_LIBXSMM)/lib + CFLAGS += -I$(LIBXSMM_INC) + DFLAGS += -D__LIBXSMM + LIBS += $(LIBXSMM_LIB)/libxsmmf.a + LIBS += $(LIBXSMM_LIB)/libxsmm.a +endif + +ifneq ($(USE_SIRIUS),) + USE_SIRIUS := $(strip $(USE_SIRIUS)) + SIRIUS_INC := $(INSTALL_PATH)/sirius-$(USE_SIRIUS)/include + SIRIUS_LIB := $(INSTALL_PATH)/sirius-$(USE_SIRIUS)/lib + SPFFT_LIB := $(INSTALL_PATH)/SpFFT-$(SPFFT_VER)/lib + SPLA_LIB := $(INSTALL_PATH)/SpLA-$(SPLA_VER)/lib + HDF5_LIB := $(INSTALL_PATH)/hdf5-$(HDF5_VER)/lib + LIBVDWXC_LIB := $(INSTALL_PATH)/libvdwxc-$(LIBVDWXC_VER)/lib + CFLAGS += -I$(SIRIUS_INC) + DFLAGS += -D__SPFFT + DFLAGS += -D__SPLA + DFLAGS += -D__HDF5 + DFLAGS += -D__LIBVDWXC + DFLAGS += -D__SIRIUS + LIBS += $(SIRIUS_LIB)/libsirius.a + LIBS += $(SPFFT_LIB)/libspfft.a + LIBS += $(SPLA_LIB)/libspla.a + LIBS += $(HDF5_LIB)/libhdf5.a + LIBS += $(LIBVDWXC_LIB)/libvdwxc.a +endif + +ifneq ($(USE_COSMA),) + USE_COSMA := $(strip $(USE_COSMA)) + COSMA_INC := $(INSTALL_PATH)/COSMA-$(USE_COSMA)/include + COSMA_LIB := $(INSTALL_PATH)/COSMA-$(USE_COSMA)/lib + CFLAGS += -I$(COSMA_INC) + DFLAGS += -D__COSMA + LIBS += $(COSMA_LIB)/libcosma_prefixed_pxgemm.a + LIBS += $(COSMA_LIB)/libcosma.a + LIBS += $(COSMA_LIB)/libcosta_prefixed_scalapack.a + LIBS += $(COSMA_LIB)/libcosta.a +endif + +ifneq ($(USE_GSL),) + USE_GSL := $(strip $(USE_GSL)) + GSL_INC := $(INSTALL_PATH)/gsl-$(USE_GSL)/include + GSL_LIB := $(INSTALL_PATH)/gsl-$(USE_GSL)/lib + CFLAGS += -I$(GSL_INC) + DFLAGS += -D__GSL + LIBS += $(GSL_LIB)/libgsl.a +endif + +CFLAGS += $(DFLAGS) + +CXXFLAGS := $(CFLAGS) -std=c++11 + +OFFLOAD_FLAGS := $(DFLAGS) -O3 -Xcompiler="-fopenmp" -arch sm_60 --std=c++11 + +FCFLAGS := $(CFLAGS) +ifeq ($(shell (( $(shell gcc -dumpversion | cut -d. -f1) > 9 )) && echo yes), yes) + FCFLAGS += -fallow-argument-mismatch +endif +FCFLAGS += -fbacktrace +FCFLAGS += -ffree-form +FCFLAGS += -ffree-line-length-none +FCFLAGS += -fno-omit-frame-pointer +FCFLAGS += -std=f2008 + +ifneq ($(CUDA_HOME),) + CUDA_LIB := $(CUDA_HOME)/lib64 + LDFLAGS := $(FCFLAGS) -L$(CUDA_LIB) -Wl,-rpath=$(CUDA_LIB) +else + LDFLAGS := $(FCFLAGS) +endif + +LIBS += -lcudart -lcublas -lcufft -lnvrtc -lrt +LIBS += -lz -ldl -lstdc++ + +# End diff --git a/arch/CRAY-XC50_gnu.psmp b/arch/CRAY-XC50_gnu.psmp deleted file mode 100644 index b5933d7346..0000000000 --- a/arch/CRAY-XC50_gnu.psmp +++ /dev/null @@ -1,213 +0,0 @@ -#!/bin/bash -# -# CP2K arch file for Cray-XC50 (Piz Daint, CSCS, GPU partition) -# -# Tested with: GFortran 8.3.0, Cray-MPICH 7.7.16, Cray-libsci 20.09.1, Cray-FFTW 3.3.8, Libint 2.6.0, -# LIBXC 5.1.7, LIBVORI 210412, LIBXSMM 1.16.2, ELPA 2021.05.002, PLUMED 2.7.3, -# SPGLIB 1.16.2, GSL 2.7, SIRIUS 7.3.0, COSMA 2.5.1 -# -# Usage: Source this arch file and then run make as instructed -# -# Author: Matthias Krack (matthias.krack@psi.ch, PSI, December 2021) -# -# \ - if [[ "${0}" == "${BASH_SOURCE}" ]]; then \ - echo "ERROR: Script ${0##*/} must be sourced"; \ - echo "Usage: source ${0##*/}"; \ - exit 1; \ - fi; \ - this_file=${BASH_SOURCE##*/}; \ - module add daint-gpu; \ - module rm PrgEnv-cray; \ - module add PrgEnv-gnu; \ - module rm gcc/10.1.0; \ - module add gcc/8.3.0; \ - module add cray-fftw; \ - module add cudatoolkit; \ - echo "Expected setup:"; \ - echo " cray-mpich/7.7.16"; \ - echo " craype-haswell"; \ - echo " daint-gpu"; \ - echo " craype/2.7.3"; \ - echo " cray-libsci/20.09.1"; \ - echo " PrgEnv-gnu/6.0.9"; \ - echo " gcc/8.3.0"; \ - echo " cray-fftw/3.3.8.8"; \ - echo " cudatoolkit/11.0.2_3.38-8.1__g5b73779"; \ - module list; \ - module -f save cp2k_gpu_gnu_psmp; \ - echo "Check the output above for error messages and consistency!"; \ - echo "To load the required modules in your batch job script, use:"; \ - echo " module restore cp2k_gpu_gnu_psmp"; \ - echo "If everything is OK, build CP2K with:"; \ - echo " make -j ARCH=${this_file%%.*} VERSION=${this_file##*.}"; \ - return - -# Set options (leave option blank or comment line to disable) -DO_CHECKS := no -GNU_VERSION := 8.3.0 -USE_ACC := yes -USE_COSMA := 2.5.1 -USE_ELPA := 2021.05.002 -USE_LIBINT := 2.6.0 -USE_LIBVORI := 210412 -USE_LIBXC := 5.1.7 -USE_LIBXSMM := 1.17 -USE_PLUMED := 2.7.3 -USE_SIRIUS := 7.3.0 -USE_SPGLIB := 1.16.2 -LMAX := 6 - -GPUVER := P100 -OFFLOAD_TARGET := cuda - -CC := cc -CXX := CC -OFFLOAD_CC := nvcc -FC := ftn -LD := ftn -AR := ar -r - -# cc, CC, and ftn include already the proper -march flag -CFLAGS := -O2 -fopenmp -fopenmp-simd -ftree-vectorize -funroll-loops -g - -DFLAGS := -D__parallel -DFLAGS += -D__SCALAPACK -DFLAGS += -D__FFTW3 -DFLAGS += -D__MAX_CONTR=4 -DFLAGS += -D__MPI_VERSION=3 - -GNU_PATH := /store/psi/psi01/mkrack/daint/gpu/gnu/$(GNU_VERSION) -MPI_PATH := $(GNU_PATH)/mpich - -ifeq ($(DO_CHECKS), yes) -DFLAGS += -D__CHECK_DIAG -endif - -ifeq ($(USE_ACC), yes) -DFLAGS += -D__DBCSR_ACC -DFLAGS += -D__GRID_CUDA -# No performance gain with PW_CUDA currently -# DFLAGS += -D__PW_CUDA -endif - -ifneq ($(USE_PLUMED),) -USE_PLUMED := $(strip $(USE_PLUMED)) -include $(MPI_PATH)/plumed2/$(USE_PLUMED)/lib/plumed/src/lib/Plumed.inc.static -DFLAGS += -D__PLUMED2 -LIBS += $(PLUMED_DEPENDENCIES) -USE_GSL := 2.7 -USE_LIBZ := yes -endif - -ifneq ($(USE_ELPA),) -USE_ELPA := $(strip $(USE_ELPA)) -ELPA_INC := $(MPI_PATH)/elpa/$(USE_ELPA)/gpu/include/elpa-$(USE_ELPA) -ELPA_LIB := $(MPI_PATH)/elpa/$(USE_ELPA)/gpu/lib -CFLAGS += -I$(ELPA_INC)/elpa -I$(ELPA_INC)/modules -DFLAGS += -D__ELPA -DFLAGS += -D__ELPA_NVIDIA_GPU -LIBS += $(ELPA_LIB)/libelpa.a -endif - -ifneq ($(USE_LIBVORI),) -USE_LIBVORI := $(strip $(USE_LIBVORI)) -LIBVORI_LIB := $(GNU_PATH)/libvori/$(USE_LIBVORI)/lib -DFLAGS += -D__LIBVORI -LIBS += $(LIBVORI_LIB)/libvori.a -endif - -ifneq ($(USE_LIBXC),) -USE_LIBXC := $(strip $(USE_LIBXC)) -LIBXC_INC := $(GNU_PATH)/libxc/$(USE_LIBXC)/include -LIBXC_LIB := $(GNU_PATH)/libxc/$(USE_LIBXC)/lib -CFLAGS += -I$(LIBXC_INC) -DFLAGS += -D__LIBXC -LIBS += $(LIBXC_LIB)/libxcf03.a -LIBS += $(LIBXC_LIB)/libxc.a -endif - -ifneq ($(USE_LIBINT),) -USE_LIBINT := $(strip $(USE_LIBINT)) -LMAX := $(strip $(LMAX)) -LIBINT_INC := $(GNU_PATH)/libint/$(USE_LIBINT)-lmax-$(LMAX)/include -LIBINT_LIB := $(GNU_PATH)/libint/$(USE_LIBINT)-lmax-$(LMAX)/lib -CFLAGS += -I$(LIBINT_INC) -DFLAGS += -D__LIBINT -LIBS += $(LIBINT_LIB)/libint2.a -endif - -ifneq ($(USE_SPGLIB),) -USE_SPGLIB := $(strip $(USE_SPGLIB)) -SPGLIB_INC := $(GNU_PATH)/spglib/$(USE_SPGLIB)/include -SPGLIB_LIB := $(GNU_PATH)/spglib/$(USE_SPGLIB)/lib -CFLAGS += -I$(SPGLIB_INC) -DFLAGS += -D__SPGLIB -LIBS += $(SPGLIB_LIB)/libsymspg.a -endif - -ifneq ($(USE_LIBXSMM),) -USE_LIBXSMM := $(strip $(USE_LIBXSMM)) -LIBXSMM_INC := $(GNU_PATH)/libxsmm/$(USE_LIBXSMM)/include -LIBXSMM_LIB := $(GNU_PATH)/libxsmm/$(USE_LIBXSMM)/lib -CFLAGS += -I$(LIBXSMM_INC) -DFLAGS += -D__LIBXSMM -LIBS += $(LIBXSMM_LIB)/libxsmmf.a -LIBS += $(LIBXSMM_LIB)/libxsmm.a -endif - -ifneq ($(USE_SIRIUS),) -USE_SIRIUS := $(strip $(USE_SIRIUS)) -SIRIUS_INC := $(GNU_PATH)/sirius/$(USE_SIRIUS)/include -SIRIUS_LIB := $(GNU_PATH)/sirius/$(USE_SIRIUS)/lib -CFLAGS += -I$(SIRIUS_INC) -DFLAGS += -D__SIRIUS -LIBS += ${SIRIUS_LIB}/libsirius.a -LIBS += $(GNU_PATH)/SpFFT/1.0.5/lib/libspfft.a -LIBS += $(GNU_PATH)/SpLA/1.5.2/lib/libspla.a -LIBS += $(GNU_PATH)/hdf5/1.12.0/lib/libhdf5.a -endif - -ifneq ($(USE_COSMA),) -USE_COSMA := $(strip $(USE_COSMA)) -COSMA_INC := $(GNU_PATH)/COSMA/$(USE_COSMA)/include -COSMA_LIB := $(GNU_PATH)/COSMA/$(USE_COSMA)/lib -CFLAGS += -I$(COSMA_INC) -DFLAGS += -D__COSMA -LIBS += $(COSMA_LIB)/libcosma_prefixed_pxgemm.a -LIBS += $(COSMA_LIB)/libcosma.a -LIBS += $(COSMA_LIB)/libcosta_prefixed_scalapack.a -LIBS += $(COSMA_LIB)/libcosta.a -endif - -ifneq ($(USE_GSL),) -USE_GSL := $(strip $(USE_GSL)) -GSL_INC := $(GNU_PATH)/gsl/$(USE_GSL)/include -GSL_LIB := $(GNU_PATH)/gsl/$(USE_GSL)/lib -CFLAGS += -I$(GSL_INC) -DFLAGS += -D__GSL -LIBS += $(GSL_LIB)/libgsl.a -endif - -ifeq ($(USE_LIBZ), yes) -LIBS += $(GNU_PATH)/lib/libz.a -endif - -CFLAGS += $(DFLAGS) - -CXXFLAGS := $(CFLAGS) -std=c++11 - -OFFLOAD_FLAGS := $(DFLAGS) -O3 -Xcompiler="-fopenmp" -arch sm_60 --std=c++11 - -FCFLAGS := $(CFLAGS) -FCFLAGS += -fbacktrace -FCFLAGS += -ffree-form -FCFLAGS += -ffree-line-length-none -FCFLAGS += -fno-omit-frame-pointer -FCFLAGS += -std=f2008 - -# -static not possible with CUDA -LDFLAGS := $(FCFLAGS) - -LIBS += -lcudart -lcublas -lcufft -lnvrtc -lrt -LIBS += -ldl -lstdc++