Switch to new CRAY-XC50 arch file

This commit is contained in:
Matthias Krack 2021-12-16 18:47:45 +01:00
parent be688fe48c
commit 067e01a673
2 changed files with 244 additions and 213 deletions

244
arch/CRAY-XC50-gnu.psmp Normal file
View file

@ -0,0 +1,244 @@
#!/bin/bash
#
# CP2K arch file for Cray-XC50 (Piz Daint, CSCS, GPU partition)
#
# Tested with: GNU 8.3.0, Cray-MPICH 7.7.16, Cray-libsci 20.09.1, Cray-FFTW 3.3.8,
# LIBINT 2.6.0, LIBXC 5.1.7, LIBVORI 210412, LIBXSMM 1.16.2,
# ELPA 2021.11.001.rc1, PLUMED 2.7.3, SPGLIB 1.16.2, GSL 2.7,
# COSMA 2.5.1, SIRIUS 7.3.0
#
# Usage: Source this arch file and then run make as instructed.
# The compiler version can be specified as argument.
# Otherwise a toolchain installation is expected as default.
#
# Author: Matthias Krack (matthias.krack@psi.ch, PSI, December 2021)
#
# \
if [[ "${0}" == "${BASH_SOURCE}" ]]; then \
echo "ERROR: Script ${0##*/} must be sourced"; \
echo "Usage: source ${0##*/}"; \
exit 1; \
fi; \
this_file=${BASH_SOURCE##*/}; \
if [[ -n "${1}" ]]; then \
module add daint-gpu; \
module rm PrgEnv-cray; \
module add PrgEnv-gnu; \
module rm gcc; \
module add gcc/${1}; \
module add cray-fftw; \
module add cudatoolkit; \
echo "Expected setup:"; \
echo " cray-mpich/7.7.16"; \
echo " craype-haswell"; \
echo " daint-gpu/20.11"; \
echo " craype/2.7.3"; \
echo " cray-libsci/20.09.1"; \
echo " PrgEnv-gnu/6.0.9"; \
echo " gcc/${1}"; \
echo " cray-fftw/3.3.8.8"; \
echo " cudatoolkit/11.0.2_3.38-8.1__g5b73779"; \
module list; \
module -f save cp2k_gpu_gnu_psmp; \
echo "To load the required modules in your batch job script, use:"; \
echo " module restore cp2k_gpu_gnu_psmp"; \
else \
printf "Sourcing ${PWD}/tools/toolchain/install/setup ... "; \
source ${PWD}/tools/toolchain/install/setup; \
printf "done\n"; \
fi; \
echo "Check the output above for error messages and consistency!"; \
echo "If everything is OK, you can build a CP2K production binary with"; \
echo " make -j ARCH=${this_file%%.*} VERSION=${this_file##*.}"; \
echo "Alternatively, you can add further checks, e.g. for regression testing, with"; \
echo " make -j ARCH=${this_file%%.*} VERSION=${this_file##*.} DO_CHECKS=yes"; \
echo "or build CP2K as a library with"; \
echo " make -j ARCH=${this_file%%.*} VERSION=${this_file##*.} libcp2k"; \
return
# Set options
DO_CHECKS := no
USE_ACC := yes
USE_COSMA := 2.5.1
USE_ELPA := 2021.11.001.rc1
USE_LIBINT := 2.6.0
USE_LIBVORI := 210412
USE_LIBXC := 5.1.7
USE_LIBXSMM := 1.17
USE_PLUMED := 2.7.3
USE_SIRIUS := 7.3.0
USE_SPGLIB := 1.16.2
# Only needed for SIRIUS
LIBVDWXC_VER := 0.4.0
SPFFT_VER := 1.0.5
SPLA_VER := 1.5.2
HDF5_VER := 1.12.0
LMAX := 5
MAX_CONTR := 4
GPUVER := P100
OFFLOAD_TARGET := cuda
CC := cc
CXX := CC
OFFLOAD_CC := nvcc
FC := ftn
LD := ftn
AR := ar -r
# cc, CC, and ftn include already the proper -march flag
CFLAGS := -O2 -fopenmp -fopenmp-simd -ftree-vectorize -funroll-loops -g
DFLAGS := -D__parallel
DFLAGS += -D__SCALAPACK
DFLAGS += -D__FFTW3
DFLAGS += -D__MPI_VERSION=3
DFLAGS += -D__MAX_CONTR=$(strip $(MAX_CONTR))
INSTALL_PATH := $(PWD)/tools/toolchain/install
ifeq ($(DO_CHECKS), yes)
DFLAGS += -D__CHECK_DIAG
endif
ifeq ($(USE_ACC), yes)
DFLAGS += -D__DBCSR_ACC
DFLAGS += -D__GRID_CUDA
# No performance gain with PW_CUDA currently
# DFLAGS += -D__PW_CUDA
endif
ifneq ($(USE_PLUMED),)
USE_PLUMED := $(strip $(USE_PLUMED))
PLUMED_LIB := $(INSTALL_PATH)/plumed-$(USE_PLUMED)/lib
DFLAGS += -D__PLUMED2
USE_GSL := 2.7
LIBS += $(PLUMED_LIB)/libplumed.a
endif
ifneq ($(USE_ELPA),)
USE_ELPA := $(strip $(USE_ELPA))
ELPA_INC := $(INSTALL_PATH)/elpa-$(USE_ELPA)/gpu/include/elpa-$(USE_ELPA)
ELPA_LIB := $(INSTALL_PATH)/elpa-$(USE_ELPA)/gpu/lib
CFLAGS += -I$(ELPA_INC)/elpa -I$(ELPA_INC)/modules
DFLAGS += -D__ELPA
DFLAGS += -D__ELPA_NVIDIA_GPU
LIBS += $(ELPA_LIB)/libelpa.a
endif
ifneq ($(USE_LIBVORI),)
USE_LIBVORI := $(strip $(USE_LIBVORI))
LIBVORI_LIB := $(INSTALL_PATH)/libvori-$(USE_LIBVORI)/lib
DFLAGS += -D__LIBVORI
LIBS += $(LIBVORI_LIB)/libvori.a
endif
ifneq ($(USE_LIBXC),)
USE_LIBXC := $(strip $(USE_LIBXC))
LIBXC_INC := $(INSTALL_PATH)/libxc-$(USE_LIBXC)/include
LIBXC_LIB := $(INSTALL_PATH)/libxc-$(USE_LIBXC)/lib
CFLAGS += -I$(LIBXC_INC)
DFLAGS += -D__LIBXC
LIBS += $(LIBXC_LIB)/libxcf03.a
LIBS += $(LIBXC_LIB)/libxc.a
endif
ifneq ($(USE_LIBINT),)
USE_LIBINT := $(strip $(USE_LIBINT))
LMAX := $(strip $(LMAX))
LIBINT_INC := $(INSTALL_PATH)/libint-v$(USE_LIBINT)-cp2k-lmax-$(LMAX)/include
LIBINT_LIB := $(INSTALL_PATH)/libint-v$(USE_LIBINT)-cp2k-lmax-$(LMAX)/lib
CFLAGS += -I$(LIBINT_INC)
DFLAGS += -D__LIBINT
LIBS += $(LIBINT_LIB)/libint2.a
endif
ifneq ($(USE_SPGLIB),)
USE_SPGLIB := $(strip $(USE_SPGLIB))
SPGLIB_INC := $(INSTALL_PATH)/spglib-$(USE_SPGLIB)/include
SPGLIB_LIB := $(INSTALL_PATH)/spglib-$(USE_SPGLIB)/lib
CFLAGS += -I$(SPGLIB_INC)
DFLAGS += -D__SPGLIB
LIBS += $(SPGLIB_LIB)/libsymspg.a
endif
ifneq ($(USE_LIBXSMM),)
USE_LIBXSMM := $(strip $(USE_LIBXSMM))
LIBXSMM_INC := $(INSTALL_PATH)/libxsmm-$(USE_LIBXSMM)/include
LIBXSMM_LIB := $(INSTALL_PATH)/libxsmm-$(USE_LIBXSMM)/lib
CFLAGS += -I$(LIBXSMM_INC)
DFLAGS += -D__LIBXSMM
LIBS += $(LIBXSMM_LIB)/libxsmmf.a
LIBS += $(LIBXSMM_LIB)/libxsmm.a
endif
ifneq ($(USE_SIRIUS),)
USE_SIRIUS := $(strip $(USE_SIRIUS))
SIRIUS_INC := $(INSTALL_PATH)/sirius-$(USE_SIRIUS)/include
SIRIUS_LIB := $(INSTALL_PATH)/sirius-$(USE_SIRIUS)/lib
SPFFT_LIB := $(INSTALL_PATH)/SpFFT-$(SPFFT_VER)/lib
SPLA_LIB := $(INSTALL_PATH)/SpLA-$(SPLA_VER)/lib
HDF5_LIB := $(INSTALL_PATH)/hdf5-$(HDF5_VER)/lib
LIBVDWXC_LIB := $(INSTALL_PATH)/libvdwxc-$(LIBVDWXC_VER)/lib
CFLAGS += -I$(SIRIUS_INC)
DFLAGS += -D__SPFFT
DFLAGS += -D__SPLA
DFLAGS += -D__HDF5
DFLAGS += -D__LIBVDWXC
DFLAGS += -D__SIRIUS
LIBS += $(SIRIUS_LIB)/libsirius.a
LIBS += $(SPFFT_LIB)/libspfft.a
LIBS += $(SPLA_LIB)/libspla.a
LIBS += $(HDF5_LIB)/libhdf5.a
LIBS += $(LIBVDWXC_LIB)/libvdwxc.a
endif
ifneq ($(USE_COSMA),)
USE_COSMA := $(strip $(USE_COSMA))
COSMA_INC := $(INSTALL_PATH)/COSMA-$(USE_COSMA)/include
COSMA_LIB := $(INSTALL_PATH)/COSMA-$(USE_COSMA)/lib
CFLAGS += -I$(COSMA_INC)
DFLAGS += -D__COSMA
LIBS += $(COSMA_LIB)/libcosma_prefixed_pxgemm.a
LIBS += $(COSMA_LIB)/libcosma.a
LIBS += $(COSMA_LIB)/libcosta_prefixed_scalapack.a
LIBS += $(COSMA_LIB)/libcosta.a
endif
ifneq ($(USE_GSL),)
USE_GSL := $(strip $(USE_GSL))
GSL_INC := $(INSTALL_PATH)/gsl-$(USE_GSL)/include
GSL_LIB := $(INSTALL_PATH)/gsl-$(USE_GSL)/lib
CFLAGS += -I$(GSL_INC)
DFLAGS += -D__GSL
LIBS += $(GSL_LIB)/libgsl.a
endif
CFLAGS += $(DFLAGS)
CXXFLAGS := $(CFLAGS) -std=c++11
OFFLOAD_FLAGS := $(DFLAGS) -O3 -Xcompiler="-fopenmp" -arch sm_60 --std=c++11
FCFLAGS := $(CFLAGS)
ifeq ($(shell (( $(shell gcc -dumpversion | cut -d. -f1) > 9 )) && echo yes), yes)
FCFLAGS += -fallow-argument-mismatch
endif
FCFLAGS += -fbacktrace
FCFLAGS += -ffree-form
FCFLAGS += -ffree-line-length-none
FCFLAGS += -fno-omit-frame-pointer
FCFLAGS += -std=f2008
ifneq ($(CUDA_HOME),)
CUDA_LIB := $(CUDA_HOME)/lib64
LDFLAGS := $(FCFLAGS) -L$(CUDA_LIB) -Wl,-rpath=$(CUDA_LIB)
else
LDFLAGS := $(FCFLAGS)
endif
LIBS += -lcudart -lcublas -lcufft -lnvrtc -lrt
LIBS += -lz -ldl -lstdc++
# End

View file

@ -1,213 +0,0 @@
#!/bin/bash
#
# CP2K arch file for Cray-XC50 (Piz Daint, CSCS, GPU partition)
#
# Tested with: GFortran 8.3.0, Cray-MPICH 7.7.16, Cray-libsci 20.09.1, Cray-FFTW 3.3.8, Libint 2.6.0,
# LIBXC 5.1.7, LIBVORI 210412, LIBXSMM 1.16.2, ELPA 2021.05.002, PLUMED 2.7.3,
# SPGLIB 1.16.2, GSL 2.7, SIRIUS 7.3.0, COSMA 2.5.1
#
# Usage: Source this arch file and then run make as instructed
#
# Author: Matthias Krack (matthias.krack@psi.ch, PSI, December 2021)
#
# \
if [[ "${0}" == "${BASH_SOURCE}" ]]; then \
echo "ERROR: Script ${0##*/} must be sourced"; \
echo "Usage: source ${0##*/}"; \
exit 1; \
fi; \
this_file=${BASH_SOURCE##*/}; \
module add daint-gpu; \
module rm PrgEnv-cray; \
module add PrgEnv-gnu; \
module rm gcc/10.1.0; \
module add gcc/8.3.0; \
module add cray-fftw; \
module add cudatoolkit; \
echo "Expected setup:"; \
echo " cray-mpich/7.7.16"; \
echo " craype-haswell"; \
echo " daint-gpu"; \
echo " craype/2.7.3"; \
echo " cray-libsci/20.09.1"; \
echo " PrgEnv-gnu/6.0.9"; \
echo " gcc/8.3.0"; \
echo " cray-fftw/3.3.8.8"; \
echo " cudatoolkit/11.0.2_3.38-8.1__g5b73779"; \
module list; \
module -f save cp2k_gpu_gnu_psmp; \
echo "Check the output above for error messages and consistency!"; \
echo "To load the required modules in your batch job script, use:"; \
echo " module restore cp2k_gpu_gnu_psmp"; \
echo "If everything is OK, build CP2K with:"; \
echo " make -j ARCH=${this_file%%.*} VERSION=${this_file##*.}"; \
return
# Set options (leave option blank or comment line to disable)
DO_CHECKS := no
GNU_VERSION := 8.3.0
USE_ACC := yes
USE_COSMA := 2.5.1
USE_ELPA := 2021.05.002
USE_LIBINT := 2.6.0
USE_LIBVORI := 210412
USE_LIBXC := 5.1.7
USE_LIBXSMM := 1.17
USE_PLUMED := 2.7.3
USE_SIRIUS := 7.3.0
USE_SPGLIB := 1.16.2
LMAX := 6
GPUVER := P100
OFFLOAD_TARGET := cuda
CC := cc
CXX := CC
OFFLOAD_CC := nvcc
FC := ftn
LD := ftn
AR := ar -r
# cc, CC, and ftn include already the proper -march flag
CFLAGS := -O2 -fopenmp -fopenmp-simd -ftree-vectorize -funroll-loops -g
DFLAGS := -D__parallel
DFLAGS += -D__SCALAPACK
DFLAGS += -D__FFTW3
DFLAGS += -D__MAX_CONTR=4
DFLAGS += -D__MPI_VERSION=3
GNU_PATH := /store/psi/psi01/mkrack/daint/gpu/gnu/$(GNU_VERSION)
MPI_PATH := $(GNU_PATH)/mpich
ifeq ($(DO_CHECKS), yes)
DFLAGS += -D__CHECK_DIAG
endif
ifeq ($(USE_ACC), yes)
DFLAGS += -D__DBCSR_ACC
DFLAGS += -D__GRID_CUDA
# No performance gain with PW_CUDA currently
# DFLAGS += -D__PW_CUDA
endif
ifneq ($(USE_PLUMED),)
USE_PLUMED := $(strip $(USE_PLUMED))
include $(MPI_PATH)/plumed2/$(USE_PLUMED)/lib/plumed/src/lib/Plumed.inc.static
DFLAGS += -D__PLUMED2
LIBS += $(PLUMED_DEPENDENCIES)
USE_GSL := 2.7
USE_LIBZ := yes
endif
ifneq ($(USE_ELPA),)
USE_ELPA := $(strip $(USE_ELPA))
ELPA_INC := $(MPI_PATH)/elpa/$(USE_ELPA)/gpu/include/elpa-$(USE_ELPA)
ELPA_LIB := $(MPI_PATH)/elpa/$(USE_ELPA)/gpu/lib
CFLAGS += -I$(ELPA_INC)/elpa -I$(ELPA_INC)/modules
DFLAGS += -D__ELPA
DFLAGS += -D__ELPA_NVIDIA_GPU
LIBS += $(ELPA_LIB)/libelpa.a
endif
ifneq ($(USE_LIBVORI),)
USE_LIBVORI := $(strip $(USE_LIBVORI))
LIBVORI_LIB := $(GNU_PATH)/libvori/$(USE_LIBVORI)/lib
DFLAGS += -D__LIBVORI
LIBS += $(LIBVORI_LIB)/libvori.a
endif
ifneq ($(USE_LIBXC),)
USE_LIBXC := $(strip $(USE_LIBXC))
LIBXC_INC := $(GNU_PATH)/libxc/$(USE_LIBXC)/include
LIBXC_LIB := $(GNU_PATH)/libxc/$(USE_LIBXC)/lib
CFLAGS += -I$(LIBXC_INC)
DFLAGS += -D__LIBXC
LIBS += $(LIBXC_LIB)/libxcf03.a
LIBS += $(LIBXC_LIB)/libxc.a
endif
ifneq ($(USE_LIBINT),)
USE_LIBINT := $(strip $(USE_LIBINT))
LMAX := $(strip $(LMAX))
LIBINT_INC := $(GNU_PATH)/libint/$(USE_LIBINT)-lmax-$(LMAX)/include
LIBINT_LIB := $(GNU_PATH)/libint/$(USE_LIBINT)-lmax-$(LMAX)/lib
CFLAGS += -I$(LIBINT_INC)
DFLAGS += -D__LIBINT
LIBS += $(LIBINT_LIB)/libint2.a
endif
ifneq ($(USE_SPGLIB),)
USE_SPGLIB := $(strip $(USE_SPGLIB))
SPGLIB_INC := $(GNU_PATH)/spglib/$(USE_SPGLIB)/include
SPGLIB_LIB := $(GNU_PATH)/spglib/$(USE_SPGLIB)/lib
CFLAGS += -I$(SPGLIB_INC)
DFLAGS += -D__SPGLIB
LIBS += $(SPGLIB_LIB)/libsymspg.a
endif
ifneq ($(USE_LIBXSMM),)
USE_LIBXSMM := $(strip $(USE_LIBXSMM))
LIBXSMM_INC := $(GNU_PATH)/libxsmm/$(USE_LIBXSMM)/include
LIBXSMM_LIB := $(GNU_PATH)/libxsmm/$(USE_LIBXSMM)/lib
CFLAGS += -I$(LIBXSMM_INC)
DFLAGS += -D__LIBXSMM
LIBS += $(LIBXSMM_LIB)/libxsmmf.a
LIBS += $(LIBXSMM_LIB)/libxsmm.a
endif
ifneq ($(USE_SIRIUS),)
USE_SIRIUS := $(strip $(USE_SIRIUS))
SIRIUS_INC := $(GNU_PATH)/sirius/$(USE_SIRIUS)/include
SIRIUS_LIB := $(GNU_PATH)/sirius/$(USE_SIRIUS)/lib
CFLAGS += -I$(SIRIUS_INC)
DFLAGS += -D__SIRIUS
LIBS += ${SIRIUS_LIB}/libsirius.a
LIBS += $(GNU_PATH)/SpFFT/1.0.5/lib/libspfft.a
LIBS += $(GNU_PATH)/SpLA/1.5.2/lib/libspla.a
LIBS += $(GNU_PATH)/hdf5/1.12.0/lib/libhdf5.a
endif
ifneq ($(USE_COSMA),)
USE_COSMA := $(strip $(USE_COSMA))
COSMA_INC := $(GNU_PATH)/COSMA/$(USE_COSMA)/include
COSMA_LIB := $(GNU_PATH)/COSMA/$(USE_COSMA)/lib
CFLAGS += -I$(COSMA_INC)
DFLAGS += -D__COSMA
LIBS += $(COSMA_LIB)/libcosma_prefixed_pxgemm.a
LIBS += $(COSMA_LIB)/libcosma.a
LIBS += $(COSMA_LIB)/libcosta_prefixed_scalapack.a
LIBS += $(COSMA_LIB)/libcosta.a
endif
ifneq ($(USE_GSL),)
USE_GSL := $(strip $(USE_GSL))
GSL_INC := $(GNU_PATH)/gsl/$(USE_GSL)/include
GSL_LIB := $(GNU_PATH)/gsl/$(USE_GSL)/lib
CFLAGS += -I$(GSL_INC)
DFLAGS += -D__GSL
LIBS += $(GSL_LIB)/libgsl.a
endif
ifeq ($(USE_LIBZ), yes)
LIBS += $(GNU_PATH)/lib/libz.a
endif
CFLAGS += $(DFLAGS)
CXXFLAGS := $(CFLAGS) -std=c++11
OFFLOAD_FLAGS := $(DFLAGS) -O3 -Xcompiler="-fopenmp" -arch sm_60 --std=c++11
FCFLAGS := $(CFLAGS)
FCFLAGS += -fbacktrace
FCFLAGS += -ffree-form
FCFLAGS += -ffree-line-length-none
FCFLAGS += -fno-omit-frame-pointer
FCFLAGS += -std=f2008
# -static not possible with CUDA
LDFLAGS := $(FCFLAGS)
LIBS += -lcudart -lcublas -lcufft -lnvrtc -lrt
LIBS += -ldl -lstdc++