diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index c60caa6343..0000000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "exts/dbcsr"] - path = exts/dbcsr - url = https://github.com/cp2k/dbcsr - branch = master diff --git a/exts/Makefile.inc b/exts/Makefile.inc deleted file mode 100644 index 50e19cce30..0000000000 --- a/exts/Makefile.inc +++ /dev/null @@ -1,63 +0,0 @@ -ifeq ("","$(wildcard $(EXTSHOME)/dbcsr/src)") -$(error "No DBCSR submodule available, please run 'git submodule update --init --recursive'") -endif - -LIBS := -L$(LIBEXTSDIR)/dbcsr -ldbcsr $(LIBS) -EXTSDEPS_LIB = $(LIBEXTSDIR)/dbcsr/libdbcsr$(ARCHIVE_EXT) -EXTSDEPS_MOD = $(OBJEXTSDIR)/dbcsr/dbcsr_api.mod $(OBJEXTSDIR)/dbcsr/dbcsr_tensor_api.mod -$(EXTSDEPS_MOD) : ; # override builtin .mod rule to prevent circular dependency - -# Set Acceleration flags if glue code is supposed to be accelerated -ifneq (,$(findstring __DBCSR_ACC,$(OFFLOAD_FLAGS) $(FCFLAGS))) -# By default, we use CUDA if OFFLOAD_TARGET is not specified -ifeq (,$(OFFLOAD_TARGET)) -DBCSR_USE_ACCEL = cuda -else -DBCSR_USE_ACCEL = $(OFFLOAD_TARGET) -endif -ifeq (cuda,$(DBCSR_USE_ACCEL)) -DBCSR_ACC := $(OFFLOAD_CC) -DBCSR_ACCFLAGS := $(OFFLOAD_FLAGS) -Xcompiler='-Wno-error' -else ifeq (hip,$(DBCSR_USE_ACCEL)) -DBCSR_ACC := $(CXX) -DBCSR_ACCFLAGS := $(CXXFLAGS) -Wno-error=unused-parameter -else ifeq (opencl,$(DBCSR_USE_ACCEL)) -DBCSR_ACC := $(CC) -DBCSR_ACCFLAGS := $(CFLAGS) -else -$(error "Invalid OFFLOAD_TARGET=$(OFFLOAD_TARGET)") -endif -endif - -extversions: dbcsrversion - -extclean: dbcsrclean - @echo "Clean EXT" - -dbcsr: - +$(MAKE) -C $(EXTSHOME)/$@ -f $(EXTSHOME)/build_$@/Makefile \ - ARCHFILE=$(ARCHDIR)/$(ARCH).$(ONEVERSION) \ - LIBDIR=$(LIBEXTSDIR)/$@ \ - OBJDIR=$(OBJEXTSDIR)/$@ \ - USE_ACCEL="$(DBCSR_USE_ACCEL)" \ - ACC="$(DBCSR_ACC)" \ - ACCFLAGS="$(strip $(DBCSR_ACCFLAGS))" - -dbcsrversion: - @$(MAKE) -C $(EXTSHOME)/dbcsr -f $(EXTSHOME)/build_dbcsr/Makefile \ - ARCHFILE=$(ARCHDIR)/$(ARCH).$(ONEVERSION) \ - USE_ACCEL="$(DBCSR_USE_ACCEL)" \ - ACC="$(DBCSR_ACC)" \ - ACCFLAGS="$(strip $(DBCSR_ACCFLAGS))" \ - version - -dbcsrclean: - @echo "Clean DBCSR" - @$(MAKE) -C $(EXTSHOME)/dbcsr -f $(EXTSHOME)/build_dbcsr/Makefile \ - ARCHFILE=$(ARCHDIR)/$(ARCH).$(ONEVERSION) \ - LIBDIR=$(LIBEXTSDIR)/$@ \ - OBJDIR=$(OBJEXTSDIR)/$@ \ - USE_ACCEL="$(DBCSR_USE_ACCEL)" \ - ACC="$(DBCSR_ACC)" \ - ACCFLAGS="$(strip $(DBCSR_ACCFLAGS))" \ - clean diff --git a/exts/README.md b/exts/README.md deleted file mode 100644 index 10d92754eb..0000000000 --- a/exts/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# This directory provides access to external libraries via GIT submodules - -Please refer to the external repositories for any changes: - -* [DBCSR](https://github.com/cp2k/dbcsr) diff --git a/exts/build_dbcsr/Makefile b/exts/build_dbcsr/Makefile deleted file mode 100644 index 0702f450d2..0000000000 --- a/exts/build_dbcsr/Makefile +++ /dev/null @@ -1,335 +0,0 @@ -# This Makefile is only used for the inclusion of the DBCSR library into CP2K. -# It is not supported by DBCSR development anymore. -############################################################ -#### DO NOT CHANGE AS PART OF THE DBCSR DEVELOPMENT!!!! #### -############################################################ -# CP2K team will update it, following the compilation flags suggested in the -# CP2K installation file. -# For this reason, this Makefile must be compatible with the CP2K -# compilation and tested within CP2K. -# -# The ARCH file can set the variables: -# -# CC => C compiler, e.g. gcc or mpicc -# FC => Fortran compiler, e.g. gfortran or mpifort -# LD => Linker, e.g. gfortran or mpifort -# AR => Archive command, e.g. ar -r -# CXXFLAGS => C++ compilation flags -# CFLAGS => C compilation flags -# FCFLAGS => Fortran compilation flags -# LDFLAGS => Linker flags -# LIBS => Libraries -# ACC => ACC can be nvcc (CUDA), mpicxx/mpicc or gcc/g++ (HIP), or mpicc/gcc (OpenCL) -# ACCFLAGS => ACC flags -# USE_ACCEL => hip, cuda, or opencl -# GPUVER => -# - for CUDA, possible values correspond to NVIDIA GPUs: -# possible values are K20X, K40, K80, P100, V100 -# - for HIP, possible values correspond to NVIDIA and AMD GPUs: -# possible values are K20X, K40, K80, P100, V100, Mi50, Mi100, Mi250 -# - for OpenCL, GPUVER maps to a file with tuned parameters: -# src/acc/opencl/smm/params/tune_multiply_GPUVER.csv, or -# src/acc/opencl/smm/tune_multiply.csv (default) -# -# Libraries for accelerator: -# - e.g. for CUDA: LIBS += -lstdc++ -lcudart -lnvrtc -lcuda -lcublas -# - e.g. for HIP: LIBS += -lstdc++ -lhiprtc -lhipblas -# - e.g. for OpenCL: LIBS += -lOpenCL - -# -SHELL = /bin/sh -# -# the home dir is taken from the current directory -# -DBCSRHOME := $(CURDIR) -DBCSRCP2K := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) -MAKEFILE := $(DBCSRCP2K)/Makefile -LIBDIR := $(DBCSRHOME)/lib -OBJDIR := $(DBCSRHOME)/obj -TOOLSDIR := $(DBCSRHOME)/tools -FYPPEXE := $(TOOLSDIR)/build_utils/fypp/bin/fypp -SRCDIR := $(DBCSRHOME)/src -TESTSDIR := $(DBCSRHOME)/tests -ARCHFILE := -CXXSTD := -std=c++14 - -PYTHON := /usr/bin/env python3 - -# Default Target ============================================================ -LIBNAME := dbcsr -LIBRARY := lib$(LIBNAME) -default_target: $(LIBRARY) - -# Read the version ========================================================== -include $(DBCSRHOME)/VERSION -ifeq ($(DATE),) - DATE = "Development Version" -endif - -# Read the configuration ==================================================== -ifeq (,$(ARCHFILE)) -$(error ARCH file must be provided: ARCHFILE=) -endif -ifeq (, $(wildcard $(ARCHFILE))) - $(error ARCH file `$(ARCHFILE)` not found) -endif -include $(ARCHFILE) - -# Set the ARCH version ====================================================== -ifeq ($(GPUVER),K20X) - ARCH_NUMBER = 35 -else ifeq ($(GPUVER),K40) - ARCH_NUMBER = 35 -else ifeq ($(GPUVER),K80) - ARCH_NUMBER = 37 -else ifeq ($(GPUVER),P100) - ARCH_NUMBER = 60 -else ifeq ($(GPUVER),V100) - ARCH_NUMBER = 70 -else ifeq ($(GPUVER),A100) - ARCH_NUMBER = 80 -else ifeq ($(GPUVER),A40) - ARCH_NUMBER = 86 - override GPUVER := A100 -else ifeq ($(GPUVER),H100) - ARCH_NUMBER = 90 -else ifeq ($(GPUVER),Mi50) - ARCH_NUMBER = gfx906 -else ifeq ($(GPUVER),Mi100) - ARCH_NUMBER = gfx908 -else ifeq ($(GPUVER),Mi250) - ARCH_NUMBER = gfx90a -else ifeq (,$(ARCH_NUMBER)) -ifneq ($(ACC),) -ifneq (opencl,$(USE_ACCEL)) - $(error Unknown ARCH_NUMBER since GPUVER="$(GPUVER)" is not recognized) -endif -endif -endif - -# Set ACCFLAGS ============================================================== - -# Adjust C++ standard level -override ACCFLAGS := $(CXXSTD) $(filter-out -std=c++11,$(ACCFLAGS)) - -# If compiling with nvcc -ifeq (cuda,$(USE_ACCEL)) - override ACCFLAGS := -D__CUDA -DARCH_NUMBER=$(ARCH_NUMBER) $(filter-out -D__CUDA,$(ACCFLAGS)) - CXXFLAGS := -D__CUDA $(filter-out -D__CUDA,$(CXXFLAGS)) - FCFLAGS := -D__CUDA $(filter-out -D__CUDA,$(FCFLAGS)) - #if "-arch" has not yet been set in ACCFLAGS - ifeq (,$(findstring -arch,$(ACCFLAGS))) - override ACCFLAGS := -arch sm_$(ARCH_NUMBER) $(ACCFLAGS) - endif - ifeq (,$(findstring -Xcompiler,$(ACCFLAGS))) - override ACCFLAGS := $(ACCFLAGS) -Xcompiler="$(CXXFLAGS)" - endif -# If compiling with hipcc -else ifeq (hip,$(USE_ACCEL)) - override ACCFLAGS := -D__HIP -DARCH_NUMBER=$(ARCH_NUMBER) $(filter-out -D__HIP,$(ACCFLAGS)) - CXXFLAGS := -D__HIP $(filter-out -D__HIP,$(CXXFLAGS)) - FCFLAGS := -D__HIP $(filter-out -D__HIP,$(FCFLAGS)) -# OpenCL backend -else ifeq (opencl,$(USE_ACCEL)) - CFLAGS := -D__OPENCL $(filter-out -D__OPENCL $(CXXSTD),$(if $(ACCFLAGS),$(ACCFLAGS),$(CFLAGS))) -endif - -# Set the configuration ===================================================== -ifneq ($(LD_SHARED),) - ARCHIVE_EXT := .so -else - ARCHIVE_EXT := .a -endif - -# Declare PHONY targets ===================================================== -.PHONY : dirs makedep \ - default_target $(LIBRARY) \ - clean version - -# Discover files and directories ============================================ -ALL_SRC_DIRS := $(shell find $(SRCDIR) -type d | awk '{printf("%s:",$$1)}') -LIBSMM_ACC_DIR := $(shell cd $(SRCDIR) ; find . -type d -name "libsmm_acc") -LIBSMM_ACC_ABS_DIR := $(shell find $(SRCDIR) -type d -name "libsmm_acc") - -ALL_PKG_FILES := $(shell find $(SRCDIR) -name "PACKAGE") -OBJ_SRC_FILES = $(shell cd $(SRCDIR); find . ! -name "dbcsr_api_c.F" ! -name "dbcsr_tensor_api_c.F" -name "*.F") -OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . -type f -name "*.c") - -# if compiling with GPU acceleration -ifneq ($(ACC),) -ifeq (cuda,$(USE_ACCEL)) - # All *.cpp files belong to the accelerator backend common between CUDA/HIP - OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . -type f ! -name "acc_cuda.cpp" ! -name "acc_hip.cpp" -name "*.cpp") - OBJ_SRC_FILES += $(LIBSMM_ACC_DIR)/../cuda/acc_cuda.cpp - # Exclude autotuning files - OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . -type f ! -name "tune_*_exe*_part*.cu" ! -name "tune_*_exe*_main*.cu" -name "*.cu") -else ifeq (hip,$(USE_ACCEL)) - # All *.cpp files belong to the accelerator backend common between CUDA/HIP - OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . -type f ! -name "acc_cuda.cpp" ! -name "acc_hip.cpp" -name "*.cpp") - OBJ_SRC_FILES += $(LIBSMM_ACC_DIR)/../hip/acc_hip.cpp -# OpenCL backend: OBJ_SRC_FILES already includes all *.c files -endif -endif - -# Include also source files which won't compile into an object file -ALL_SRC_FILES = $(strip $(subst $(NULL) .,$(NULL) $(SRCDIR),$(NULL) $(OBJ_SRC_FILES))) -ALL_SRC_FILES += $(shell find $(SRCDIR) -name "*.f90") -ALL_SRC_FILES += $(shell find $(SRCDIR) -name "*.h") -ALL_SRC_FILES += $(shell find $(SRCDIR) -name "*.hpp") - -# stage 1: create dirs and run makedep.py. -# Afterwards, call make recursively again with -C $(OBJDIR) and INCLUDE_DEPS=true -ifeq ($(INCLUDE_DEPS),) -$(LIBRARY): dirs makedep - @+$(MAKE) --no-print-directory -C $(OBJDIR) -f $(MAKEFILE) $(LIBDIR)/$(LIBRARY)$(ARCHIVE_EXT) INCLUDE_DEPS=true DBCSRHOME=$(DBCSRHOME) -dirs: - @mkdir -p $(OBJDIR) - @mkdir -p $(LIBDIR) -version: - @echo "DBCSR Version: "$(MAJOR)"."$(MINOR)"."$(PATCH)" ("$(DATE)")" -else -# stage 2: Include $(OBJDIR)/all.dep, expand target all, and get list of dependencies. - -# Check if FYPP is available =============================================== -ifeq (, $(shell which $(FYPPEXE) 2>/dev/null )) - $(error No FYPP submodule available, please read README.md on how to properly download DBCSR) -endif - -endif - -clean: - rm -f $(LIBSMM_ACC_ABS_DIR)/parameters.h $(LIBSMM_ACC_ABS_DIR)/smm_acc_kernels.h $(LIBSMM_ACC_ABS_DIR)/*.so - rm -f $(LIBSMM_ACC_ABS_DIR)/../opencl/smm/opencl_kernels.h - rm -f $(TESTSDIR)/libsmm_acc_unittest_multiply.cpp - rm -f $(TESTSDIR)/libsmm_acc_timer_multiply.cpp - rm -rf $(OBJDIR) - -# Libsmm_acc stuff ========================================================== -ifneq ($(ACC),) -ifneq (opencl,$(USE_ACCEL)) -ACC_KERNEL := $(wildcard $(LIBSMM_ACC_ABS_DIR)/kernels/*.h) -ACC_PARAMS := $(wildcard $(LIBSMM_ACC_ABS_DIR)/parameters/parameters_$(GPUVER).json) -$(LIBSMM_ACC_ABS_DIR)/parameters.h: $(LIBSMM_ACC_ABS_DIR)/generate_parameters.py $(ACC_PARAMS) - cd $(LIBSMM_ACC_ABS_DIR); $(PYTHON) generate_parameters.py --gpu_version=$(GPUVER) -$(LIBSMM_ACC_ABS_DIR)/smm_acc_kernels.h: $(LIBSMM_ACC_ABS_DIR)/generate_kernels.py $(ACC_KERNEL) - cd $(LIBSMM_ACC_ABS_DIR); $(PYTHON) generate_kernels.py -endif -endif - -# automatic dependency generation =========================================== -MODDEPS = "lower" -MAKEDEPMODE = "normal" -ifeq ($(HACKDEP),yes) -MAKEDEPMODE = "hackdep" -endif - -# this happens at stage 1 -makedep: $(ALL_SRC_FILES) $(ALL_PKG_FILES) dirs -ifeq ($(LD_SHARED),) - @echo "Removing stale archives ... " - @$(PYTHON) $(DBCSRCP2K)/check_archives.py $(firstword $(AR)) $(SRCDIR) $(LIBDIR) -endif - @echo "Resolving dependencies ... " - @$(PYTHON) $(DBCSRCP2K)/makedep.py $(OBJDIR)/all.dep dbcsr $(MODDEPS) $(MAKEDEPMODE) $(ARCHIVE_EXT) $(SRCDIR) $(OBJ_SRC_FILES) - -# at stage 2, load the rules generated by makedep.py -ifeq ($(INCLUDE_DEPS), true) - include $(OBJDIR)/all.dep -endif - - -# ================= Stuff need for compiling (stage 2) ====================== -# These rules are executed in a recursive call to make -C $(OBJDIR) -# The change of $(CURDIR) allows to find targets without abs paths and vpaths. - - -### Slave rules ### -vpath %.F $(ALL_SRC_DIRS) -vpath %.h $(ALL_SRC_DIRS) -vpath %.hpp $(ALL_SRC_DIRS) -vpath %.f90 $(ALL_SRC_DIRS) -vpath %.cu $(ALL_SRC_DIRS) -vpath %.c $(ALL_SRC_DIRS) -vpath %.cpp $(ALL_SRC_DIRS) - -# $(FCLOGPIPE) can be used to store compiler output, e.g. warnings, for each F-file separately. -# This is used e.g. by the convention checker. - -FYPPFLAGS ?= -n - -%.o: %.F - $(PYTHON) $(FYPPEXE) $(FYPPFLAGS) $< $*.F90 - $(FC) -c $(FCFLAGS) -D__SHORT_FILE__="\"$(notdir $<)\"" -I'$(dir $<)' -I'$(SRCDIR)' $*.F90 $(FCLOGPIPE) - -%.mod: %.o - @true - -%.o: %.c - $(CC) -c $(CFLAGS) $< - -ifneq ($(ACC),) -# if compiling CUDA/HIP backend -ifneq (opencl,$(USE_ACCEL)) -%.o: %.cpp parameters.h smm_acc_kernels.h - $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $< -# if compiling CUDA backend -ifeq (cuda,$(USE_ACCEL)) -%.o: %.cu parameters.h smm_acc_kernels.h - $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $< -calculate_norms.o: calculate_norms.cpp - $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' -x cu $< -acc_cuda.o: acc_cuda.cpp acc_cuda.h - $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $< -# if compiling HIP backend -else -calculate_norms.o: calculate_norms.cpp - $(OFFLOAD_CC) -D__HIP -c $(OFFLOAD_FLAGS) -I'$(SRCDIR)' $< -acc_hip.o: acc_hip.cpp acc_hip.h - $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $< -endif - -# if compiling OpenCL backend -else -OPENCL_KRNLGEN := $(LIBSMM_ACC_ABS_DIR)/../opencl/acc_opencl.sh -OPENCL_KERNELS := $(wildcard $(LIBSMM_ACC_ABS_DIR)/../opencl/smm/kernels/*.cl) -OPENCL_DEFAULT := $(wildcard $(LIBSMM_ACC_ABS_DIR)/../opencl/smm/tune_multiply.csv) -OPENCL_WITHGPU := $(wildcard $(LIBSMM_ACC_ABS_DIR)/../opencl/smm/params/tune_multiply_*.csv) -OPENCL_PARAMS := $(if $(OPENCL_DEFAULT),$(OPENCL_DEFAULT),$(OPENCL_WITHGPU)) -OPENCL_COMMON := $(wildcard $(LIBSMM_ACC_ABS_DIR)/../opencl/common/*.h) -$(LIBSMM_ACC_ABS_DIR)/../opencl/smm/opencl_kernels.h: $(OPENCL_KRNLGEN) $(OPENCL_KERNELS) $(OPENCL_COMMON) $(OPENCL_PARAMS) - $(OPENCL_KRNLGEN) $(OPENCL_KERNELS) $(OPENCL_PARAMS) $@ -opencl_libsmm.o: opencl_libsmm.c $(LIBSMM_ACC_ABS_DIR)/../opencl/smm/opencl_kernels.h -ifeq (Darwin,$(shell uname)) - LDFLAGS += -framework OpenCL -else - # OpenCL include directory (cl.h not installed per "opencl-headers" package) - NVCC := $(shell which nvcc 2>/dev/null) - NVCC_PATH := $(if $(NVCC),$(realpath $(dir $(NVCC))/..)) - CUDA_FILE := $(wildcard $(NVCC_PATH)/../cuda/include/cuda.h) - CUDA_PATH := $(if $(CUDA_FILE),$(abspath $(NVCC_PATH)/../cuda),$(NVCC_PATH)) - CUDA_LIBS := $(if $(wildcard $(CUDA_PATH)/lib64),lib64,lib) - ifneq (,$(CUDA_PATH)) - CFLAGS += -I$(CUDA_PATH)/include - else ifneq (,$(wildcard $(OPENCL_ROOT)/include/CL/cl.h)) - CFLAGS += -I$(OPENCL_ROOT)/include - else - OPENCL_ROOT := $(abspath $(dir $(shell which icx 2>/dev/null))/..) - CLINC := $(wildcard $(OPENCL_ROOT)/include/sycl/CL/cl.h $(OPENCL_ROOT)/include/CL/cl.h) - ifneq (,$(CLINC)) - CFLAGS += -I$(abspath $(dir $(firstword $(CLINC)))/..) - endif - endif -endif -endif -endif - -$(LIBDIR)/%: -ifneq ($(LD_SHARED),) - @echo "Creating shared library $@" - @$(LD_SHARED) $(LDFLAGS) -o $(@:.a=.so) $^ $(LIBS) -else - @echo "Updating archive $@" - @$(AR) $@ $? -endif - -#EOF diff --git a/exts/build_dbcsr/check_archives.py b/exts/build_dbcsr/check_archives.py deleted file mode 100755 index 8444514ee1..0000000000 --- a/exts/build_dbcsr/check_archives.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Check that a static archive contains only the objects specified in the PACKAGES files. -""" - -# author: Ole Schuett - - -import subprocess -import os -from os import path -import argparse - -KNOWN_EXTENSIONS = ("F", "c", "cu", "cpp", "cxx", "cc") - - -def main(ar_exe, src_dir, lib_dir): - # Search all files belonging to a given archive - archives_files = {} - - for root, _, files in os.walk(src_dir): - if "PACKAGE" in files: - with open(path.join(root, "PACKAGE")) as fhandle: - content = fhandle.read() - - package = eval(content) - - archive = "libdbcsr" + path.basename(root) - if "archive" in package.keys(): - archive = package["archive"] - - file_parts = [fn.rsplit(".", 1) for fn in files] - src_basenames = [ - parts[0] for parts in file_parts if parts[-1] in KNOWN_EXTENSIONS - ] - - if archive in archives_files: - archives_files[archive] |= set(src_basenames) - else: - archives_files[archive] = set(src_basenames) - - # Check if the symbols in each archive have a corresponding source file - for archive in archives_files: - archive_fn = path.join(lib_dir, archive + ".a") - - if not path.exists(archive_fn): - continue - - output = subprocess.check_output([ar_exe, "t", archive_fn]) - for line in output.decode("utf8").strip().splitlines(): - if line == "__.SYMDEF SORTED": - continue # needed for MacOS - - assert line.endswith( - ".o" - ), "discovered a non-object file inside a static archive" - - if line[:-2] not in archives_files[archive]: - print( - "Could not find source for object '{}' in archive '{}', removing archive.".format( - line, archive_fn - ) - ) - os.remove(archive_fn) - break - - -# =============================================================================== -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=""" - Parse files and package manifests in the source tree to create rules for objects and executables - - This script is part of the build utility scripts for DBCSR. - """, - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument("ar_executable", metavar="ar_executable", type=str) - parser.add_argument("src_dir", metavar="src_dir", type=str) - parser.add_argument("lib_dir", metavar="lib_dir", type=str) - - args = parser.parse_args() - main(ar_exe=args.ar_executable, src_dir=args.src_dir, lib_dir=args.lib_dir) diff --git a/exts/build_dbcsr/makedep.py b/exts/build_dbcsr/makedep.py deleted file mode 100755 index c2e3c9f4ac..0000000000 --- a/exts/build_dbcsr/makedep.py +++ /dev/null @@ -1,441 +0,0 @@ -#!/usr/bin/env python3 - -import re -import sys -import ast -from os import path -from os.path import dirname, basename, normpath -import glob -import argparse - -# pre-compiled regular expressions -re_module = re.compile(r"(?:^|\n)\s*module\s+(\w+)\s.*\n\s*end\s*module", re.DOTALL) -re_program = re.compile(r"\n\s*end\s*program") -re_main = re.compile(r"\sint\s+main\s*\(") -re_use = re.compile(r"\n\s*use\s+(\w+)") -re_incl_fypp = re.compile(r"\n#:include\s+['\"](.+)['\"]") -re_incl_cpp = re.compile(r"\n#include\s+['\"](.+)['\"]") -re_incl_fort = re.compile(r"\n\s*include\s+['\"](.+)['\"]") - - -# ============================================================================ -def main(out_fn, project_name, mod_format, mode, archive_ext, src_dir, src_files): - messages = [] - # process arguments - src_files = [normpath(path.join(src_dir, f)) for f in src_files] - - if mod_format not in ("lower", "upper", "no"): - error('Module filename format must be either of "lower", "upper", or "no".') - - if mode not in ("normal", "hackdep", "mod_compiler"): - error('Mode must be either of "normal", "hackdep", or "mod_compiler".') - - for fn in src_files: - if not fn.startswith("/"): - error("Path of source-file not absolute: " + fn) - - src_basenames = [basename(fn).rsplit(".", 1)[0] for fn in src_files] - for bfn in src_basenames: - if src_basenames.count(bfn) > 1: - error("Multiple source files with the same basename: " + bfn) - - # parse files - parsed_files = dict() - for fn in src_files: - parse_file(parsed_files, fn, src_dir) # parses also included files - messages.append("Parsed %d files" % len(parsed_files)) - - # create table mapping fortran module-names to file-name - mod2fn = dict() - for fn in src_files: - for m in parsed_files[fn]["module"]: - if m in mod2fn.keys(): - error('Multiple declarations of module "%s"' % m) - mod2fn[m] = fn - messages.append("Created mod2fn table, found %d modules." % len(mod2fn)) - - # check "one module per file"-convention - for m, fn in mod2fn.items(): - if basename(fn) != m + ".F": - error("Names of module and file do not match: " + fn) - - # read package manifests - packages = dict() - for fn in src_files: - p = normpath(dirname(fn)) - read_pkg_manifest(project_name, packages, p) - messages.append("Read %d package manifests" % len(packages)) - - # check dependencies against package manifests - n_deps = 0 - for fn in src_files: - p = normpath(dirname(fn)) - if not parsed_files[fn]["program"]: - packages[p]["objects"].append(src2obj(basename(fn))) - deps = collect_include_deps(parsed_files, fn, src_dir) - deps += [ - mod2fn[m] - for m in collect_use_deps(parsed_files, fn, src_dir) - if m in mod2fn.keys() - ] - n_deps += len(deps) - for d in deps: - dp = normpath(dirname(d)) - if dp not in packages[p]["allowed_deps"]: - error( - "Dependency forbidden according to package manifest: %s -> %s" - % (fn, d) - ) - if dp != p and "public_files" in packages[dp].keys(): - if basename(d) not in packages[dp]["public_files"]: - error( - "File not public according to package manifest: %s -> %s" - % (fn, d) - ) - messages.append("Checked %d dependencies" % n_deps) - - # check for circular dependencies - for fn in parsed_files.keys(): - find_cycles(parsed_files, mod2fn, fn, src_dir) - - # write messages as comments - makefile = "\n".join("#makedep: {}".format(m) for m in messages) - makefile += "\n\n" - - # write rules for archives - for pkg in packages.keys(): - if packages[pkg]["objects"]: - makefile += """\ -# Package {pkg} -$(LIBDIR)/{archive}{ext} : {objs} - -""".format( - pkg=pkg, - archive=packages[pkg]["archive"], - ext=archive_ext, - objs=" ".join(packages[pkg]["objects"]), - ) - - # write rules for public files - for pkg in packages.keys(): - if "public" in packages[pkg].keys(): - makefile += """\ -# Public modules for package {pkg} -install: PUBLICFILES += {pubfiles} - -""".format( - pkg=pkg, pubfiles=" ".join(mod for mod in packages[pkg]["public"]) - ) - - # write rules for executables - archive_postfix = archive_ext.rsplit(".", 1)[0] - for fn in src_files: - if not parsed_files[fn]["program"]: - continue - - bfn = basename(fn).rsplit(".", 1)[0] - p = normpath(dirname(fn)) - - deps = collect_pkg_deps(packages, p) - assert all(a.startswith("lib") for a in deps) - cflagsvar = " $(LDFLAGS_C)" if fn.endswith(".c") or fn.endswith(".cu") else "" - makefile += """\ -# Program {fn} -$(EXEDIR)/{bfn}.$(ONEVERSION) : {bfn}.o {deps} -\t$(LD) $(LDFLAGS) {cflagsvar} -L$(LIBDIR) -o $@ {bfn}.o $(EXTERNAL_OBJECTS) {linkerdeps} $(LIBS) - -""".format( - fn=fn, - bfn=bfn, - deps=" ".join(["$(LIBDIR)/" + a + archive_ext for a in deps]), - cflagsvar=cflagsvar, - linkerdeps=" ".join("-l{}{}".format(a[3:], archive_postfix) for a in deps), - ) - - # write rules for objects - for fn in src_files: - deps = collect_include_deps(parsed_files, fn, src_dir) - - mods = collect_use_deps(parsed_files, fn, src_dir) - mods.sort(key=cmp_mods) # sort mods to speedup compilation - deps += [mod2modfile(m, mod_format) for m in mods if m in mod2fn.keys()] - - if mode == "hackdep": - deps = [] - - deps = " ".join(deps) - - bfn = basename(fn) - provides = [mod2modfile(m, mod_format) for m in parsed_files[fn]["module"]] - - makefile += "# Object {bfn}\n".format(bfn=bfn) - for mfn in provides: - makefile += "{mfn} : {bfn} {deps}\n".format(mfn=mfn, bfn=bfn, deps=deps) - - makefile += "{bfnobj} : {bfn} {deps}".format( - bfnobj=src2obj(bfn), bfn=bfn, deps=deps - ) - - if mode == "mod_compiler": - makefile += " " + " ".join(provides) - - makefile += "\n\n" - - with open(out_fn, "w", encoding="utf8") as fhandle: - fhandle.write(makefile) - - -# ============================================================================ -def cmp_mods(mod): - # list "type" modules first, they are probably on the critical path - if "type" in mod: - return 0 - return 1 - - -# ============================================================================ -def parse_file(parsed_files, fn, src_dir): - if fn in parsed_files: - return - - with open(fn, encoding="utf8") as fhandle: - content = fhandle.read() - - # re.IGNORECASE is horribly expensive. Converting to lower-case upfront - content_lower = content.lower() - - # all files are parsed for cpp includes - incls = re_incl_cpp.findall(content) # CPP includes (case-sensitive) - - mods = [] - uses = [] - prog = False - - if fn[-2:] == ".F" or fn[-4:] == ".f90" or fn[-5:] == ".fypp": - mods += re_module.findall(content_lower) - prog = True if re_program.search(content_lower) is not None else False - uses += re_use.findall(content_lower) - incls += re_incl_fypp.findall(content) # Fypp includes (case-sensitive) - incl_fort_iter = re_incl_fort.finditer(content_lower) # fortran includes - incls += [content[m.start(1) : m.end(1)] for m in incl_fort_iter] - - if fn[-2:] == ".c" or fn[-3:] == ".cu": - prog = ( - True if re_main.search(content) is not None else False - ) # C is case-sensitive - - # exclude included files from outside the source tree - def incl_fn(i): - return normpath(path.join(dirname(fn), i)) - - def incl_fn_src(i): - return normpath(path.join(src_dir, i)) - - existing_incl = [i for i in incls if path.exists(incl_fn(i))] - existing_incl_src = [i for i in incls if path.exists(incl_fn_src(i))] - - # store everything in parsed_files cache - parsed_files[fn] = { - "module": mods, - "program": prog, - "use": uses, - "include": existing_incl, - "include_src": existing_incl_src, - } - - # parse included files - for i in existing_incl: - parse_file(parsed_files, incl_fn(i), src_dir) - - for i in existing_incl_src: - parse_file(parsed_files, incl_fn_src(i), src_dir) - - -# ============================================================================ -def read_pkg_manifest(project_name, packages, p): - if p in packages.keys(): - return - - fn = path.join(p, "PACKAGE") - if not path.exists(fn): - error("Could not open PACKAGE manifest: " + fn) - - with open(fn, encoding="utf8") as fhandle: - content = fhandle.read() - - packages[p] = ast.literal_eval(content) - packages[p]["objects"] = [] - if "archive" not in packages[p].keys(): - packages[p]["archive"] = "lib{}{}".format(project_name, basename(p)) - packages[p]["allowed_deps"] = [normpath(p)] - packages[p]["allowed_deps"] += [ - normpath(path.join(p, r)) for r in packages[p]["requires"] - ] - - for r in packages[p]["requires"]: - read_pkg_manifest(project_name, packages, normpath(path.join(p, r))) - - if "public" in packages[p].keys(): - public_files = [] - for fn in packages[p]["public"]: - public_files += glob.glob(path.join(p, fn)) - packages[p]["public_files"] = [basename(fn) for fn in public_files] - - -# ============================================================================ -def mod2modfile(m, mod_format): - if mod_format == "no": - return "" - - if mod_format == "lower": - return m.lower() + ".mod" - - if mod_format == "upper": - return m.upper() + ".mod" - - assert False # modeps unknown - - -# ============================================================================ -def src2obj(src_fn): - return basename(src_fn).rsplit(".", 1)[0] + ".o" - - -# ============================================================================ -def collect_include_deps(parsed_files, fn, src_dir): - pf = parsed_files[fn] - incs = [] - - for i in pf["include"]: - fn_inc = normpath(path.join(dirname(fn), i)) - if fn_inc in parsed_files.keys(): - incs.append(fn_inc) - incs += collect_include_deps(parsed_files, fn_inc, src_dir) - - for i in pf["include_src"]: - fn_inc = normpath(path.join(src_dir, i)) - if fn_inc in parsed_files.keys(): - incs.append(fn_inc) - incs += collect_include_deps(parsed_files, fn_inc, src_dir) - - return sorted(set(incs)) - - -# ============================================================================ -def collect_use_deps(parsed_files, fn, src_dir): - pf = parsed_files[fn] - uses = pf["use"] - - for i in pf["include"]: - fn_inc = normpath(path.join(dirname(fn), i)) - if fn_inc in parsed_files.keys(): - uses += collect_use_deps(parsed_files, fn_inc, src_dir) - - for i in pf["include_src"]: - fn_inc = normpath(path.join(src_dir, i)) - if fn_inc in parsed_files.keys(): - uses += collect_use_deps(parsed_files, fn_inc, src_dir) - - return sorted(set(uses)) - - -# ============================================================================ -def find_cycles(parsed_files, mod2fn, fn, src_dir, S=None): - pf = parsed_files[fn] - if "visited" in pf.keys(): - return - - if S is None: - S = [] - - for m in pf["module"]: - if m in S: - i = S.index(m) - n = S[i:] - else: - n = [] - - if 0 == len(n) or (1 == len(n) and m in n): - S.append(m) - else: - error("Circular dependency: " + " -> ".join(n + [m])) - - for m in collect_use_deps(parsed_files, fn, src_dir): - if m in mod2fn.keys(): - find_cycles(parsed_files, mod2fn, mod2fn[m], src_dir, S) - - for m in pf["module"]: - S.pop() - - pf["visited"] = True - - -# ============================================================================ -def collect_pkg_deps(packages, p, archives=None, S=None): - if archives is None: - archives = [] - - if S is None: - S = [] - - a = packages[p]["archive"] - if a in archives: - return archives - - if a in S: - i = S.index(a) - b = S[i:] - else: - b = [] - - if 0 == len(b) or (1 == len(b) and a in b): - S.append(a) - else: - error("Circular package dependency: " + " -> ".join(b + [a])) - - for r in packages[p]["requires"]: - d = normpath(path.join(p, r)) - collect_pkg_deps(packages, d, archives, S) - - S.pop() - - if packages[p]["objects"]: - archives.insert(0, packages[p]["archive"]) - - return archives - - -# ============================================================================ -def error(msg): - sys.stderr.write("makedep error: %s\n" % msg) - sys.exit(1) - - -# ============================================================================ -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=""" - Parse files and package manifests in the source tree to create rules for objects and executables - - This script is part of the build utility scripts for DBCSR. - """, - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument("outfile", metavar="outfile", type=str) - parser.add_argument("project_name", metavar="project_name", type=str) - parser.add_argument("format", metavar="format", type=str) - parser.add_argument("mode", metavar="mode", type=str) - parser.add_argument("archive_ext", metavar="archive_ext", type=str) - parser.add_argument("src_dir", metavar="src_dir", type=str) - parser.add_argument("src_file", metavar="src_file", nargs="+", type=str) - - args = parser.parse_args() - main( - out_fn=args.outfile, - project_name=args.project_name, - mod_format=args.format, - mode=args.mode, - archive_ext=args.archive_ext, - src_dir=args.src_dir, - src_files=args.src_file, - ) diff --git a/exts/build_dbcsr/test_makedep.py b/exts/build_dbcsr/test_makedep.py deleted file mode 100755 index 7e76e72769..0000000000 --- a/exts/build_dbcsr/test_makedep.py +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Unittests for the makedep.py script -""" - -import unittest -import tempfile -import shutil -import json -import io -import os -from os import path - -import makedep - - -def _mkpkg(content, base_dir): - """ - Create a PACKAGE file in the given directory. - If the content is a list or a dict, serialize the structure using json.dump. - """ - - with open(path.join(base_dir, "PACKAGE"), "w") as fhandle: - if isinstance(content, (dict, list)): - json.dump(content, fhandle, ensure_ascii=False) - else: - fhandle.write(content) - - -class TestCheckArchives(unittest.TestCase): - """Test case for the main method in makedep""" - - def setUp(self): - self.base_dir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.base_dir) - - def test_empty_dir(self): - """ - running on an empty dir with no files generates an empty deps file - """ - my_dir = path.join(self.base_dir, "empty") - out_fn = path.join(my_dir, "all.dep") - - os.mkdir(my_dir) - - makedep.main(out_fn, "empty_project", "lower", "normal", ".a", my_dir, []) - - with open(out_fn, "r") as fhandle: - no_comment_lines = [ - l.strip() for l in fhandle if not l.startswith("#") and l.strip() - ] - - self.assertEqual(len(no_comment_lines), 0) - - def test_single_empty(self): - """ - running on a dir with a single file - """ - - my_dir = path.join(self.base_dir, "single_empty") - out_fn = path.join(my_dir, "all.dep") - single_fn = path.join(my_dir, "single.F") - - os.mkdir(my_dir) - with open(single_fn, "w") as fhandle: - fhandle.write("\n") - - with self.assertRaises(SystemExit): - # should throw an exception due to missing PACKAGES - makedep.main( - out_fn, - "faulty_project", - "lower", - "normal", - ".a", - my_dir, - ["./single.F"], - ) - - _mkpkg( - { - "description": "Nothing", - "archive": "test", - "public": ["*.F"], - "requires": [], - }, - my_dir, - ) - - makedep.main( - out_fn, "single_empty", "lower", "normal", ".a", my_dir, ["./single.F"] - ) - - with open(out_fn, "r") as fhandle: - no_comment_lines = [ - l.strip() for l in fhandle if not l.startswith("#") and l.strip() - ] - - self.assertEqual( - no_comment_lines, - [ - "$(LIBDIR)/test.a : single.o", - "install: PUBLICFILES += *.F", - "single.o : single.F", - ], - ) - - def test_unicode(self): - """ - running on a dir with a single file - """ - - my_dir = path.join(self.base_dir, "unicode") - out_fn = path.join(my_dir, "all.dep") - single_fn = path.join(my_dir, "single.F") - - os.mkdir(my_dir) - with io.open(single_fn, "w", encoding="utf8") as fhandle: - fhandle.write(u"! Ångström\n") - - _mkpkg( - """{ - "description": "unicode test with just an Ångström 😉", - "archive": "test", - "public": ["*.F"], - "requires": [] -}""", - my_dir, - ) - - makedep.main( - out_fn, "unicode_project", "lower", "normal", ".a", my_dir, ["./single.F"] - ) - - with open(out_fn, "r") as fhandle: - no_comment_lines = [ - l.strip() for l in fhandle if not l.startswith("#") and l.strip() - ] - - self.assertEqual( - no_comment_lines, - [ - "$(LIBDIR)/test.a : single.o", - "install: PUBLICFILES += *.F", - "single.o : single.F", - ], - ) - - def test_subpackage(self): - """ - running on a dir with a single file - """ - - my_dir = path.join(self.base_dir, "subpackage") - out_fn = path.join(my_dir, "all.dep") - sub_dir = path.join(my_dir, "sub") - single_fn = path.join(my_dir, "single.F") - sub_fn = path.join(sub_dir, "sub.F") - - os.mkdir(my_dir) - os.mkdir(sub_dir) - - for fname in [single_fn, sub_fn]: - with open(fname, "w") as fhandle: - fhandle.write("") - - _mkpkg( - """{ - "description": "test with a simple subpackage", - "archive": "test", - "public": ["*.F"], - "requires": ["./sub"] -}""", - my_dir, - ) - - _mkpkg( - """{ - "description": "the sub package", - "requires": [""] -}""", - sub_dir, - ) - - makedep.main( - out_fn, - "sub_pkg_without_dep", - "lower", - "normal", - ".a", - my_dir, - ["./single.F"], - ) - - with open(out_fn, "r") as fhandle: - no_comment_lines = [ - l.strip() for l in fhandle if not l.startswith("#") and l.strip() - ] - - self.assertEqual( - no_comment_lines, - [ - "$(LIBDIR)/test.a : single.o", - "install: PUBLICFILES += *.F", - "single.o : single.F", - ], - ) - - with open(single_fn, "w") as fhandle: - fhandle.write( - """module single - use :: sub -end module""" - ) - - with open(sub_fn, "w") as fhandle: - fhandle.write( - """module sub -end module""" - ) - - makedep.main( - out_fn, - "sub_pkg_with_dep", - "lower", - "normal", - ".a", - my_dir, - ["./single.F", "./sub/sub.F"], - ) - - with open(out_fn, "r") as fhandle: - no_comment_lines = [ - l.strip() for l in fhandle if not l.startswith("#") and l.strip() - ] - - self.assertEqual( - sorted(no_comment_lines), - [ - "$(LIBDIR)/libsub_pkg_with_depsub.a : sub.o", - "$(LIBDIR)/test.a : single.o", - "install: PUBLICFILES += *.F", - "single.mod : single.F", - "single.o : single.F", - "sub.o : sub.F", - ], - ) - - -# ============================================================================ -if __name__ == "__main__": - unittest.main() diff --git a/exts/dbcsr b/exts/dbcsr deleted file mode 160000 index 2cdbf1331e..0000000000 --- a/exts/dbcsr +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2cdbf1331eb9f0e8265f4ca048cedb125f3e38f9