Remove DBCSR submodule

This commit is contained in:
Ole Schütt 2025-12-18 15:01:07 +01:00 committed by Ole Schütt
parent e525afefe5
commit c8cf32602f
8 changed files with 0 additions and 1188 deletions

4
.gitmodules vendored
View file

@ -1,4 +0,0 @@
[submodule "exts/dbcsr"]
path = exts/dbcsr
url = https://github.com/cp2k/dbcsr
branch = master

View file

@ -1,63 +0,0 @@
ifeq ("","$(wildcard $(EXTSHOME)/dbcsr/src)")
$(error "No DBCSR submodule available, please run 'git submodule update --init --recursive'")
endif
LIBS := -L$(LIBEXTSDIR)/dbcsr -ldbcsr $(LIBS)
EXTSDEPS_LIB = $(LIBEXTSDIR)/dbcsr/libdbcsr$(ARCHIVE_EXT)
EXTSDEPS_MOD = $(OBJEXTSDIR)/dbcsr/dbcsr_api.mod $(OBJEXTSDIR)/dbcsr/dbcsr_tensor_api.mod
$(EXTSDEPS_MOD) : ; # override builtin .mod rule to prevent circular dependency
# Set Acceleration flags if glue code is supposed to be accelerated
ifneq (,$(findstring __DBCSR_ACC,$(OFFLOAD_FLAGS) $(FCFLAGS)))
# By default, we use CUDA if OFFLOAD_TARGET is not specified
ifeq (,$(OFFLOAD_TARGET))
DBCSR_USE_ACCEL = cuda
else
DBCSR_USE_ACCEL = $(OFFLOAD_TARGET)
endif
ifeq (cuda,$(DBCSR_USE_ACCEL))
DBCSR_ACC := $(OFFLOAD_CC)
DBCSR_ACCFLAGS := $(OFFLOAD_FLAGS) -Xcompiler='-Wno-error'
else ifeq (hip,$(DBCSR_USE_ACCEL))
DBCSR_ACC := $(CXX)
DBCSR_ACCFLAGS := $(CXXFLAGS) -Wno-error=unused-parameter
else ifeq (opencl,$(DBCSR_USE_ACCEL))
DBCSR_ACC := $(CC)
DBCSR_ACCFLAGS := $(CFLAGS)
else
$(error "Invalid OFFLOAD_TARGET=$(OFFLOAD_TARGET)")
endif
endif
extversions: dbcsrversion
extclean: dbcsrclean
@echo "Clean EXT"
dbcsr:
+$(MAKE) -C $(EXTSHOME)/$@ -f $(EXTSHOME)/build_$@/Makefile \
ARCHFILE=$(ARCHDIR)/$(ARCH).$(ONEVERSION) \
LIBDIR=$(LIBEXTSDIR)/$@ \
OBJDIR=$(OBJEXTSDIR)/$@ \
USE_ACCEL="$(DBCSR_USE_ACCEL)" \
ACC="$(DBCSR_ACC)" \
ACCFLAGS="$(strip $(DBCSR_ACCFLAGS))"
dbcsrversion:
@$(MAKE) -C $(EXTSHOME)/dbcsr -f $(EXTSHOME)/build_dbcsr/Makefile \
ARCHFILE=$(ARCHDIR)/$(ARCH).$(ONEVERSION) \
USE_ACCEL="$(DBCSR_USE_ACCEL)" \
ACC="$(DBCSR_ACC)" \
ACCFLAGS="$(strip $(DBCSR_ACCFLAGS))" \
version
dbcsrclean:
@echo "Clean DBCSR"
@$(MAKE) -C $(EXTSHOME)/dbcsr -f $(EXTSHOME)/build_dbcsr/Makefile \
ARCHFILE=$(ARCHDIR)/$(ARCH).$(ONEVERSION) \
LIBDIR=$(LIBEXTSDIR)/$@ \
OBJDIR=$(OBJEXTSDIR)/$@ \
USE_ACCEL="$(DBCSR_USE_ACCEL)" \
ACC="$(DBCSR_ACC)" \
ACCFLAGS="$(strip $(DBCSR_ACCFLAGS))" \
clean

View file

@ -1,5 +0,0 @@
# This directory provides access to external libraries via GIT submodules
Please refer to the external repositories for any changes:
* [DBCSR](https://github.com/cp2k/dbcsr)

View file

@ -1,335 +0,0 @@
# This Makefile is only used for the inclusion of the DBCSR library into CP2K.
# It is not supported by DBCSR development anymore.
############################################################
#### DO NOT CHANGE AS PART OF THE DBCSR DEVELOPMENT!!!! ####
############################################################
# CP2K team will update it, following the compilation flags suggested in the
# CP2K installation file.
# For this reason, this Makefile must be compatible with the CP2K
# compilation and tested within CP2K.
#
# The ARCH file can set the variables:
#
# CC => C compiler, e.g. gcc or mpicc
# FC => Fortran compiler, e.g. gfortran or mpifort
# LD => Linker, e.g. gfortran or mpifort
# AR => Archive command, e.g. ar -r
# CXXFLAGS => C++ compilation flags
# CFLAGS => C compilation flags
# FCFLAGS => Fortran compilation flags
# LDFLAGS => Linker flags
# LIBS => Libraries
# ACC => ACC can be nvcc (CUDA), mpicxx/mpicc or gcc/g++ (HIP), or mpicc/gcc (OpenCL)
# ACCFLAGS => ACC flags
# USE_ACCEL => hip, cuda, or opencl
# GPUVER =>
# - for CUDA, possible values correspond to NVIDIA GPUs:
# possible values are K20X, K40, K80, P100, V100
# - for HIP, possible values correspond to NVIDIA and AMD GPUs:
# possible values are K20X, K40, K80, P100, V100, Mi50, Mi100, Mi250
# - for OpenCL, GPUVER maps to a file with tuned parameters:
# src/acc/opencl/smm/params/tune_multiply_GPUVER.csv, or
# src/acc/opencl/smm/tune_multiply.csv (default)
#
# Libraries for accelerator:
# - e.g. for CUDA: LIBS += -lstdc++ -lcudart -lnvrtc -lcuda -lcublas
# - e.g. for HIP: LIBS += -lstdc++ -lhiprtc -lhipblas
# - e.g. for OpenCL: LIBS += -lOpenCL
#
SHELL = /bin/sh
#
# the home dir is taken from the current directory
#
DBCSRHOME := $(CURDIR)
DBCSRCP2K := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
MAKEFILE := $(DBCSRCP2K)/Makefile
LIBDIR := $(DBCSRHOME)/lib
OBJDIR := $(DBCSRHOME)/obj
TOOLSDIR := $(DBCSRHOME)/tools
FYPPEXE := $(TOOLSDIR)/build_utils/fypp/bin/fypp
SRCDIR := $(DBCSRHOME)/src
TESTSDIR := $(DBCSRHOME)/tests
ARCHFILE :=
CXXSTD := -std=c++14
PYTHON := /usr/bin/env python3
# Default Target ============================================================
LIBNAME := dbcsr
LIBRARY := lib$(LIBNAME)
default_target: $(LIBRARY)
# Read the version ==========================================================
include $(DBCSRHOME)/VERSION
ifeq ($(DATE),)
DATE = "Development Version"
endif
# Read the configuration ====================================================
ifeq (,$(ARCHFILE))
$(error ARCH file must be provided: ARCHFILE=<CP2K ARCH file>)
endif
ifeq (, $(wildcard $(ARCHFILE)))
$(error ARCH file `$(ARCHFILE)` not found)
endif
include $(ARCHFILE)
# Set the ARCH version ======================================================
ifeq ($(GPUVER),K20X)
ARCH_NUMBER = 35
else ifeq ($(GPUVER),K40)
ARCH_NUMBER = 35
else ifeq ($(GPUVER),K80)
ARCH_NUMBER = 37
else ifeq ($(GPUVER),P100)
ARCH_NUMBER = 60
else ifeq ($(GPUVER),V100)
ARCH_NUMBER = 70
else ifeq ($(GPUVER),A100)
ARCH_NUMBER = 80
else ifeq ($(GPUVER),A40)
ARCH_NUMBER = 86
override GPUVER := A100
else ifeq ($(GPUVER),H100)
ARCH_NUMBER = 90
else ifeq ($(GPUVER),Mi50)
ARCH_NUMBER = gfx906
else ifeq ($(GPUVER),Mi100)
ARCH_NUMBER = gfx908
else ifeq ($(GPUVER),Mi250)
ARCH_NUMBER = gfx90a
else ifeq (,$(ARCH_NUMBER))
ifneq ($(ACC),)
ifneq (opencl,$(USE_ACCEL))
$(error Unknown ARCH_NUMBER since GPUVER="$(GPUVER)" is not recognized)
endif
endif
endif
# Set ACCFLAGS ==============================================================
# Adjust C++ standard level
override ACCFLAGS := $(CXXSTD) $(filter-out -std=c++11,$(ACCFLAGS))
# If compiling with nvcc
ifeq (cuda,$(USE_ACCEL))
override ACCFLAGS := -D__CUDA -DARCH_NUMBER=$(ARCH_NUMBER) $(filter-out -D__CUDA,$(ACCFLAGS))
CXXFLAGS := -D__CUDA $(filter-out -D__CUDA,$(CXXFLAGS))
FCFLAGS := -D__CUDA $(filter-out -D__CUDA,$(FCFLAGS))
#if "-arch" has not yet been set in ACCFLAGS
ifeq (,$(findstring -arch,$(ACCFLAGS)))
override ACCFLAGS := -arch sm_$(ARCH_NUMBER) $(ACCFLAGS)
endif
ifeq (,$(findstring -Xcompiler,$(ACCFLAGS)))
override ACCFLAGS := $(ACCFLAGS) -Xcompiler="$(CXXFLAGS)"
endif
# If compiling with hipcc
else ifeq (hip,$(USE_ACCEL))
override ACCFLAGS := -D__HIP -DARCH_NUMBER=$(ARCH_NUMBER) $(filter-out -D__HIP,$(ACCFLAGS))
CXXFLAGS := -D__HIP $(filter-out -D__HIP,$(CXXFLAGS))
FCFLAGS := -D__HIP $(filter-out -D__HIP,$(FCFLAGS))
# OpenCL backend
else ifeq (opencl,$(USE_ACCEL))
CFLAGS := -D__OPENCL $(filter-out -D__OPENCL $(CXXSTD),$(if $(ACCFLAGS),$(ACCFLAGS),$(CFLAGS)))
endif
# Set the configuration =====================================================
ifneq ($(LD_SHARED),)
ARCHIVE_EXT := .so
else
ARCHIVE_EXT := .a
endif
# Declare PHONY targets =====================================================
.PHONY : dirs makedep \
default_target $(LIBRARY) \
clean version
# Discover files and directories ============================================
ALL_SRC_DIRS := $(shell find $(SRCDIR) -type d | awk '{printf("%s:",$$1)}')
LIBSMM_ACC_DIR := $(shell cd $(SRCDIR) ; find . -type d -name "libsmm_acc")
LIBSMM_ACC_ABS_DIR := $(shell find $(SRCDIR) -type d -name "libsmm_acc")
ALL_PKG_FILES := $(shell find $(SRCDIR) -name "PACKAGE")
OBJ_SRC_FILES = $(shell cd $(SRCDIR); find . ! -name "dbcsr_api_c.F" ! -name "dbcsr_tensor_api_c.F" -name "*.F")
OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . -type f -name "*.c")
# if compiling with GPU acceleration
ifneq ($(ACC),)
ifeq (cuda,$(USE_ACCEL))
# All *.cpp files belong to the accelerator backend common between CUDA/HIP
OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . -type f ! -name "acc_cuda.cpp" ! -name "acc_hip.cpp" -name "*.cpp")
OBJ_SRC_FILES += $(LIBSMM_ACC_DIR)/../cuda/acc_cuda.cpp
# Exclude autotuning files
OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . -type f ! -name "tune_*_exe*_part*.cu" ! -name "tune_*_exe*_main*.cu" -name "*.cu")
else ifeq (hip,$(USE_ACCEL))
# All *.cpp files belong to the accelerator backend common between CUDA/HIP
OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . -type f ! -name "acc_cuda.cpp" ! -name "acc_hip.cpp" -name "*.cpp")
OBJ_SRC_FILES += $(LIBSMM_ACC_DIR)/../hip/acc_hip.cpp
# OpenCL backend: OBJ_SRC_FILES already includes all *.c files
endif
endif
# Include also source files which won't compile into an object file
ALL_SRC_FILES = $(strip $(subst $(NULL) .,$(NULL) $(SRCDIR),$(NULL) $(OBJ_SRC_FILES)))
ALL_SRC_FILES += $(shell find $(SRCDIR) -name "*.f90")
ALL_SRC_FILES += $(shell find $(SRCDIR) -name "*.h")
ALL_SRC_FILES += $(shell find $(SRCDIR) -name "*.hpp")
# stage 1: create dirs and run makedep.py.
# Afterwards, call make recursively again with -C $(OBJDIR) and INCLUDE_DEPS=true
ifeq ($(INCLUDE_DEPS),)
$(LIBRARY): dirs makedep
@+$(MAKE) --no-print-directory -C $(OBJDIR) -f $(MAKEFILE) $(LIBDIR)/$(LIBRARY)$(ARCHIVE_EXT) INCLUDE_DEPS=true DBCSRHOME=$(DBCSRHOME)
dirs:
@mkdir -p $(OBJDIR)
@mkdir -p $(LIBDIR)
version:
@echo "DBCSR Version: "$(MAJOR)"."$(MINOR)"."$(PATCH)" ("$(DATE)")"
else
# stage 2: Include $(OBJDIR)/all.dep, expand target all, and get list of dependencies.
# Check if FYPP is available ===============================================
ifeq (, $(shell which $(FYPPEXE) 2>/dev/null ))
$(error No FYPP submodule available, please read README.md on how to properly download DBCSR)
endif
endif
clean:
rm -f $(LIBSMM_ACC_ABS_DIR)/parameters.h $(LIBSMM_ACC_ABS_DIR)/smm_acc_kernels.h $(LIBSMM_ACC_ABS_DIR)/*.so
rm -f $(LIBSMM_ACC_ABS_DIR)/../opencl/smm/opencl_kernels.h
rm -f $(TESTSDIR)/libsmm_acc_unittest_multiply.cpp
rm -f $(TESTSDIR)/libsmm_acc_timer_multiply.cpp
rm -rf $(OBJDIR)
# Libsmm_acc stuff ==========================================================
ifneq ($(ACC),)
ifneq (opencl,$(USE_ACCEL))
ACC_KERNEL := $(wildcard $(LIBSMM_ACC_ABS_DIR)/kernels/*.h)
ACC_PARAMS := $(wildcard $(LIBSMM_ACC_ABS_DIR)/parameters/parameters_$(GPUVER).json)
$(LIBSMM_ACC_ABS_DIR)/parameters.h: $(LIBSMM_ACC_ABS_DIR)/generate_parameters.py $(ACC_PARAMS)
cd $(LIBSMM_ACC_ABS_DIR); $(PYTHON) generate_parameters.py --gpu_version=$(GPUVER)
$(LIBSMM_ACC_ABS_DIR)/smm_acc_kernels.h: $(LIBSMM_ACC_ABS_DIR)/generate_kernels.py $(ACC_KERNEL)
cd $(LIBSMM_ACC_ABS_DIR); $(PYTHON) generate_kernels.py
endif
endif
# automatic dependency generation ===========================================
MODDEPS = "lower"
MAKEDEPMODE = "normal"
ifeq ($(HACKDEP),yes)
MAKEDEPMODE = "hackdep"
endif
# this happens at stage 1
makedep: $(ALL_SRC_FILES) $(ALL_PKG_FILES) dirs
ifeq ($(LD_SHARED),)
@echo "Removing stale archives ... "
@$(PYTHON) $(DBCSRCP2K)/check_archives.py $(firstword $(AR)) $(SRCDIR) $(LIBDIR)
endif
@echo "Resolving dependencies ... "
@$(PYTHON) $(DBCSRCP2K)/makedep.py $(OBJDIR)/all.dep dbcsr $(MODDEPS) $(MAKEDEPMODE) $(ARCHIVE_EXT) $(SRCDIR) $(OBJ_SRC_FILES)
# at stage 2, load the rules generated by makedep.py
ifeq ($(INCLUDE_DEPS), true)
include $(OBJDIR)/all.dep
endif
# ================= Stuff need for compiling (stage 2) ======================
# These rules are executed in a recursive call to make -C $(OBJDIR)
# The change of $(CURDIR) allows to find targets without abs paths and vpaths.
### Slave rules ###
vpath %.F $(ALL_SRC_DIRS)
vpath %.h $(ALL_SRC_DIRS)
vpath %.hpp $(ALL_SRC_DIRS)
vpath %.f90 $(ALL_SRC_DIRS)
vpath %.cu $(ALL_SRC_DIRS)
vpath %.c $(ALL_SRC_DIRS)
vpath %.cpp $(ALL_SRC_DIRS)
# $(FCLOGPIPE) can be used to store compiler output, e.g. warnings, for each F-file separately.
# This is used e.g. by the convention checker.
FYPPFLAGS ?= -n
%.o: %.F
$(PYTHON) $(FYPPEXE) $(FYPPFLAGS) $< $*.F90
$(FC) -c $(FCFLAGS) -D__SHORT_FILE__="\"$(notdir $<)\"" -I'$(dir $<)' -I'$(SRCDIR)' $*.F90 $(FCLOGPIPE)
%.mod: %.o
@true
%.o: %.c
$(CC) -c $(CFLAGS) $<
ifneq ($(ACC),)
# if compiling CUDA/HIP backend
ifneq (opencl,$(USE_ACCEL))
%.o: %.cpp parameters.h smm_acc_kernels.h
$(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $<
# if compiling CUDA backend
ifeq (cuda,$(USE_ACCEL))
%.o: %.cu parameters.h smm_acc_kernels.h
$(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $<
calculate_norms.o: calculate_norms.cpp
$(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' -x cu $<
acc_cuda.o: acc_cuda.cpp acc_cuda.h
$(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $<
# if compiling HIP backend
else
calculate_norms.o: calculate_norms.cpp
$(OFFLOAD_CC) -D__HIP -c $(OFFLOAD_FLAGS) -I'$(SRCDIR)' $<
acc_hip.o: acc_hip.cpp acc_hip.h
$(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $<
endif
# if compiling OpenCL backend
else
OPENCL_KRNLGEN := $(LIBSMM_ACC_ABS_DIR)/../opencl/acc_opencl.sh
OPENCL_KERNELS := $(wildcard $(LIBSMM_ACC_ABS_DIR)/../opencl/smm/kernels/*.cl)
OPENCL_DEFAULT := $(wildcard $(LIBSMM_ACC_ABS_DIR)/../opencl/smm/tune_multiply.csv)
OPENCL_WITHGPU := $(wildcard $(LIBSMM_ACC_ABS_DIR)/../opencl/smm/params/tune_multiply_*.csv)
OPENCL_PARAMS := $(if $(OPENCL_DEFAULT),$(OPENCL_DEFAULT),$(OPENCL_WITHGPU))
OPENCL_COMMON := $(wildcard $(LIBSMM_ACC_ABS_DIR)/../opencl/common/*.h)
$(LIBSMM_ACC_ABS_DIR)/../opencl/smm/opencl_kernels.h: $(OPENCL_KRNLGEN) $(OPENCL_KERNELS) $(OPENCL_COMMON) $(OPENCL_PARAMS)
$(OPENCL_KRNLGEN) $(OPENCL_KERNELS) $(OPENCL_PARAMS) $@
opencl_libsmm.o: opencl_libsmm.c $(LIBSMM_ACC_ABS_DIR)/../opencl/smm/opencl_kernels.h
ifeq (Darwin,$(shell uname))
LDFLAGS += -framework OpenCL
else
# OpenCL include directory (cl.h not installed per "opencl-headers" package)
NVCC := $(shell which nvcc 2>/dev/null)
NVCC_PATH := $(if $(NVCC),$(realpath $(dir $(NVCC))/..))
CUDA_FILE := $(wildcard $(NVCC_PATH)/../cuda/include/cuda.h)
CUDA_PATH := $(if $(CUDA_FILE),$(abspath $(NVCC_PATH)/../cuda),$(NVCC_PATH))
CUDA_LIBS := $(if $(wildcard $(CUDA_PATH)/lib64),lib64,lib)
ifneq (,$(CUDA_PATH))
CFLAGS += -I$(CUDA_PATH)/include
else ifneq (,$(wildcard $(OPENCL_ROOT)/include/CL/cl.h))
CFLAGS += -I$(OPENCL_ROOT)/include
else
OPENCL_ROOT := $(abspath $(dir $(shell which icx 2>/dev/null))/..)
CLINC := $(wildcard $(OPENCL_ROOT)/include/sycl/CL/cl.h $(OPENCL_ROOT)/include/CL/cl.h)
ifneq (,$(CLINC))
CFLAGS += -I$(abspath $(dir $(firstword $(CLINC)))/..)
endif
endif
endif
endif
endif
$(LIBDIR)/%:
ifneq ($(LD_SHARED),)
@echo "Creating shared library $@"
@$(LD_SHARED) $(LDFLAGS) -o $(@:.a=.so) $^ $(LIBS)
else
@echo "Updating archive $@"
@$(AR) $@ $?
endif
#EOF

View file

@ -1,84 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Check that a static archive contains only the objects specified in the PACKAGES files.
"""
# author: Ole Schuett
import subprocess
import os
from os import path
import argparse
KNOWN_EXTENSIONS = ("F", "c", "cu", "cpp", "cxx", "cc")
def main(ar_exe, src_dir, lib_dir):
# Search all files belonging to a given archive
archives_files = {}
for root, _, files in os.walk(src_dir):
if "PACKAGE" in files:
with open(path.join(root, "PACKAGE")) as fhandle:
content = fhandle.read()
package = eval(content)
archive = "libdbcsr" + path.basename(root)
if "archive" in package.keys():
archive = package["archive"]
file_parts = [fn.rsplit(".", 1) for fn in files]
src_basenames = [
parts[0] for parts in file_parts if parts[-1] in KNOWN_EXTENSIONS
]
if archive in archives_files:
archives_files[archive] |= set(src_basenames)
else:
archives_files[archive] = set(src_basenames)
# Check if the symbols in each archive have a corresponding source file
for archive in archives_files:
archive_fn = path.join(lib_dir, archive + ".a")
if not path.exists(archive_fn):
continue
output = subprocess.check_output([ar_exe, "t", archive_fn])
for line in output.decode("utf8").strip().splitlines():
if line == "__.SYMDEF SORTED":
continue # needed for MacOS
assert line.endswith(
".o"
), "discovered a non-object file inside a static archive"
if line[:-2] not in archives_files[archive]:
print(
"Could not find source for object '{}' in archive '{}', removing archive.".format(
line, archive_fn
)
)
os.remove(archive_fn)
break
# ===============================================================================
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="""
Parse files and package manifests in the source tree to create rules for objects and executables
This script is part of the build utility scripts for DBCSR.
""",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("ar_executable", metavar="ar_executable", type=str)
parser.add_argument("src_dir", metavar="src_dir", type=str)
parser.add_argument("lib_dir", metavar="lib_dir", type=str)
args = parser.parse_args()
main(ar_exe=args.ar_executable, src_dir=args.src_dir, lib_dir=args.lib_dir)

View file

@ -1,441 +0,0 @@
#!/usr/bin/env python3
import re
import sys
import ast
from os import path
from os.path import dirname, basename, normpath
import glob
import argparse
# pre-compiled regular expressions
re_module = re.compile(r"(?:^|\n)\s*module\s+(\w+)\s.*\n\s*end\s*module", re.DOTALL)
re_program = re.compile(r"\n\s*end\s*program")
re_main = re.compile(r"\sint\s+main\s*\(")
re_use = re.compile(r"\n\s*use\s+(\w+)")
re_incl_fypp = re.compile(r"\n#:include\s+['\"](.+)['\"]")
re_incl_cpp = re.compile(r"\n#include\s+['\"](.+)['\"]")
re_incl_fort = re.compile(r"\n\s*include\s+['\"](.+)['\"]")
# ============================================================================
def main(out_fn, project_name, mod_format, mode, archive_ext, src_dir, src_files):
messages = []
# process arguments
src_files = [normpath(path.join(src_dir, f)) for f in src_files]
if mod_format not in ("lower", "upper", "no"):
error('Module filename format must be either of "lower", "upper", or "no".')
if mode not in ("normal", "hackdep", "mod_compiler"):
error('Mode must be either of "normal", "hackdep", or "mod_compiler".')
for fn in src_files:
if not fn.startswith("/"):
error("Path of source-file not absolute: " + fn)
src_basenames = [basename(fn).rsplit(".", 1)[0] for fn in src_files]
for bfn in src_basenames:
if src_basenames.count(bfn) > 1:
error("Multiple source files with the same basename: " + bfn)
# parse files
parsed_files = dict()
for fn in src_files:
parse_file(parsed_files, fn, src_dir) # parses also included files
messages.append("Parsed %d files" % len(parsed_files))
# create table mapping fortran module-names to file-name
mod2fn = dict()
for fn in src_files:
for m in parsed_files[fn]["module"]:
if m in mod2fn.keys():
error('Multiple declarations of module "%s"' % m)
mod2fn[m] = fn
messages.append("Created mod2fn table, found %d modules." % len(mod2fn))
# check "one module per file"-convention
for m, fn in mod2fn.items():
if basename(fn) != m + ".F":
error("Names of module and file do not match: " + fn)
# read package manifests
packages = dict()
for fn in src_files:
p = normpath(dirname(fn))
read_pkg_manifest(project_name, packages, p)
messages.append("Read %d package manifests" % len(packages))
# check dependencies against package manifests
n_deps = 0
for fn in src_files:
p = normpath(dirname(fn))
if not parsed_files[fn]["program"]:
packages[p]["objects"].append(src2obj(basename(fn)))
deps = collect_include_deps(parsed_files, fn, src_dir)
deps += [
mod2fn[m]
for m in collect_use_deps(parsed_files, fn, src_dir)
if m in mod2fn.keys()
]
n_deps += len(deps)
for d in deps:
dp = normpath(dirname(d))
if dp not in packages[p]["allowed_deps"]:
error(
"Dependency forbidden according to package manifest: %s -> %s"
% (fn, d)
)
if dp != p and "public_files" in packages[dp].keys():
if basename(d) not in packages[dp]["public_files"]:
error(
"File not public according to package manifest: %s -> %s"
% (fn, d)
)
messages.append("Checked %d dependencies" % n_deps)
# check for circular dependencies
for fn in parsed_files.keys():
find_cycles(parsed_files, mod2fn, fn, src_dir)
# write messages as comments
makefile = "\n".join("#makedep: {}".format(m) for m in messages)
makefile += "\n\n"
# write rules for archives
for pkg in packages.keys():
if packages[pkg]["objects"]:
makefile += """\
# Package {pkg}
$(LIBDIR)/{archive}{ext} : {objs}
""".format(
pkg=pkg,
archive=packages[pkg]["archive"],
ext=archive_ext,
objs=" ".join(packages[pkg]["objects"]),
)
# write rules for public files
for pkg in packages.keys():
if "public" in packages[pkg].keys():
makefile += """\
# Public modules for package {pkg}
install: PUBLICFILES += {pubfiles}
""".format(
pkg=pkg, pubfiles=" ".join(mod for mod in packages[pkg]["public"])
)
# write rules for executables
archive_postfix = archive_ext.rsplit(".", 1)[0]
for fn in src_files:
if not parsed_files[fn]["program"]:
continue
bfn = basename(fn).rsplit(".", 1)[0]
p = normpath(dirname(fn))
deps = collect_pkg_deps(packages, p)
assert all(a.startswith("lib") for a in deps)
cflagsvar = " $(LDFLAGS_C)" if fn.endswith(".c") or fn.endswith(".cu") else ""
makefile += """\
# Program {fn}
$(EXEDIR)/{bfn}.$(ONEVERSION) : {bfn}.o {deps}
\t$(LD) $(LDFLAGS) {cflagsvar} -L$(LIBDIR) -o $@ {bfn}.o $(EXTERNAL_OBJECTS) {linkerdeps} $(LIBS)
""".format(
fn=fn,
bfn=bfn,
deps=" ".join(["$(LIBDIR)/" + a + archive_ext for a in deps]),
cflagsvar=cflagsvar,
linkerdeps=" ".join("-l{}{}".format(a[3:], archive_postfix) for a in deps),
)
# write rules for objects
for fn in src_files:
deps = collect_include_deps(parsed_files, fn, src_dir)
mods = collect_use_deps(parsed_files, fn, src_dir)
mods.sort(key=cmp_mods) # sort mods to speedup compilation
deps += [mod2modfile(m, mod_format) for m in mods if m in mod2fn.keys()]
if mode == "hackdep":
deps = []
deps = " ".join(deps)
bfn = basename(fn)
provides = [mod2modfile(m, mod_format) for m in parsed_files[fn]["module"]]
makefile += "# Object {bfn}\n".format(bfn=bfn)
for mfn in provides:
makefile += "{mfn} : {bfn} {deps}\n".format(mfn=mfn, bfn=bfn, deps=deps)
makefile += "{bfnobj} : {bfn} {deps}".format(
bfnobj=src2obj(bfn), bfn=bfn, deps=deps
)
if mode == "mod_compiler":
makefile += " " + " ".join(provides)
makefile += "\n\n"
with open(out_fn, "w", encoding="utf8") as fhandle:
fhandle.write(makefile)
# ============================================================================
def cmp_mods(mod):
# list "type" modules first, they are probably on the critical path
if "type" in mod:
return 0
return 1
# ============================================================================
def parse_file(parsed_files, fn, src_dir):
if fn in parsed_files:
return
with open(fn, encoding="utf8") as fhandle:
content = fhandle.read()
# re.IGNORECASE is horribly expensive. Converting to lower-case upfront
content_lower = content.lower()
# all files are parsed for cpp includes
incls = re_incl_cpp.findall(content) # CPP includes (case-sensitive)
mods = []
uses = []
prog = False
if fn[-2:] == ".F" or fn[-4:] == ".f90" or fn[-5:] == ".fypp":
mods += re_module.findall(content_lower)
prog = True if re_program.search(content_lower) is not None else False
uses += re_use.findall(content_lower)
incls += re_incl_fypp.findall(content) # Fypp includes (case-sensitive)
incl_fort_iter = re_incl_fort.finditer(content_lower) # fortran includes
incls += [content[m.start(1) : m.end(1)] for m in incl_fort_iter]
if fn[-2:] == ".c" or fn[-3:] == ".cu":
prog = (
True if re_main.search(content) is not None else False
) # C is case-sensitive
# exclude included files from outside the source tree
def incl_fn(i):
return normpath(path.join(dirname(fn), i))
def incl_fn_src(i):
return normpath(path.join(src_dir, i))
existing_incl = [i for i in incls if path.exists(incl_fn(i))]
existing_incl_src = [i for i in incls if path.exists(incl_fn_src(i))]
# store everything in parsed_files cache
parsed_files[fn] = {
"module": mods,
"program": prog,
"use": uses,
"include": existing_incl,
"include_src": existing_incl_src,
}
# parse included files
for i in existing_incl:
parse_file(parsed_files, incl_fn(i), src_dir)
for i in existing_incl_src:
parse_file(parsed_files, incl_fn_src(i), src_dir)
# ============================================================================
def read_pkg_manifest(project_name, packages, p):
if p in packages.keys():
return
fn = path.join(p, "PACKAGE")
if not path.exists(fn):
error("Could not open PACKAGE manifest: " + fn)
with open(fn, encoding="utf8") as fhandle:
content = fhandle.read()
packages[p] = ast.literal_eval(content)
packages[p]["objects"] = []
if "archive" not in packages[p].keys():
packages[p]["archive"] = "lib{}{}".format(project_name, basename(p))
packages[p]["allowed_deps"] = [normpath(p)]
packages[p]["allowed_deps"] += [
normpath(path.join(p, r)) for r in packages[p]["requires"]
]
for r in packages[p]["requires"]:
read_pkg_manifest(project_name, packages, normpath(path.join(p, r)))
if "public" in packages[p].keys():
public_files = []
for fn in packages[p]["public"]:
public_files += glob.glob(path.join(p, fn))
packages[p]["public_files"] = [basename(fn) for fn in public_files]
# ============================================================================
def mod2modfile(m, mod_format):
if mod_format == "no":
return ""
if mod_format == "lower":
return m.lower() + ".mod"
if mod_format == "upper":
return m.upper() + ".mod"
assert False # modeps unknown
# ============================================================================
def src2obj(src_fn):
return basename(src_fn).rsplit(".", 1)[0] + ".o"
# ============================================================================
def collect_include_deps(parsed_files, fn, src_dir):
pf = parsed_files[fn]
incs = []
for i in pf["include"]:
fn_inc = normpath(path.join(dirname(fn), i))
if fn_inc in parsed_files.keys():
incs.append(fn_inc)
incs += collect_include_deps(parsed_files, fn_inc, src_dir)
for i in pf["include_src"]:
fn_inc = normpath(path.join(src_dir, i))
if fn_inc in parsed_files.keys():
incs.append(fn_inc)
incs += collect_include_deps(parsed_files, fn_inc, src_dir)
return sorted(set(incs))
# ============================================================================
def collect_use_deps(parsed_files, fn, src_dir):
pf = parsed_files[fn]
uses = pf["use"]
for i in pf["include"]:
fn_inc = normpath(path.join(dirname(fn), i))
if fn_inc in parsed_files.keys():
uses += collect_use_deps(parsed_files, fn_inc, src_dir)
for i in pf["include_src"]:
fn_inc = normpath(path.join(src_dir, i))
if fn_inc in parsed_files.keys():
uses += collect_use_deps(parsed_files, fn_inc, src_dir)
return sorted(set(uses))
# ============================================================================
def find_cycles(parsed_files, mod2fn, fn, src_dir, S=None):
pf = parsed_files[fn]
if "visited" in pf.keys():
return
if S is None:
S = []
for m in pf["module"]:
if m in S:
i = S.index(m)
n = S[i:]
else:
n = []
if 0 == len(n) or (1 == len(n) and m in n):
S.append(m)
else:
error("Circular dependency: " + " -> ".join(n + [m]))
for m in collect_use_deps(parsed_files, fn, src_dir):
if m in mod2fn.keys():
find_cycles(parsed_files, mod2fn, mod2fn[m], src_dir, S)
for m in pf["module"]:
S.pop()
pf["visited"] = True
# ============================================================================
def collect_pkg_deps(packages, p, archives=None, S=None):
if archives is None:
archives = []
if S is None:
S = []
a = packages[p]["archive"]
if a in archives:
return archives
if a in S:
i = S.index(a)
b = S[i:]
else:
b = []
if 0 == len(b) or (1 == len(b) and a in b):
S.append(a)
else:
error("Circular package dependency: " + " -> ".join(b + [a]))
for r in packages[p]["requires"]:
d = normpath(path.join(p, r))
collect_pkg_deps(packages, d, archives, S)
S.pop()
if packages[p]["objects"]:
archives.insert(0, packages[p]["archive"])
return archives
# ============================================================================
def error(msg):
sys.stderr.write("makedep error: %s\n" % msg)
sys.exit(1)
# ============================================================================
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="""
Parse files and package manifests in the source tree to create rules for objects and executables
This script is part of the build utility scripts for DBCSR.
""",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("outfile", metavar="outfile", type=str)
parser.add_argument("project_name", metavar="project_name", type=str)
parser.add_argument("format", metavar="format", type=str)
parser.add_argument("mode", metavar="mode", type=str)
parser.add_argument("archive_ext", metavar="archive_ext", type=str)
parser.add_argument("src_dir", metavar="src_dir", type=str)
parser.add_argument("src_file", metavar="src_file", nargs="+", type=str)
args = parser.parse_args()
main(
out_fn=args.outfile,
project_name=args.project_name,
mod_format=args.format,
mode=args.mode,
archive_ext=args.archive_ext,
src_dir=args.src_dir,
src_files=args.src_file,
)

View file

@ -1,255 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Unittests for the makedep.py script
"""
import unittest
import tempfile
import shutil
import json
import io
import os
from os import path
import makedep
def _mkpkg(content, base_dir):
"""
Create a PACKAGE file in the given directory.
If the content is a list or a dict, serialize the structure using json.dump.
"""
with open(path.join(base_dir, "PACKAGE"), "w") as fhandle:
if isinstance(content, (dict, list)):
json.dump(content, fhandle, ensure_ascii=False)
else:
fhandle.write(content)
class TestCheckArchives(unittest.TestCase):
"""Test case for the main method in makedep"""
def setUp(self):
self.base_dir = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.base_dir)
def test_empty_dir(self):
"""
running on an empty dir with no files generates an empty deps file
"""
my_dir = path.join(self.base_dir, "empty")
out_fn = path.join(my_dir, "all.dep")
os.mkdir(my_dir)
makedep.main(out_fn, "empty_project", "lower", "normal", ".a", my_dir, [])
with open(out_fn, "r") as fhandle:
no_comment_lines = [
l.strip() for l in fhandle if not l.startswith("#") and l.strip()
]
self.assertEqual(len(no_comment_lines), 0)
def test_single_empty(self):
"""
running on a dir with a single file
"""
my_dir = path.join(self.base_dir, "single_empty")
out_fn = path.join(my_dir, "all.dep")
single_fn = path.join(my_dir, "single.F")
os.mkdir(my_dir)
with open(single_fn, "w") as fhandle:
fhandle.write("\n")
with self.assertRaises(SystemExit):
# should throw an exception due to missing PACKAGES
makedep.main(
out_fn,
"faulty_project",
"lower",
"normal",
".a",
my_dir,
["./single.F"],
)
_mkpkg(
{
"description": "Nothing",
"archive": "test",
"public": ["*.F"],
"requires": [],
},
my_dir,
)
makedep.main(
out_fn, "single_empty", "lower", "normal", ".a", my_dir, ["./single.F"]
)
with open(out_fn, "r") as fhandle:
no_comment_lines = [
l.strip() for l in fhandle if not l.startswith("#") and l.strip()
]
self.assertEqual(
no_comment_lines,
[
"$(LIBDIR)/test.a : single.o",
"install: PUBLICFILES += *.F",
"single.o : single.F",
],
)
def test_unicode(self):
"""
running on a dir with a single file
"""
my_dir = path.join(self.base_dir, "unicode")
out_fn = path.join(my_dir, "all.dep")
single_fn = path.join(my_dir, "single.F")
os.mkdir(my_dir)
with io.open(single_fn, "w", encoding="utf8") as fhandle:
fhandle.write(u"! Ångström\n")
_mkpkg(
"""{
"description": "unicode test with just an Ångström 😉",
"archive": "test",
"public": ["*.F"],
"requires": []
}""",
my_dir,
)
makedep.main(
out_fn, "unicode_project", "lower", "normal", ".a", my_dir, ["./single.F"]
)
with open(out_fn, "r") as fhandle:
no_comment_lines = [
l.strip() for l in fhandle if not l.startswith("#") and l.strip()
]
self.assertEqual(
no_comment_lines,
[
"$(LIBDIR)/test.a : single.o",
"install: PUBLICFILES += *.F",
"single.o : single.F",
],
)
def test_subpackage(self):
"""
running on a dir with a single file
"""
my_dir = path.join(self.base_dir, "subpackage")
out_fn = path.join(my_dir, "all.dep")
sub_dir = path.join(my_dir, "sub")
single_fn = path.join(my_dir, "single.F")
sub_fn = path.join(sub_dir, "sub.F")
os.mkdir(my_dir)
os.mkdir(sub_dir)
for fname in [single_fn, sub_fn]:
with open(fname, "w") as fhandle:
fhandle.write("")
_mkpkg(
"""{
"description": "test with a simple subpackage",
"archive": "test",
"public": ["*.F"],
"requires": ["./sub"]
}""",
my_dir,
)
_mkpkg(
"""{
"description": "the sub package",
"requires": [""]
}""",
sub_dir,
)
makedep.main(
out_fn,
"sub_pkg_without_dep",
"lower",
"normal",
".a",
my_dir,
["./single.F"],
)
with open(out_fn, "r") as fhandle:
no_comment_lines = [
l.strip() for l in fhandle if not l.startswith("#") and l.strip()
]
self.assertEqual(
no_comment_lines,
[
"$(LIBDIR)/test.a : single.o",
"install: PUBLICFILES += *.F",
"single.o : single.F",
],
)
with open(single_fn, "w") as fhandle:
fhandle.write(
"""module single
use :: sub
end module"""
)
with open(sub_fn, "w") as fhandle:
fhandle.write(
"""module sub
end module"""
)
makedep.main(
out_fn,
"sub_pkg_with_dep",
"lower",
"normal",
".a",
my_dir,
["./single.F", "./sub/sub.F"],
)
with open(out_fn, "r") as fhandle:
no_comment_lines = [
l.strip() for l in fhandle if not l.startswith("#") and l.strip()
]
self.assertEqual(
sorted(no_comment_lines),
[
"$(LIBDIR)/libsub_pkg_with_depsub.a : sub.o",
"$(LIBDIR)/test.a : single.o",
"install: PUBLICFILES += *.F",
"single.mod : single.F",
"single.o : single.F",
"sub.o : sub.F",
],
)
# ============================================================================
if __name__ == "__main__":
unittest.main()

@ -1 +0,0 @@
Subproject commit 2cdbf1331eb9f0e8265f4ca048cedb125f3e38f9