diff --git a/tools/prettify/prettify.py b/tools/prettify/prettify.py index cc9d00fd1c..f89a974026 100755 --- a/tools/prettify/prettify.py +++ b/tools/prettify/prettify.py @@ -1,11 +1,16 @@ #!/usr/bin/env python +from __future__ import print_function + import sys import re import tempfile import os -import os.path +from os import path import logging +import argparse +import errno +import traceback try: from hashlib import md5 @@ -73,7 +78,7 @@ TO_UPCASE_RE = re.compile( ) TO_UPCASE_OMP_RE = re.compile( - """ + r""" (? (?: @@ -124,21 +129,17 @@ def upcaseStringKeywords(line): return res -def upcaseOMP(line): - """Upcases OpenMP stuff.""" - return TO_UPCASE_OMP_RE.sub(lambda match: match.group("toUpcase").upper(), line) - - def upcaseKeywords(infile, outfile, upcase_omp): """Writes infile to outfile with all the fortran keywords upcased""" - while 1: - line = infile.readline() - if not line: - break + + for line in infile: line = upcaseStringKeywords(line) - if upcase_omp: - if normalizeFortranFile.OMP_DIR_RE.match(line): - line = upcaseOMP(line) + + if upcase_omp and normalizeFortranFile.OMP_DIR_RE.match(line): + line = TO_UPCASE_OMP_RE.sub( + lambda match: match.group("toUpcase").upper(), line + ) + outfile.write(line) @@ -163,145 +164,140 @@ def prettifyFile( to false) does not close the input file""" - ifile = infile - orig_filename = filename - tmpfile = None max_pretty_iter = 5 - n_pretty_iter = 0 - if is_fypp(ifile): - logger = logging.getLogger("fprettify-logger") - logger.error(orig_filename + ": fypp directives not supported.\n") - return ifile + logger = logging.getLogger("fprettify-logger") - while True: - n_pretty_iter += 1 - hash_prev = md5() - hash_prev.update(ifile.read().encode("utf8")) - ifile.seek(0) + if is_fypp(infile): + logger.error("{}: fypp directives not supported.\n".format(filename)) + return infile + + # create a temporary file first as a copy of the input file + tmpfile = tempfile.TemporaryFile(mode="w+") + tmpfile.write(infile.read()) + tmpfile.seek(0) + + infile = tmpfile # overwrite the handle since we don't need it anymore + + hash_prev = md5(infile.read().encode("utf8")) + infile.seek(0) + + for _ in range(max_pretty_iter): try: if replace: - tmpfile2 = tempfile.TemporaryFile(mode="w+") - replacer.replaceWords(ifile, tmpfile2) - tmpfile2.seek(0) - if tmpfile: - tmpfile.close() - tmpfile = tmpfile2 - ifile = tmpfile + tmpfile = tempfile.TemporaryFile(mode="w+") + replacer.replaceWords(infile, tmpfile) + tmpfile.seek(0) + infile.close() + infile = tmpfile + if reformat: # reformat needs to be done first - tmpfile2 = tempfile.TemporaryFile(mode="w+") + tmpfile = tempfile.TemporaryFile(mode="w+") try: reformat_ffile( - ifile, - tmpfile2, + infile, + tmpfile, indent_size=indent, whitespace=whitespace, - orig_filename=orig_filename, + orig_filename=filename, ) except fparse_utils.FprettifyParseException as e: log_exception( e, "fprettify could not parse file, file is not prettified" ) - tmpfile2.write(ifile.read()) + tmpfile.write(infile.read()) + + tmpfile.seek(0) + infile.close() + infile = tmpfile - tmpfile2.seek(0) - if tmpfile: - tmpfile.close() - tmpfile = tmpfile2 - ifile = tmpfile if normalize_use: - tmpfile2 = tempfile.TemporaryFile(mode="w+") + tmpfile = tempfile.TemporaryFile(mode="w+") normalizeFortranFile.rewriteFortranFile( - ifile, - tmpfile2, + infile, + tmpfile, indent, decl_linelength, decl_offset, - orig_filename=orig_filename, + orig_filename=filename, ) - tmpfile2.seek(0) - if tmpfile: - tmpfile.close() - tmpfile = tmpfile2 - ifile = tmpfile + tmpfile.seek(0) + infile.close() + infile = tmpfile + if upcase_keywords: - tmpfile2 = tempfile.TemporaryFile(mode="w+") - upcaseKeywords(ifile, tmpfile2, upcase_omp) - tmpfile2.seek(0) - if tmpfile: - tmpfile.close() - tmpfile = tmpfile2 - ifile = tmpfile - hash_next = md5() - hash_next.update(ifile.read().encode("utf8")) - ifile.seek(0) - if hash_prev.digest() == hash_next.digest(): - return ifile - elif n_pretty_iter >= max_pretty_iter: - raise RuntimeError( - "Prettify did not converge in", max_pretty_iter, "steps." - ) + tmpfile = tempfile.TemporaryFile(mode="w+") + upcaseKeywords(infile, tmpfile, upcase_omp) + tmpfile.seek(0) + infile.close() + infile = tmpfile + + hash_new = md5(infile.read().encode("utf8")) + infile.seek(0) + + if hash_prev.digest() == hash_new.digest(): + return infile + + hash_prev = hash_new + except: - logger = logging.getLogger("fprettify-logger") - logger.critical("error processing file '" + infile.name + "'\n") + logger.critical("error processing file '{}'\n".format(filename)) raise + else: + raise RuntimeError( + "Prettify did not converge in {} steps.".format(max_pretty_iter) + ) -def prettfyInplace(fileName, bkDir=None, stdout=False, **kwargs): + +def prettifyInplace(filename, backupdir=None, stdout=False, **kwargs): """Same as prettify, but inplace, replaces only if needed""" - if fileName == "stdin": + if filename == "stdin": infile = tempfile.TemporaryFile(mode="r+") infile.write(sys.stdin.read()) + infile.seek(0) else: - infile = open(fileName, "r") + infile = open(filename, "r") + + outfile = prettifyFile(infile=infile, filename=filename, **kwargs) if stdout: - outfile = prettifyFile(infile=infile, filename=fileName, **kwargs) outfile.seek(0) sys.stdout.write(outfile.read()) outfile.close() return - if bkDir and not os.path.exists(bkDir): - os.mkdir(bkDir) - if bkDir and not os.path.isdir(bkDir): - raise Error("bk-dir must be a directory, was " + bkDir) - - outfile = prettifyFile(infile=infile, filename=fileName, **kwargs) if infile == outfile: + infile.close() return + infile.seek(0) outfile.seek(0) - same = 1 + changed = True + for line1, line2 in zip(infile, outfile): + if line1 != line2: + break + else: + changed = False + + if changed: + if backupdir: + bkName = path.join(backupdir, path.basename(filename)) + + with open(bkName, "w") as fhandle: + infile.seek(0) + fhandle.write(infile.read()) + + infile.close() # close it here since we're going to overwrite it + + with open(filename, "w") as fhandle: + outfile.seek(0) + fhandle.write(outfile.read()) + + else: + infile.close() - while 1: - l1 = outfile.readline() - l2 = infile.readline() - if l1 != l2: - same = 0 - break - if not l1: - break - if not same: - bkFile = None - if bkDir: - bkName = os.path.join(bkDir, os.path.basename(fileName)) - bName = bkName - i = 0 - while os.path.exists(bkName): - i += 1 - bkName = bName + "." + str(i) - bkFile = open(bkName, "w") - infile.seek(0) - if bkFile: - bkFile.write(infile.read()) - bkFile.close() - outfile.seek(0) - newFile = open(fileName, "w") - newFile.write(outfile.read()) - newFile.close() - infile.close() outfile.close() @@ -320,161 +316,148 @@ def is_fypp(infile): return False -def main(argv=None): - if argv is None: - argv = sys.argv - defaultsDict = { - "upcase": 1, - "normalize-use": 1, - "omp-upcase": 1, - "decl-linelength": 100, - "decl-offset": 50, - "reformat": 1, - "indent": 3, - "whitespace": 1, - "replace": 1, - "stdout": 0, - "do-backup": 0, - "backup-dir": "preprettify", - "report-errors": 1, - "debug": 0, - } +# based on https://stackoverflow.com/a/31347222 +def argparse_add_bool_arg(parser, name, default, helptxt): + dname = name.replace("-", "_") + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument( + "--{}".format(name), dest=dname, action="store_true", help=helptxt + ) + group.add_argument("--no-{}".format(name), dest=dname, action="store_false") + parser.set_defaults(**{dname: default}) - usageDesc = ( - "usage:\nfprettify" - + """ - [--[no-]upcase] [--[no-]normalize-use] [--[no-]omp-upcase] [--[no-]replace] - [--[no-]reformat] [--indent=3] [--whitespace=1] [--help] - [--[no-]stdout] [--[no-]do-backup] [--backup-dir=bk_dir] [--[no-]report-errors] file1 [file2 ...] - [--[no-]debug] - Auto-format F90 source file1, file2, ...: - If no files are given, stdin is used. - --normalize-use - Sorting and alignment of variable declarations and USE statements, removal of unused list entries. - The line length of declarations is controlled by --decl-linelength=n, the offset of the variable list - is controlled by --decl-offset=n. - --reformat - Auto-indentation, auto-alignment and whitespace formatting. - Amount of whitespace controlled by --whitespace = 0, 1, 2. - For indenting with a relative width of n columns specify --indent=n. - For manual formatting of specific lines: - - disable auto-alignment by starting line continuation with an ampersand '&'. - - completely disable reformatting by adding a comment '!&'. - For manual formatting of a code block, use: - - start a manually formatted block with a '!&<' comment and close it with a '!&>' comment. - --upcase - Upcasing fortran keywords. - --omp-upcase - Upcasing OMP directives. - --replace - If requested the replacements performed by the replacer.py script are also performed. Note: these replacements are specific to CP2K. - --stdout - write output to stdout - --[no-]do-backup - store backups of original files in backup-dir (--backup-dir option) - --[no-]report-errors - report warnings and errors +# from https://stackoverflow.com/a/600612 +def mkdir_p(p): + try: + os.makedirs(p) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and path.isdir(p): + pass + else: + raise - Note: for editor integration, use options --no-normalize-use --no-report-errors - Defaults: - """ - + str(defaultsDict) +# from https://stackoverflow.com/a/14981125 +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def abspath(p): + return path.abspath(path.expanduser(p)) + + +def main(argv): + parser = argparse.ArgumentParser( + description="Auto-format F90 source files", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + epilog="""\ + If no files are given, stdin is used. + Note: for editor integration, use options --no-normalize-use --no-report-errors""", ) - replace = None + parser.add_argument("--indent", type=int, default=3) + parser.add_argument("--whitespace", type=int, default=1, choices=range(3)) + parser.add_argument("--decl-linelength", type=int, default=100) + parser.add_argument("--decl-offset", type=int, default=50) + parser.add_argument("--backup-dir", type=abspath, default=abspath("preprettify")) - if "--help" in argv: - sys.stderr.write(usageDesc + "\n") - return 0 - args = [] - for arg in argv[1:]: - m = re.match( - r"--(no-)?(normalize-use|upcase|omp-upcase|replace|reformat|stdout|do-backup|report-errors|debug)", - arg, - ) - if m: - defaultsDict[m.groups()[1]] = not m.groups()[0] - else: - m = re.match(r"--(indent|whitespace|decl-linelength|decl-offset)=(.*)", arg) - if m: - defaultsDict[m.groups()[0]] = int(m.groups()[1]) + argparse_add_bool_arg(parser, "upcase", True, "Upcasing fortran keywords."), + argparse_add_bool_arg( + parser, + "normalize-use", + True, + """\ + Sorting and alignment of variable declarations and USE statements, removal of unused list entries. + The line length of declarations is controlled by --decl-linelength=n, the offset of the variable list + is controlled by --decl-offset=n.""", + ) + argparse_add_bool_arg(parser, "omp-upcase", True, "Upcasing OMP directives.") + argparse_add_bool_arg( + parser, + "reformat", + True, + """\ + Auto-indentation, auto-alignment and whitespace formatting. + Amount of whitespace controlled by --whitespace = 0, 1, 2. + For indenting with a relative width of n columns specify --indent=n. + For manual formatting of specific lines: + - disable auto-alignment by starting line continuation with an ampersand '&'. + - completely disable reformatting by adding a comment '!&'. + For manual formatting of a code block, use: + - start a manually formatted block with a '!&<' comment and close it with a '!&>' comment.""", + ) + argparse_add_bool_arg( + parser, + "replace", + True, + "If requested the replacements performed by the replacer.py script are also performed. Note: these replacements are specific to CP2K.", + ) + argparse_add_bool_arg(parser, "stdout", False, "write output to stdout") + argparse_add_bool_arg( + parser, + "do-backup", + False, + "store backups of original files in backup-dir (--backup-dir option)", + ) + argparse_add_bool_arg(parser, "report-errors", True, "report warnings and errors") + argparse_add_bool_arg(parser, "debug", False, "increase log level to debug") + + parser.add_argument("files", metavar="file", type=str, nargs="*", default=["stdin"]) + + args = parser.parse_args(argv) + + if args.do_backup and not (args.stdout or args.files == ["stdin"]): + mkdir_p(args.backup_dir) + + failure = 0 + + for filename in args.files: + if not path.isfile(filename) and not filename == "stdin": + eprint("file '{}' does not exist!".format(filename)) + failure += 1 + continue + + level = logging.CRITICAL + + if args.report_errors: + if args.debug: + level = logging.DEBUG else: - m = re.match(r"--(backup-dir)=(.*)", arg) - if m: - path = os.path.abspath(os.path.expanduser(m.groups()[1])) - defaultsDict[m.groups()[0]] = path - else: - if arg.startswith("--"): - sys.stderr.write("unknown option " + arg + "\n") - else: - args.append(arg) - bkDir = "" - if defaultsDict["do-backup"]: - bkDir = defaultsDict["backup-dir"] - if bkDir and not os.path.exists(bkDir): - # Another parallel running instance might just have created the - # dir. + level = logging.INFO + + logger = logging.getLogger("fprettify-logger") + logger.setLevel(level) + sh = logging.StreamHandler() + sh.setLevel(level) + formatter = logging.Formatter("%(levelname)s - %(message)s") + sh.setFormatter(formatter) + logger.addHandler(sh) + try: - os.mkdir(bkDir) + prettifyInplace( + filename, + backupdir=args.backup_dir if args.do_backup else None, + stdout=args.stdout or filename == "stdin", + normalize_use=args.normalize_use, + decl_linelength=args.decl_linelength, + decl_offset=args.decl_offset, + reformat=args.reformat, + indent=args.indent, + whitespace=args.whitespace, + upcase_keywords=args.upcase, + upcase_omp=args.omp_upcase, + replace=args.replace, + ) except: - assert os.path.exists(bkDir) - if bkDir and not os.path.isdir(bkDir): - sys.stderr.write("bk-dir must be a directory" + "\n") - sys.stderr.write(usageDesc + "\n") - else: - failure = 0 - if not args: - args = ["stdin"] - for fileName in args: - if not os.path.isfile(fileName) and not fileName == "stdin": - sys.stderr.write("file " + fileName + " does not exists!\n") - else: - stdout = defaultsDict["stdout"] or fileName == "stdin" + eprint("-" * 60) + traceback.print_exc(file=sys.stderr) + eprint("-" * 60) + eprint("Processing file '{}'".format(filename)) + failure += 1 - if defaultsDict["report-errors"]: - if defaultsDict["debug"]: - level = logging.DEBUG - else: - level = logging.INFO - - else: - level = logging.CRITICAL - - logger = logging.getLogger("fprettify-logger") - logger.setLevel(level) - sh = logging.StreamHandler() - sh.setLevel(level) - formatter = logging.Formatter("%(levelname)s - %(message)s") - sh.setFormatter(formatter) - logger.addHandler(sh) - - try: - prettfyInplace( - fileName, - bkDir=bkDir, - stdout=stdout, - normalize_use=defaultsDict["normalize-use"], - decl_linelength=defaultsDict["decl-linelength"], - decl_offset=defaultsDict["decl-offset"], - reformat=defaultsDict["reformat"], - indent=defaultsDict["indent"], - whitespace=defaultsDict["whitespace"], - upcase_keywords=defaultsDict["upcase"], - upcase_omp=defaultsDict["omp-upcase"], - replace=defaultsDict["replace"], - ) - except: - failure += 1 - import traceback - - sys.stderr.write("-" * 60 + "\n") - traceback.print_exc(file=sys.stderr) - sys.stderr.write("-" * 60 + "\n") - sys.stderr.write("Processing file '" + fileName + "'\n") - return failure > 0 + return failure > 0 if __name__ == "__main__": - sys.exit(main()) + sys.exit(main(sys.argv[1:])) diff --git a/tools/prettify/prettify_cp2k/normalizeFortranFile.py b/tools/prettify/prettify_cp2k/normalizeFortranFile.py index da374d8412..3356a36d33 100644 --- a/tools/prettify/prettify_cp2k/normalizeFortranFile.py +++ b/tools/prettify/prettify_cp2k/normalizeFortranFile.py @@ -10,16 +10,20 @@ except ImportError: R_USE = 0 R_VAR = 0 -VAR_RE = re.compile(r" *(?P[a-zA-Z_0-9]+) *(?P(?:\((?P(?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\))? *(?:= *(?P(:?[^\"',()]+|\((?:[^()\"']+|\([^()\"']*\)|\"[^\"]*\"|'[^']*')*\)|\"[^\"]*\"|'[^']*')+))?)? *(?:(?P,)|\n?) *", re.IGNORECASE) +VAR_RE = re.compile( + r" *(?P[a-zA-Z_0-9]+) *(?P(?:\((?P(?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\))? *(?:= *(?P(:?[^\"',()]+|\((?:[^()\"']+|\([^()\"']*\)|\"[^\"]*\"|'[^']*')*\)|\"[^\"]*\"|'[^']*')+))?)? *(?:(?P,)|\n?) *", + re.IGNORECASE, +) USE_PARSE_RE = re.compile( r" *use +(?P[a-zA-Z_][a-zA-Z_0-9]*)(?P *, *only *:)? *(?P.*)$", - flags=re.IGNORECASE) -commonUsesRe = re.compile( - "^#include *\"([^\"]*(cp_common_uses.f90|base_uses.f90))\"") -localNameRe = re.compile( - " *(?P[a-zA-Z_0-9]+)(?: *= *> *[a-zA-Z_0-9]+)? *$") + flags=re.IGNORECASE, +) +commonUsesRe = re.compile('^#include *"([^"]*(cp_common_uses.f90|base_uses.f90))"') +localNameRe = re.compile(" *(?P[a-zA-Z_0-9]+)(?: *= *> *[a-zA-Z_0-9]+)? *$") VAR_DECL_RE = re.compile( - r" *(?Pinteger(?: *\* *[0-9]+)?|logical|character(?: *\* *[0-9]+)?|real(?: *\* *[0-9]+)?|complex(?: *\* *[0-9]+)?|type) *(?P\((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*\))? *(?P(?: *, *[a-zA-Z_0-9]+(?: *\((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*\))?)+)? *(?P::)?(?P[^\n]+)\n?", re.IGNORECASE) # $ + r" *(?Pinteger(?: *\* *[0-9]+)?|logical|character(?: *\* *[0-9]+)?|real(?: *\* *[0-9]+)?|complex(?: *\* *[0-9]+)?|type) *(?P\((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*\))? *(?P(?: *, *[a-zA-Z_0-9]+(?: *\((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*\))?)+)? *(?P::)?(?P[^\n]+)\n?", + re.IGNORECASE, +) # $ INDENT_SIZE = 2 DECL_LINELENGTH = 100 DECL_OFFSET = 50 @@ -36,7 +40,7 @@ class CharFilter(object): def __init__(self, it): self._it = it - self._instring = '' + self._instring = "" def __iter__(self): return self @@ -44,13 +48,13 @@ class CharFilter(object): def __next__(self): """ python 3 version""" pos, char = next(self._it) - if not self._instring and char == '!': + if not self._instring and char == "!": raise StopIteration # detect start/end of a string if char == '"' or char == "'": if self._instring == char: - self._instring = '' + self._instring = "" elif not self._instring: self._instring = char @@ -62,13 +66,13 @@ class CharFilter(object): def next(self): """ python 2 version""" pos, char = self._it.next() - if not self._instring and char == '!': + if not self._instring and char == "!": raise StopIteration # detect start/end of a string if char == '"' or char == "'": if self._instring == char: - self._instring = '' + self._instring = "" elif not self._instring: self._instring = char @@ -96,7 +100,8 @@ class InputStream(object): lineRe = re.compile( # $ r"(?:(?P#.*\n?)| *(&)?(?P(?:!\$|[^&!\"']+|\"[^\"]*\"|'[^']*')*)(?P&)? *(?P!.*)?\n?)", - re.IGNORECASE) + re.IGNORECASE, + ) joinedLine = "" comments = [] lines = [] @@ -111,21 +116,25 @@ class InputStream(object): is_omp_conditional = False omp_indent = 0 if OMP_RE.match(line): - omp_indent = len(line) - len(line.lstrip(' ')) - line = OMP_RE.sub('', line, count=1) + omp_indent = len(line) - len(line.lstrip(" ")) + line = OMP_RE.sub("", line, count=1) is_omp_conditional = True line_start = 0 for pos, char in CharFilter(enumerate(line)): - if char == ';' or pos + 1 == len(line): - self.line_buffer.append(omp_indent * ' ' + '!$' * is_omp_conditional + - line[line_start:pos + 1]) + if char == ";" or pos + 1 == len(line): + self.line_buffer.append( + omp_indent * " " + + "!$" * is_omp_conditional + + line[line_start : pos + 1] + ) omp_indent = 0 is_omp_conditional = False line_start = pos + 1 - if(line_start < len(line)): + if line_start < len(line): # line + comment - self.line_buffer.append('!$' * is_omp_conditional + - line[line_start:]) + self.line_buffer.append( + "!$" * is_omp_conditional + line[line_start:] + ) if self.line_buffer: line = self.line_buffer.popleft() @@ -143,24 +152,26 @@ class InputStream(object): if m.group("preprocessor"): if len(lines) > 1: raise SyntaxError( - "continuation to a preprocessor line not supported " + repr(line)) + "continuation to a preprocessor line not supported " + + repr(line) + ) comments.append(line) break coreAtt = m.group("core") if OMP_RE.match(coreAtt) and joinedLine.strip(): # remove omp '!$' for line continuation - coreAtt = OMP_RE.sub('', coreAtt, count=1).lstrip() + coreAtt = OMP_RE.sub("", coreAtt, count=1).lstrip() joinedLine = joinedLine.rstrip("\n") + coreAtt if coreAtt and not coreAtt.isspace(): continuation = 0 if m.group("continue"): continuation = 1 - if line.lstrip().startswith('!') and not OMP_RE.search(line): - comments.append(line.rstrip('\n')) + if line.lstrip().startswith("!") and not OMP_RE.search(line): + comments.append(line.rstrip("\n")) elif m.group("comment"): comments.append(m.group("comment")) else: - comments.append('') + comments.append("") if not continuation: break return (joinedLine, comments, lines) @@ -168,41 +179,53 @@ class InputStream(object): def parseRoutine(inFile): """Parses a routine""" - logger = logging.getLogger('prettify-logger') + logger = logging.getLogger("prettify-logger") FCT_RE = re.compile( r"^([^\"'!]* )?FUNCTION\s+\w+\s*(\(.*\))?(\s*RESULT\s*\(\w+\))?\s*;?\s*$", - re.IGNORECASE) + re.IGNORECASE, + ) SUBR_RE = re.compile( - r"^([^\"'!]* )?SUBROUTINE\s+\w+\s*(\(.*\))?\s*;?\s*$", re.IGNORECASE) + r"^([^\"'!]* )?SUBROUTINE\s+\w+\s*(\(.*\))?\s*;?\s*$", re.IGNORECASE + ) endRe = re.compile(r" *end\s*(?:subroutine|function)", re.IGNORECASE) - startRoutineRe = re.compile(r"^([^\"'!]* )?(?Psubroutine|function) +(?P[a-zA-Z_][a-zA-Z_0-9]*) *(?:\((?P[^()]*)\))? *(?:result *\( *(?P[a-zA-Z_][a-zA-Z_0-9]*) *\))? *(?:bind *\([^()]+\))? *\n?", re.IGNORECASE) # $ - typeBeginRe = re.compile(r" *(?Pinteger(?: *\* *[0-9]+)?|logical|character(?: *\* *[0-9]+)?|real(?: *\* *[0-9]+)?|complex(?: *\* *[0-9]+)?|type)[,( ]", - re.IGNORECASE) + startRoutineRe = re.compile( + r"^([^\"'!]* )?(?Psubroutine|function) +(?P[a-zA-Z_][a-zA-Z_0-9]*) *(?:\((?P[^()]*)\))? *(?:result *\( *(?P[a-zA-Z_][a-zA-Z_0-9]*) *\))? *(?:bind *\([^()]+\))? *\n?", + re.IGNORECASE, + ) # $ + typeBeginRe = re.compile( + r" *(?Pinteger(?: *\* *[0-9]+)?|logical|character(?: *\* *[0-9]+)?|real(?: *\* *[0-9]+)?|complex(?: *\* *[0-9]+)?|type)[,( ]", + re.IGNORECASE, + ) attributeRe = re.compile( - r" *, *(?P[a-zA-Z_0-9]+) *(?:\( *(?P(?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\))? *", re.IGNORECASE) + r" *, *(?P[a-zA-Z_0-9]+) *(?:\( *(?P(?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\))? *", + re.IGNORECASE, + ) ignoreRe = re.compile(r" *(?:|implicit +none *)$", re.IGNORECASE) interfaceStartRe = re.compile(r" *interface *$", re.IGNORECASE) interfaceEndRe = re.compile(r" *end +interface *$", re.IGNORECASE) - routine = {'preRoutine': [], - 'core': [], - 'strippedCore': [], - 'begin': [], - 'end': [], - 'preDeclComments': [], - 'declarations': [], - 'declComments': [], - 'postDeclComments': [], - 'parsedDeclarations': [], - 'postRoutine': [], - 'kind': None, 'name': None, 'arguments': None, 'result': None, - 'interfaceCount': 0, - 'use': [] - } - includeRe = re.compile( - r"#? *include +[\"'](?P.+)[\"'] *$", re.IGNORECASE) + routine = { + "preRoutine": [], + "core": [], + "strippedCore": [], + "begin": [], + "end": [], + "preDeclComments": [], + "declarations": [], + "declComments": [], + "postDeclComments": [], + "parsedDeclarations": [], + "postRoutine": [], + "kind": None, + "name": None, + "arguments": None, + "result": None, + "interfaceCount": 0, + "use": [], + } + includeRe = re.compile(r"#? *include +[\"'](?P.+)[\"'] *$", re.IGNORECASE) stream = InputStream(inFile) while 1: (jline, _, lines) = stream.nextFortranLine() @@ -210,97 +233,110 @@ def parseRoutine(inFile): break if FCT_RE.match(jline) or SUBR_RE.match(jline): break - routine['preRoutine'].extend(lines) + routine["preRoutine"].extend(lines) m = includeRe.match(lines[0]) if m: try: - subF = open(m.group('file'), 'r') + subF = open(m.group("file"), "r") subStream = InputStream(subF) while 1: (subjline, _, sublines) = subStream.nextFortranLine() if not sublines: break - routine['strippedCore'].append(subjline) + routine["strippedCore"].append(subjline) subF.close() except: import traceback + + logger.debug("error trying to follow include " + m.group("file") + "\n") logger.debug( - "error trying to follow include " + m.group('file') + '\n') - logger.debug( - "warning this might lead to the removal of used variables\n") + "warning this might lead to the removal of used variables\n" + ) if logger.isEnabledFor(logging.DEBUG): traceback.print_exc() if jline: - routine['begin'] = lines + routine["begin"] = lines m = startRoutineRe.match(jline) if not m or m.span()[1] != len(jline): - raise SyntaxError( - "unexpected subroutine start format:" + repr(lines)) - routine['name'] = m.group('name') - routine['kind'] = m.group('kind') - if (m.group('arguments') and m.group('arguments').strip()): - routine['arguments'] = list(map(lambda x: x.strip(), - m.group('arguments').split(","))) - if (m.group('result')): - routine['result'] = m.group('result') - if (not routine['result'])and(routine['kind'].lower() == "function"): - routine['result'] = routine['name'] + raise SyntaxError("unexpected subroutine start format:" + repr(lines)) + routine["name"] = m.group("name") + routine["kind"] = m.group("kind") + if m.group("arguments") and m.group("arguments").strip(): + routine["arguments"] = list( + x.strip() for x in m.group("arguments").split(",") + ) + if m.group("result"): + routine["result"] = m.group("result") + if (not routine["result"]) and (routine["kind"].lower() == "function"): + routine["result"] = routine["name"] while 1: (jline, comment_list, lines) = stream.nextFortranLine() - comments = '\n'.join(_ for _ in comment_list) + comments = "\n".join(_ for _ in comment_list) if len(lines) == 0: break if lines[0].lower().startswith("#include"): break if not ignoreRe.match(jline): if typeBeginRe.match(jline): - if routine['postDeclComments']: - routine['declComments'].extend(routine['postDeclComments']) - routine['postDeclComments'] = [] + if routine["postDeclComments"]: + routine["declComments"].extend(routine["postDeclComments"]) + routine["postDeclComments"] = [] if typeBeginRe.match(jline): m = VAR_DECL_RE.match(jline) - if (m.group('type').lower() == 'type' and - not m.group('parameters')): + if m.group("type").lower() == "type" and not m.group("parameters"): break if not m or m.span()[1] != len(jline): raise SyntaxError("unexpected type format:" + repr(jline)) - decl = {'type': m.group("type"), - 'parameters': None, - 'attributes': [], - 'vars': []} - if m.group('parameters'): - decl['parameters'] = (m.group("parameters").replace(" ", ""). - replace(",", ", ")) + decl = { + "type": m.group("type"), + "parameters": None, + "attributes": [], + "vars": [], + } + if m.group("parameters"): + decl["parameters"] = ( + m.group("parameters").replace(" ", "").replace(",", ", ") + ) str = m.group("attributes") - while(str): + while str: m2 = attributeRe.match(str) if not m2: - raise SyntaxError("unexpected attribute format " + - repr(str) + " in " + repr(lines)) - decl['attributes'].append(m2.group().replace(" ", ""). - replace(",", ", ")[2:]) - str = str[m2.span()[1]:] + raise SyntaxError( + "unexpected attribute format " + + repr(str) + + " in " + + repr(lines) + ) + decl["attributes"].append( + m2.group().replace(" ", "").replace(",", ", ")[2:] + ) + str = str[m2.span()[1] :] str = m.group("vars") while 1: m2 = VAR_RE.match(str) if not m2: - raise SyntaxError("unexpected var format " + - repr(str) + " in " + repr(lines)) + raise SyntaxError( + "unexpected var format " + repr(str) + " in " + repr(lines) + ) var = m2.group("var") if m2.group("param"): var += "(" + m2.group("param") + ")" if m2.group("value"): var += " = " var += m2.group("value") - decl['vars'].append(var) - str = str[m2.span()[1]:] + decl["vars"].append(var) + str = str[m2.span()[1] :] if not m2.group("continue"): if str: - raise SyntaxError("error parsing vars (leftover=" + - repr(str) + ") in " + repr(lines)) + raise SyntaxError( + "error parsing vars (leftover=" + + repr(str) + + ") in " + + repr(lines) + ) break - routine['parsedDeclarations'].append(decl) + routine["parsedDeclarations"].append(decl) elif interfaceStartRe.match(jline): istart = lines interfaceDeclFile = StringIO() @@ -314,63 +350,69 @@ def parseRoutine(inFile): iroutines = [] while 1: iroutine = parseRoutine(interfaceDeclFile) - if not iroutine['kind']: + if not iroutine["kind"]: if len(iroutines) == 0: interfaceDeclFile.seek(0) - raise SyntaxError("error parsing interface:" + - repr(interfaceDeclFile.read())) - iroutines[-1]['postRoutine'].extend( - iroutine['preRoutine']) + raise SyntaxError( + "error parsing interface:" + + repr(interfaceDeclFile.read()) + ) + iroutines[-1]["postRoutine"].extend(iroutine["preRoutine"]) break iroutines.append(iroutine) for iroutine in iroutines: - routine['interfaceCount'] += 1 - decl = {'type': 'z_interface%02d' % (routine['interfaceCount']), - 'parameters': None, - 'attributes': [], - 'vars': [iroutine['name']], - 'iroutine': iroutine, - 'istart': istart, - 'iend': iend - } - routine['parsedDeclarations'].append(decl) + routine["interfaceCount"] += 1 + decl = { + "type": "z_interface%02d" % (routine["interfaceCount"]), + "parameters": None, + "attributes": [], + "vars": [iroutine["name"]], + "iroutine": iroutine, + "istart": istart, + "iend": iend, + } + routine["parsedDeclarations"].append(decl) elif USE_PARSE_RE.match(jline): - routine['use'].append("".join(lines)) + routine["use"].append("".join(lines)) else: break - routine['declarations'].append("".join(lines)) - if (len(routine['parsedDeclarations']) == 0 and len(routine['use']) == 0 and - not re.match(" *implicit +none *$", jline, re.IGNORECASE)): - routine['preDeclComments'].append("".join(lines)) + routine["declarations"].append("".join(lines)) + if ( + len(routine["parsedDeclarations"]) == 0 + and len(routine["use"]) == 0 + and not re.match(" *implicit +none *$", jline, re.IGNORECASE) + ): + routine["preDeclComments"].append("".join(lines)) else: - routine['postDeclComments'].append(comments) + routine["postDeclComments"].append(comments) containsRe = re.compile(r" *contains *$", re.IGNORECASE) while len(lines) > 0: if endRe.match(jline): - routine['end'] = lines + routine["end"] = lines break - routine['strippedCore'].append(jline) - routine['core'].append("".join(lines)) + routine["strippedCore"].append(jline) + routine["core"].append("".join(lines)) if containsRe.match(lines[0]): break m = includeRe.match(lines[0]) if m: try: - subF = open(m.group('file'), 'r') + subF = open(m.group("file"), "r") subStream = InputStream(subF) while 1: (subjline, _, sublines) = subStream.nextFortranLine() if not sublines: break - routine['strippedCore'].append(subjline) + routine["strippedCore"].append(subjline) subF.close() except: import traceback + + logger.debug("error trying to follow include " + m.group("file") + "\n") logger.debug( - "error trying to follow include " + m.group('file') + '\n') - logger.debug( - "warning this might lead to the removal of used variables\n") + "warning this might lead to the removal of used variables\n" + ) if logger.isEnabledFor(logging.DEBUG): traceback.print_exc() (jline, _, lines) = stream.nextFortranLine() @@ -381,8 +423,14 @@ def findWord(word, text, options=re.IGNORECASE): """Returns the position of word in text or -1 if not found. A match is valid only if it is a whole word (i.e. findWord('try','retry') returns false)""" - wordRe = re.compile("(? 100000: raise Error("could not enforce all constraints") - m = VAR_RE.match(declarations[idecl2]['vars'][ivar2]) - if (ivar == 0 and - findWord(m.group('var').lower(), typeParam) != -1): - declarations.insert( - idecl2 + 1, declarations[idecl]) + m = VAR_RE.match(declarations[idecl2]["vars"][ivar2]) + if ( + ivar == 0 + and findWord(m.group("var").lower(), typeParam) != -1 + ): + declarations.insert(idecl2 + 1, declarations[idecl]) del declarations[idecl] ivar = 0 moved = 1 break - if rest and findWord(m.group('var').lower(), rest) != -1: - if len(declarations[idecl]['vars']) > 1: + if rest and findWord(m.group("var").lower(), rest) != -1: + if len(declarations[idecl]["vars"]) > 1: newDecl = {} newDecl.update(declarations[idecl]) - newDecl['vars'] = [ - declarations[idecl]['vars'][ivar]] + newDecl["vars"] = [declarations[idecl]["vars"][ivar]] declarations.insert(idecl2 + 1, newDecl) - del declarations[idecl]['vars'][ivar] + del declarations[idecl]["vars"][ivar] else: - declarations.insert(idecl2 + 1, - declarations[idecl]) + declarations.insert(idecl2 + 1, declarations[idecl]) del declarations[idecl] ivar = 0 moved = 1 @@ -457,9 +506,11 @@ def enforceDeclDependecies(declarations): idecl += 1 for i in range(len(declarations) - 1, 0, -1): - if (declarations[i]['normalizedType'].lower() == - declarations[i - 1]['normalizedType'].lower()): - declarations[i - 1]['vars'].extend(declarations[i]['vars']) + if ( + declarations[i]["normalizedType"].lower() + == declarations[i - 1]["normalizedType"].lower() + ): + declarations[i - 1]["vars"].extend(declarations[i]["vars"]) del declarations[i] @@ -467,16 +518,18 @@ def sortDeclarations(declarations): """sorts, compacts declarations and respects dependencies normalizedType has to be defined for the declarations""" - declarations.sort(key=lambda x: x['normalizedType'].lower()) + declarations.sort(key=lambda x: x["normalizedType"].lower()) for i in range(len(declarations) - 1, 0, -1): - if (declarations[i]['normalizedType'].lower() == - declarations[i - 1]['normalizedType'].lower()): - declarations[i - 1]['vars'].extend(declarations[i]['vars']) + if ( + declarations[i]["normalizedType"].lower() + == declarations[i - 1]["normalizedType"].lower() + ): + declarations[i - 1]["vars"].extend(declarations[i]["vars"]) del declarations[i] for decl in declarations: - decl['vars'].sort(key=lambda x: x.lower()) + decl["vars"].sort(key=lambda x: x.lower()) enforceDeclDependecies(declarations) @@ -532,29 +585,28 @@ def writeInCols(dLine, indentCol, maxCol, indentAtt, file): def writeCompactDeclaration(declaration, file): """Writes a declaration in a compact way""" d = declaration - if 'iroutine' in d.keys(): - file.writelines(d['istart']) - writeRoutine(d['iroutine'], file) - file.writelines(d['iend']) + if "iroutine" in d.keys(): + file.writelines(d["istart"]) + writeRoutine(d["iroutine"], file) + file.writelines(d["iend"]) else: - if len(d['vars']) > 0: - decl = " " * INDENT_SIZE * 2 + d['type'] - if d['parameters']: # do not drop empty parameter lists? - decl += d['parameters'] - if d['attributes']: - for a in d['attributes']: + if len(d["vars"]) > 0: + decl = " " * INDENT_SIZE * 2 + d["type"] + if d["parameters"]: # do not drop empty parameter lists? + decl += d["parameters"] + if d["attributes"]: + for a in d["attributes"]: decl += ", " + a decl += " :: " dLine = [decl] - for var in d['vars']: + for var in d["vars"]: cur_len = sum([len(l) for l in dLine]) - if(len(dLine) > 1 and cur_len + len(var) > 600): - writeInCols(dLine, 3 * INDENT_SIZE, - DECL_LINELENGTH, 0, file) + if len(dLine) > 1 and cur_len + len(var) > 600: + writeInCols(dLine, 3 * INDENT_SIZE, DECL_LINELENGTH, 0, file) file.write("\n") dLine = [decl] - if(len(dLine) > 1): + if len(dLine) > 1: dLine[-1] += ", " dLine.append(var) writeInCols(dLine, 3 * INDENT_SIZE, DECL_LINELENGTH, 0, file) @@ -564,35 +616,37 @@ def writeCompactDeclaration(declaration, file): def writeExtendedDeclaration(declaration, file): """Writes a declaration in a nicer way (using more space)""" d = declaration - if len(d['vars']) == 0: + if len(d["vars"]) == 0: return - if 'iroutine' in d.keys(): - file.writelines(d['istart']) - writeRoutine(d['iroutine'], file) - file.writelines(d['iend']) + if "iroutine" in d.keys(): + file.writelines(d["istart"]) + writeRoutine(d["iroutine"], file) + file.writelines(d["iend"]) else: dLine = [] - dLine.append(" " * INDENT_SIZE * 2 + d['type']) - if d['parameters']: # do not drop empty parameter lists? - dLine.append(d['parameters']) - if d['attributes']: - for a in d['attributes']: + dLine.append(" " * INDENT_SIZE * 2 + d["type"]) + if d["parameters"]: # do not drop empty parameter lists? + dLine.append(d["parameters"]) + if d["attributes"]: + for a in d["attributes"]: dLine[-1:] = [dLine[-1] + ", "] dLine.append(a) - indentAtt = writeInCols(dLine, 3 * INDENT_SIZE, - DECL_OFFSET + 1 + 2 * INDENT_SIZE, 0, file) + indentAtt = writeInCols( + dLine, 3 * INDENT_SIZE, DECL_OFFSET + 1 + 2 * INDENT_SIZE, 0, file + ) file.write(" " * (DECL_OFFSET + 2 * INDENT_SIZE - indentAtt)) file.write(" :: ") indentAtt = DECL_OFFSET + 8 dLine = [] - for var in d['vars'][:-1]: + for var in d["vars"][:-1]: dLine.append(var + ", ") - dLine.append(d['vars'][-1]) + dLine.append(d["vars"][-1]) - writeInCols(dLine, DECL_OFFSET + 4 + 2 * INDENT_SIZE, - DECL_LINELENGTH, indentAtt, file) + writeInCols( + dLine, DECL_OFFSET + 4 + 2 * INDENT_SIZE, DECL_LINELENGTH, indentAtt, file + ) file.write("\n") @@ -601,7 +655,7 @@ def writeDeclarations(parsedDeclarations, file): for d in parsedDeclarations: maxLenVar = 0 totalLen = 0 - for v in d['vars']: + for v in d["vars"]: maxLenVar = max(maxLenVar, len(v)) totalLen += len(v) if maxLenVar > 30 or totalLen > DECL_LINELENGTH - 4: @@ -613,48 +667,58 @@ def writeDeclarations(parsedDeclarations, file): def cleanDeclarations(routine): """cleans up the declaration part of the given parsed routine removes unused variables""" - logger = logging.getLogger('prettify-logger') + logger = logging.getLogger("prettify-logger") global R_VAR containsRe = re.compile(r" *contains *$", re.IGNORECASE) - if routine['core']: - if containsRe.match(routine['core'][-1]): - logger.debug("routine %s contains other routines\ndeclarations not cleaned\n" % - (routine['name'])) + if routine["core"]: + if containsRe.match(routine["core"][-1]): + logger.debug( + "routine %s contains other routines\ndeclarations not cleaned\n" + % (routine["name"]) + ) return commentToRemoveRe = re.compile( - r" *! *(?:interface|arguments|parameters|locals?|\** *local +variables *\**|\** *local +parameters *\**) *$", re.IGNORECASE) + r" *! *(?:interface|arguments|parameters|locals?|\** *local +variables *\**|\** *local +parameters *\**) *$", + re.IGNORECASE, + ) nullifyRe = re.compile( - r" *nullify *\(([^()]+)\) *\n?", re.IGNORECASE | re.MULTILINE) + r" *nullify *\(([^()]+)\) *\n?", re.IGNORECASE | re.MULTILINE + ) - if not routine['kind']: + if not routine["kind"]: return - if (routine['core']): - if re.match(" *type *[a-zA-Z_]+ *$", routine['core'][0], re.IGNORECASE): - logger.debug("routine %s contains local types, not fully cleaned\n" % - (routine['name'])) - if re.match(" *import+ *$", routine['core'][0], re.IGNORECASE): - logger.debug("routine %s contains import, not fully cleaned\n" % - (routine['name'])) - if re.search("^#", "".join(routine['declarations']), re.MULTILINE): - logger.debug("routine %s declarations contain preprocessor directives\ndeclarations not cleaned\n" % ( - routine['name'])) + if routine["core"]: + if re.match(" *type *[a-zA-Z_]+ *$", routine["core"][0], re.IGNORECASE): + logger.debug( + "routine %s contains local types, not fully cleaned\n" + % (routine["name"]) + ) + if re.match(" *import+ *$", routine["core"][0], re.IGNORECASE): + logger.debug( + "routine %s contains import, not fully cleaned\n" % (routine["name"]) + ) + if re.search("^#", "".join(routine["declarations"]), re.MULTILINE): + logger.debug( + "routine %s declarations contain preprocessor directives\ndeclarations not cleaned\n" + % (routine["name"]) + ) return try: - rest = "".join(routine['strippedCore']).lower() + rest = "".join(routine["strippedCore"]).lower() nullifys = ",".join(nullifyRe.findall(rest)) rest = nullifyRe.sub("", rest) paramDecl = [] decls = [] - for d in routine['parsedDeclarations']: - d['normalizedType'] = d['type'] - if d['parameters']: - d['normalizedType'] += d['parameters'] - if (d["attributes"]): - d['attributes'].sort(key=lambda x: x.lower()) - d['normalizedType'] += ', ' - d['normalizedType'] += ', '.join(d['attributes']) - if "parameter" in map(str.lower, d['attributes']): + for d in routine["parsedDeclarations"]: + d["normalizedType"] = d["type"] + if d["parameters"]: + d["normalizedType"] += d["parameters"] + if d["attributes"]: + d["attributes"].sort(key=lambda x: x.lower()) + d["normalizedType"] += ", " + d["normalizedType"] += ", ".join(d["attributes"]) + if any(a.lower() == "parameter" for a in d["attributes"]): paramDecl.append(d) else: decls.append(d) @@ -664,69 +728,83 @@ def cleanDeclarations(routine): has_routinen = 0 pos_routinep = -1 for d in paramDecl: - for i in range(len(d['vars'])): - v = d['vars'][i] + for i in range(len(d["vars"])): + v = d["vars"][i] m = VAR_RE.match(v) lowerV = m.group("var").lower() if lowerV == "routinen": has_routinen = 1 - d['vars'][i] = "routineN = '" + routine['name'] + "'" + d["vars"][i] = "routineN = '" + routine["name"] + "'" elif lowerV == "routinep": pos_routinep = i - d['vars'][i] = "routineP = moduleN//':'//routineN" + d["vars"][i] = "routineP = moduleN//':'//routineN" if not has_routinen and pos_routinep >= 0: - d['vars'].insert( - pos_routinep, "routineN = '" + routine['name'] + "'") + d["vars"].insert(pos_routinep, "routineN = '" + routine["name"] + "'") - if routine['arguments']: - routine['lowercaseArguments'] = list(map( - lambda x: x.lower(), routine['arguments'])) + if routine["arguments"]: + routine["lowercaseArguments"] = list( + x.lower() for x in routine["arguments"] + ) else: - routine['lowercaseArguments'] = [] - if routine['result']: - routine['lowercaseArguments'].append(routine['result'].lower()) + routine["lowercaseArguments"] = [] + if routine["result"]: + routine["lowercaseArguments"].append(routine["result"].lower()) argDeclDict = {} localDecl = [] for d in decls: localD = {} localD.update(d) - localD['vars'] = [] + localD["vars"] = [] argD = None - for v in d['vars']: + for v in d["vars"]: m = VAR_RE.match(v) lowerV = m.group("var").lower() - if lowerV in routine['lowercaseArguments']: + if lowerV in routine["lowercaseArguments"]: argD = {} argD.update(d) - argD['vars'] = [v] + argD["vars"] = [v] if lowerV in argDeclDict.keys(): raise SyntaxError( - "multiple declarations not supported. var=" + v + - " declaration=" + str(d) + "routine=" + routine['name']) + "multiple declarations not supported. var=" + + v + + " declaration=" + + str(d) + + "routine=" + + routine["name"] + ) argDeclDict[lowerV] = argD else: pos = findWord(lowerV, rest) - if (pos != -1): - localD['vars'].append(v) + if pos != -1: + localD["vars"].append(v) else: if findWord(lowerV, nullifys) != -1: - if not rmNullify(lowerV, routine['core']): + if not rmNullify(lowerV, routine["core"]): raise SyntaxError( - "could not remove nullify of " + lowerV + - " as expected, routine=" + routine['name']) - logger.info("removed var %s in routine %s\n" % - (lowerV, routine['name'])) + "could not remove nullify of " + + lowerV + + " as expected, routine=" + + routine["name"] + ) + logger.info( + "removed var %s in routine %s\n" % (lowerV, routine["name"]) + ) R_VAR += 1 - if (len(localD['vars'])): + if len(localD["vars"]): localDecl.append(localD) argDecl = [] - for arg in routine['lowercaseArguments']: + for arg in routine["lowercaseArguments"]: if arg in argDeclDict.keys(): argDecl.append(argDeclDict[arg]) else: - logger.debug("warning, implicitly typed argument '" + - arg + "' in routine " + routine['name'] + '\n') - if routine['kind'].lower() == 'function': + logger.debug( + "warning, implicitly typed argument '" + + arg + + "' in routine " + + routine["name"] + + "\n" + ) + if routine["kind"].lower() == "function": aDecl = argDecl[:-1] else: aDecl = argDecl @@ -737,7 +815,7 @@ def cleanDeclarations(routine): enforceDeclDependecies(argDecl) splitPos = 0 for i in range(len(argDecl) - 1, -1, -1): - if not 'parameter' in map(str.lower, argDecl[i]['attributes']): + if not any(a.lower() == "parameter" for a in argDecl[i]["attributes"]): splitPos = i + 1 break paramDecl = argDecl[splitPos:] @@ -746,17 +824,17 @@ def cleanDeclarations(routine): enforceDeclDependecies(paramDecl) splitPos = 0 for i in range(len(paramDecl) - 1, -1, -1): - if 'parameter' in map(str.lower, paramDecl[i]['attributes']): + if any(a.lower() == "parameter" for a in paramDecl[i]["attributes"]): splitPos = i + 1 break localDecl = paramDecl[splitPos:] paramDecl = paramDecl[:splitPos] newDecl = StringIO() - for comment in routine['preDeclComments']: + for comment in routine["preDeclComments"]: if not commentToRemoveRe.match(comment): newDecl.write(comment) - newDecl.writelines(routine['use']) + newDecl.writelines(routine["use"]) writeDeclarations(argDecl, newDecl) if argDecl and paramDecl: newDecl.write("\n") @@ -767,36 +845,34 @@ def cleanDeclarations(routine): if argDecl or paramDecl or localDecl: newDecl.write("\n") wrote = 0 - for comment in routine['declComments']: + for comment in routine["declComments"]: if comment.strip() and not commentToRemoveRe.match(comment): newDecl.write(comment.strip()) newDecl.write("\n") wrote = 1 if wrote: newDecl.write("\n") - routine['declarations'] = [newDecl.getvalue()] + routine["declarations"] = [newDecl.getvalue()] except: - if 'name' in routine.keys(): - logger.critical("exception cleaning routine " + - routine['name']) - logger.critical("parsedDeclartions=" + - str(routine['parsedDeclarations'])) + if "name" in routine.keys(): + logger.critical("exception cleaning routine " + routine["name"]) + logger.critical("parsedDeclartions=" + str(routine["parsedDeclarations"])) raise newDecl = StringIO() - if routine['postDeclComments']: + if routine["postDeclComments"]: comment_start = 0 - for comment in routine['postDeclComments']: + for comment in routine["postDeclComments"]: if comment.strip(): break else: comment_start += 1 - for comment in routine['postDeclComments'][comment_start:]: + for comment in routine["postDeclComments"][comment_start:]: if not commentToRemoveRe.match(comment): newDecl.write(comment) newDecl.write("\n") - routine['declarations'][0] += newDecl.getvalue() + routine["declarations"][0] += newDecl.getvalue() def rmNullify(var, strings): @@ -804,7 +880,8 @@ def rmNullify(var, strings): var = var.lower() nullifyRe = re.compile(r" *nullify *\(", re.IGNORECASE) nullify2Re = re.compile( - r"(?P *nullify *\()(?P[^()!&]+)\)", re.IGNORECASE) + r"(?P *nullify *\()(?P[^()!&]+)\)", re.IGNORECASE + ) for i in range(len(strings) - 1, -1, -1): line = strings[i] @@ -827,11 +904,12 @@ def rmNullify(var, strings): comments.append(l[pos2:] + "\n") m = nullify2Re.match(core) if not m: - raise SyntaxError("could not match nullify to " + repr(core) + - "in" + repr(line)) + raise SyntaxError( + "could not match nullify to " + repr(core) + "in" + repr(line) + ) allVars = [] vars = m.group("vars") - v = list(map(str.strip, vars.split(","))) + v = list(s.strip() for s in vars.split(",")) removedNow = 0 for j in range(len(v) - 1, -1, -1): if findWord(var, v[j].lower()) != -1: @@ -849,8 +927,7 @@ def rmNullify(var, strings): v[-1] += ")" newS = StringIO() v.insert(0, m.group("nullif")) - writeInCols(v, len(v[0]) - - len(v[0].lstrip()) + 5, 77, 0, newS) + writeInCols(v, len(v[0]) - len(v[0].lstrip()) + 5, 77, 0, newS) newS.write("\n") if comments: for c in comments: @@ -876,7 +953,7 @@ def parseUse(inFile): stream = InputStream(inFile) while 1: (jline, comment_list, lines) = stream.nextFortranLine() - comments = '\n'.join(_ for _ in comment_list if _) + comments = "\n".join(_ for _ in comment_list if _) lineNr = lineNr + len(lines) if not lines: break @@ -884,18 +961,18 @@ def parseUse(inFile): # parse use m = USE_PARSE_RE.match(jline) if m: - useAtt = {'module': m.group('module'), 'comments': []} + useAtt = {"module": m.group("module"), "comments": []} - if m.group('only'): - useAtt['only'] = list(map(str.strip, - str.split(m.group('imports'), ','))) + if m.group("only"): + useAtt["only"] = list(s.strip() for s in m.group("imports").split(",")) else: - useAtt['renames'] = list(map(str.strip, - str.split(m.group('imports'), ','))) - if useAtt['renames'] == [""]: - del useAtt['renames'] + useAtt["renames"] = list( + s.strip() for s in m.group("imports").split(",") + ) + if useAtt["renames"] == [""]: + del useAtt["renames"] if comments: - useAtt['comments'].append(comments) + useAtt["comments"].append(comments) # add use to modules modules.append(useAtt) elif jline and not jline.isspace(): @@ -906,40 +983,46 @@ def parseUse(inFile): elif len(modules) == 0: preComments.append(("".join(lines))) elif comments: - modules[-1]['comments'].append(comments) + modules[-1]["comments"].append(comments) - return {'modules': modules, 'preComments': preComments, 'commonUses': commonUses, - 'postLine': "".join(lines), 'origLines': origLines[:-1]} + return { + "modules": modules, + "preComments": preComments, + "commonUses": commonUses, + "postLine": "".join(lines), + "origLines": origLines[:-1], + } def normalizeModules(modules): """Sorts the modules and their export and removes duplicates. renames aren't sorted correctly""" # orders modules - modules.sort(key=lambda x: x['module']) + modules.sort(key=lambda x: x["module"]) for i in range(len(modules) - 1, 0, -1): - if modules[i]['module'].lower() == modules[i - 1]['module'].lower(): - if not ('only' in modules[i - 1].keys() and - 'only' in modules[i].keys()): - raise SyntaxError('rejoining of module ' + - str(modules[i]['module']) + - ' failed as at least one of the use is not a use ...,only:') - modules[i - 1]['only'].extend(modules[i]['only']) + if modules[i]["module"].lower() == modules[i - 1]["module"].lower(): + if not ("only" in modules[i - 1].keys() and "only" in modules[i].keys()): + raise SyntaxError( + "rejoining of module " + + str(modules[i]["module"]) + + " failed as at least one of the use is not a use ...,only:" + ) + modules[i - 1]["only"].extend(modules[i]["only"]) del modules[i] # orders imports for m in modules: - if 'only' in m.keys(): - m['only'].sort() - for i in range(len(m['only']) - 1, 0, -1): - if m['only'][i - 1].lower() == m['only'][i].lower(): - del m['only'][i] + if "only" in m.keys(): + m["only"].sort() + for i in range(len(m["only"]) - 1, 0, -1): + if m["only"][i - 1].lower() == m["only"][i].lower(): + del m["only"][i] def writeUses(modules, outFile): """Writes the use declaration using a long or short form depending on how many only statements there are""" for m in modules: - if 'only' in m.keys() and len(m['only']) > 8: + if "only" in m.keys() and len(m["only"]) > 8: writeUseShort(m, outFile) else: writeUseLong(m, outFile) @@ -947,49 +1030,56 @@ def writeUses(modules, outFile): def writeUseLong(m, outFile): """Writes a use declaration in a nicer, but longer way""" - if 'only' in m.keys(): - outFile.write(INDENT_SIZE * ' ' + "USE " + m['module'] + "," + - str.rjust('ONLY: ', 38 - len(m['module']))) - if m['only']: - outFile.write(m['only'][0]) - for i in range(1, len(m['only'])): - outFile.write(",&\n" + str.ljust("", 43 + - INDENT_SIZE) + m['only'][i]) + if "only" in m.keys(): + outFile.write( + INDENT_SIZE * " " + + "USE " + + m["module"] + + "," + + "ONLY: ".rjust(38 - len(m["module"])) + ) + if m["only"]: + outFile.write(m["only"][0]) + for i in range(1, len(m["only"])): + outFile.write(",&\n" + "".ljust(43 + INDENT_SIZE) + m["only"][i]) else: - outFile.write(INDENT_SIZE * ' ' + "USE " + m['module']) - if 'renames' in m.keys() and m['renames']: - outFile.write("," + str.ljust("", 38) + - m['renames'][0]) - for i in range(1, len(m['renames'])): - outFile.write(",&\n" + str.ljust("", 43 + - INDENT_SIZE) + m['renames'][i]) - if m['comments']: + outFile.write(INDENT_SIZE * " " + "USE " + m["module"]) + if "renames" in m.keys() and m["renames"]: + outFile.write("," + "".ljust(38) + m["renames"][0]) + for i in range(1, len(m["renames"])): + outFile.write(",&\n" + "".ljust(43 + INDENT_SIZE) + m["renames"][i]) + if m["comments"]: outFile.write("\n") - outFile.write('\n'.join(m['comments'])) + outFile.write("\n".join(m["comments"])) outFile.write("\n") def writeUseShort(m, file): """Writes a use declaration in a compact way""" uLine = [] - if 'only' in m.keys(): - file.write(INDENT_SIZE * ' ' + "USE " + m['module'] + "," + - str.rjust('ONLY: &\n', 40 - len(m['module']))) - for k in m['only'][:-1]: + if "only" in m.keys(): + file.write( + INDENT_SIZE * " " + + "USE " + + m["module"] + + "," + + "ONLY: &\n".rjust(40 - len(m["module"])) + ) + for k in m["only"][:-1]: uLine.append(k + ", ") - uLine.append(m['only'][-1]) + uLine.append(m["only"][-1]) uLine[0] = " " * (5 + INDENT_SIZE) + uLine[0] - elif 'renames' in m.keys() and m['renames']: - uLine.append(INDENT_SIZE * ' ' + "USE " + m['module'] + ", ") - for k in m['renames'][:-1]: + elif "renames" in m.keys() and m["renames"]: + uLine.append(INDENT_SIZE * " " + "USE " + m["module"] + ", ") + for k in m["renames"][:-1]: uLine.append(k + ", ") - uLine.append(m['renames'][-1]) + uLine.append(m["renames"][-1]) else: - uLine.append(INDENT_SIZE * ' ' + "USE " + m['module']) + uLine.append(INDENT_SIZE * " " + "USE " + m["module"]) writeInCols(uLine, 5 + INDENT_SIZE, DECL_LINELENGTH, 0, file) - if m['comments']: + if m["comments"]: file.write("\n") - file.write('\n'.join(m['comments'])) + file.write("\n".join(m["comments"])) file.write("\n") @@ -999,81 +1089,90 @@ def prepareImplicitUses(modules): wich is true if the whole mosule is implicitly present""" mods = {} for m in modules: - m_name = m['module'].lower() + m_name = m["module"].lower() if m_name not in mods.keys(): - mods[m['module']] = {'_WHOLE_': 0} + mods[m["module"]] = {"_WHOLE_": 0} m_att = mods[m_name] - if 'only' in m.keys(): - for k in m['only']: + if "only" in m.keys(): + for k in m["only"]: m = localNameRe.match(k) if not m: - raise SyntaxError('could not parse use only:' + repr(k)) - impAtt = m.group('localName').lower() + raise SyntaxError("could not parse use only:" + repr(k)) + impAtt = m.group("localName").lower() m_att[impAtt] = 1 else: - m_att['_WHOLE_'] = 1 + m_att["_WHOLE_"] = 1 return mods def cleanUse(modulesDict, rest, implicitUses=None): """Removes the unneded modules (the ones that are not used in rest)""" - logger = logging.getLogger('prettify-logger') + logger = logging.getLogger("prettify-logger") global R_USE exceptions = {} - modules = modulesDict['modules'] + modules = modulesDict["modules"] rest = rest.lower() for i in range(len(modules) - 1, -1, -1): m_att = {} - m_name = modules[i]['module'].lower() + m_name = modules[i]["module"].lower() if implicitUses and m_name in implicitUses.keys(): m_att = implicitUses[m_name] - if '_WHOLE_' in m_att.keys() and m_att['_WHOLE_']: + if "_WHOLE_" in m_att.keys() and m_att["_WHOLE_"]: R_USE += 1 logger.info("removed USE of module " + m_name + "\n") del modules[i] - elif 'only' in modules[i].keys(): - els = modules[i]['only'] + elif "only" in modules[i].keys(): + els = modules[i]["only"] for j in range(len(els) - 1, -1, -1): m = localNameRe.match(els[j]) if not m: - raise SyntaxError( - 'could not parse use only:' + repr(els[j])) - impAtt = m.group('localName').lower() + raise SyntaxError("could not parse use only:" + repr(els[j])) + impAtt = m.group("localName").lower() if impAtt in m_att.keys(): R_USE += 1 - logger.info("removed USE " + m_name + - ", only: " + repr(els[j]) + "\n") + logger.info( + "removed USE " + m_name + ", only: " + repr(els[j]) + "\n" + ) del els[j] elif impAtt not in exceptions.keys(): if findWord(impAtt, rest) == -1: R_USE += 1 - logger.info("removed USE " + m_name + - ", only: " + repr(els[j]) + "\n") + logger.info( + "removed USE " + m_name + ", only: " + repr(els[j]) + "\n" + ) del els[j] - if len(modules[i]['only']) == 0: - if modules[i]['comments']: - modulesDict['preComments'].extend( - list(map(lambda x: x + "\n", modules[i]['comments']))) + if len(modules[i]["only"]) == 0: + if modules[i]["comments"]: + modulesDict["preComments"] += [ + x + "\n" for x in modules[i]["comments"] + ] del modules[i] def resetModuleN(moduleName, lines): "resets the moduleN variable to the module name in the lines lines" - moduleNRe = re.compile(r".*:: *moduleN *= *(['\"])[a-zA-Z_0-9]+\1", - flags=re.IGNORECASE) + moduleNRe = re.compile( + r".*:: *moduleN *= *(['\"])[a-zA-Z_0-9]+\1", flags=re.IGNORECASE + ) for i in range(len(lines)): lines[i] = moduleNRe.sub( - " " * INDENT_SIZE + "CHARACTER(len=*), PARAMETER, PRIVATE :: moduleN = '" + - moduleName + "'", - lines[i]) + " " * INDENT_SIZE + + "CHARACTER(len=*), PARAMETER, PRIVATE :: moduleN = '" + + moduleName + + "'", + lines[i], + ) -def rewriteFortranFile(inFile, outFile, indent, decl_linelength, decl_offset, orig_filename=None): +def rewriteFortranFile( + inFile, outFile, indent, decl_linelength, decl_offset, orig_filename=None +): """rewrites the use statements and declarations of inFile to outFile. It sorts them and removes the repetitions.""" import os.path - logger = logging.getLogger('prettify-logger') + + logger = logging.getLogger("prettify-logger") global INDENT_SIZE global DECL_OFFSET @@ -1082,14 +1181,16 @@ def rewriteFortranFile(inFile, outFile, indent, decl_linelength, decl_offset, or DECL_OFFSET = decl_offset DECL_LINELENGTH = decl_linelength - moduleRe = re.compile(r" *(?:module|program) +(?P[a-zA-Z_][a-zA-Z_0-9]*) *(?:!.*)?$", - flags=re.IGNORECASE) + moduleRe = re.compile( + r" *(?:module|program) +(?P[a-zA-Z_][a-zA-Z_0-9]*) *(?:!.*)?$", + flags=re.IGNORECASE, + ) coreLines = [] while 1: line = inFile.readline() if not line: break - if line[0] == '#': + if line[0] == "#": coreLines.append(line) outFile.write(line) m = moduleRe.match(line) @@ -1102,64 +1203,65 @@ def rewriteFortranFile(inFile, outFile, indent, decl_linelength, decl_offset, or try: modulesDict = parseUse(inFile) routines = [] - coreLines.append(modulesDict['postLine']) + coreLines.append(modulesDict["postLine"]) routine = parseRoutine(inFile) - coreLines.extend(routine['preRoutine']) + coreLines.extend(routine["preRoutine"]) if m: - resetModuleN(m.group('moduleName'), routine['preRoutine']) + resetModuleN(m.group("moduleName"), routine["preRoutine"]) routines.append(routine) - while routine['kind']: + while routine["kind"]: routine = parseRoutine(inFile) routines.append(routine) for routine in routines: cleanDeclarations(routine) # in-place modification of 'routine' - coreLines.extend(routine['declarations']) - coreLines.extend(routine['strippedCore']) + coreLines.extend(routine["declarations"]) + coreLines.extend(routine["strippedCore"]) rest = "".join(coreLines) nonStPrep = 0 - for line in modulesDict['origLines']: - if (re.search('^#', line) and not commonUsesRe.match(line)): - logger.debug('noMatch ' + repr(line) + - '\n') # what does it mean? + for line in modulesDict["origLines"]: + if re.search("^#", line) and not commonUsesRe.match(line): + logger.debug("noMatch " + repr(line) + "\n") # what does it mean? nonStPrep = 1 if nonStPrep: logger.debug( - "use statements contains preprocessor directives, not cleaning\n") - outFile.writelines(modulesDict['origLines']) + "use statements contains preprocessor directives, not cleaning\n" + ) + outFile.writelines(modulesDict["origLines"]) else: implicitUses = None - if modulesDict['commonUses']: + if modulesDict["commonUses"]: try: - inc_fn = commonUsesRe.match( - modulesDict['commonUses']).group(1) - inc_absfn = os.path.join( - os.path.dirname(orig_filename), inc_fn) - f = open(inc_absfn, 'r') + inc_fn = commonUsesRe.match(modulesDict["commonUses"]).group(1) + inc_absfn = os.path.join(os.path.dirname(orig_filename), inc_fn) + f = open(inc_absfn, "r") implicitUsesRaw = parseUse(f) f.close() - implicitUses = prepareImplicitUses( - implicitUsesRaw['modules']) + implicitUses = prepareImplicitUses(implicitUsesRaw["modules"]) except: logger.critical( - "ERROR trying to parse use statements contained in common uses precompiler file " + inc_absfn + '\n') + "ERROR trying to parse use statements contained in common uses precompiler file " + + inc_absfn + + "\n" + ) raise - cleanUse(modulesDict, rest, - implicitUses=implicitUses) - normalizeModules(modulesDict['modules']) - outFile.writelines(modulesDict['preComments']) - writeUses(modulesDict['modules'], outFile) - outFile.write(modulesDict['commonUses']) - if modulesDict['modules']: - outFile.write('\n') - outFile.write(modulesDict['postLine']) + cleanUse(modulesDict, rest, implicitUses=implicitUses) + normalizeModules(modulesDict["modules"]) + outFile.writelines(modulesDict["preComments"]) + writeUses(modulesDict["modules"], outFile) + outFile.write(modulesDict["commonUses"]) + if modulesDict["modules"]: + outFile.write("\n") + outFile.write(modulesDict["postLine"]) for routine in routines: writeRoutine(routine, outFile) except: import traceback - logger.critical('-' * 60 + "\n") + + logger.critical("-" * 60 + "\n") traceback.print_exc(file=sys.stderr) - logger.critical('-' * 60 + "\n") + logger.critical("-" * 60 + "\n") logger.critical("Processing file '" + orig_filename + "'\n") raise + # EOF diff --git a/tools/prettify/prettify_test.py b/tools/prettify/prettify_test.py index 6011974912..ac6accb3e5 100755 --- a/tools/prettify/prettify_test.py +++ b/tools/prettify/prettify_test.py @@ -11,6 +11,7 @@ import sys from prettify import main from prettify_cp2k import selftest + class TestSingleFileFolder(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() @@ -26,7 +27,7 @@ class TestSingleFileFolder(unittest.TestCase): def test_prettify(self): # call prettify, the return value should be 0 (OK) - self.assertEqual(main([sys.argv[0], self.fname]), 0) + self.assertEqual(main([self.fname]), 0) # check if file was altered (it shouldn't) with open(self.fname) as fhandle: @@ -34,5 +35,6 @@ class TestSingleFileFolder(unittest.TestCase): self.assertEqual(result.splitlines(), selftest.content.splitlines()) -if __name__ == '__main__': + +if __name__ == "__main__": unittest.main()