From 20e0d61ddc3a44c04b655856dba3fef2ad60e4df Mon Sep 17 00:00:00 2001 From: Patrick Seewald Date: Mon, 14 Mar 2016 14:40:03 +0000 Subject: [PATCH] Prettify update (defaults for make pretty still unchanged): - auto-alignment and whitespace-formatting can be disabled for specific lines - full reformatting of OpenMP-conditional Fortran lines - basic reformatting of OpenMP directives (upper case notation for all OMP keywords) - some cleanups and replacing workarounds by proper solutions - updates of docu and command line interface svn-origin-rev: 16692 --- tools/formatting/normalizeFortranFile.py | 56 ++-- ...lizeExtended.py => reformatFortranFile.py} | 274 +++++++++++------- tools/prettify.py | 94 ++++-- 3 files changed, 282 insertions(+), 142 deletions(-) rename tools/formatting/{normalizeExtended.py => reformatFortranFile.py} (74%) diff --git a/tools/formatting/normalizeFortranFile.py b/tools/formatting/normalizeFortranFile.py index 14e6d53290..15cfae8987 100644 --- a/tools/formatting/normalizeFortranFile.py +++ b/tools/formatting/normalizeFortranFile.py @@ -19,6 +19,9 @@ localNameRe=re.compile(" *(?P[a-zA-Z_0-9]+)(?: *= *> *[a-zA-Z_0-9]+)? typeRe=re.compile(r" *(?Pinteger(?: *\* *[0-9]+)?|logical|character(?: *\* *[0-9]+)?|real(?: *\* *[0-9]+)?|complex(?: *\* *[0-9]+)?|type) *(?P\((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*\))? *(?P(?: *, *[a-zA-Z_0-9]+(?: *\((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*\))?)+)? *(?P::)?(?P[^\n]+)\n?",re.IGNORECASE)#$ indentSize=2 +ompDirRe = re.compile(r"^\s*(!\$omp)",re.IGNORECASE) +ompRe = re.compile(r"^\s*(!\$)", re.IGNORECASE) + class CharFilter(object): """ An iterator to wrap the iterator returned by `enumerate` @@ -71,7 +74,7 @@ class InputStream(object): r"(?:(?P#.*\n?)| *(&)?(?P(?:!\$|[^&!\"']+|\"[^\"]*\"|'[^']*')*)(?P&)? *(?P!.*)?\n?)",#$ re.IGNORECASE) joinedLine="" - comments=None + comments=[] lines=[] continuation=0 @@ -79,14 +82,26 @@ class InputStream(object): if not self.line_buffer: line=self.infile.readline().replace("\t",8*" ") self.line_nr += 1 - + # convert OMP-conditional fortran statements into normal fortran statements + # but remember to convert them back + is_omp_conditional = False + omp_indent = 0 + if ompRe.match(line): + omp_indent = len(line) - len(line.lstrip(' ')) + line = ompRe.sub('', line, count=1) + is_omp_conditional = True line_start = 0 for pos, char in CharFilter(enumerate(line)): if char == ';' or pos + 1 == len(line): - self.line_buffer.append(line[line_start:pos+1]) + self.line_buffer.append(omp_indent*' ' + '!$'*is_omp_conditional + + line[line_start:pos+1]) + omp_indent = 0 + is_omp_conditional = False line_start = pos+1 - - if(line_start < len(line)): self.line_buffer.append(line[line_start:]) # append comment + if(line_start < len(line)): + # line + comment + self.line_buffer.append('!$'*is_omp_conditional + + line[line_start:]) if self.line_buffer: line = self.line_buffer.popleft() @@ -96,21 +111,26 @@ class InputStream(object): lines.append(line) m=lineRe.match(line) if not m or m.span()[1]!=len(line): + # FIXME: does not handle line continuation of + # omp conditional fortran statements + # starting with an ampersand. raise SyntaxError("unexpected line format:"+repr(line)) if m.group("preprocessor"): if len(lines)>1: raise SyntaxError("continuation to a preprocessor line not supported "+repr(line)) - comments=line + comments.append(line) break coreAtt=m.group("core") + if ompRe.match(coreAtt) and joinedLine.strip(): + # remove omp '!$' for line continuation + coreAtt = ompRe.sub('', coreAtt, count=1).lstrip() joinedLine = joinedLine.rstrip("\n") + coreAtt if coreAtt and not coreAtt.isspace(): continuation=0 if m.group("continue"): continuation=1 if m.group("comment"): - if comments: - comments+="\n"+m.group("comment") - else: - comments=m.group("comment") + comments.append(m.group("comment")) + else: + comments.append('') if not continuation: break return (joinedLine,comments,lines) @@ -142,7 +162,7 @@ def parseRoutine(inFile): includeRe=re.compile(r"#? *include +[\"'](?P.+)[\"'] *$",re.IGNORECASE) stream = InputStream(inFile) while 1: - (jline,comments,lines)=stream.nextFortranLine() + (jline,_,lines)=stream.nextFortranLine() if len(lines)==0: break if startRe.match(jline):break routine['preRoutine'].extend(lines) @@ -152,7 +172,7 @@ def parseRoutine(inFile): subF=file(m.group('file')) subStream = InputStream(subF) while 1: - (subjline,subcomments,sublines)=subStream.nextFortranLine() + (subjline,_,sublines)=subStream.nextFortranLine() if not sublines: break routine['strippedCore'].append(subjline) @@ -177,7 +197,8 @@ def parseRoutine(inFile): if (not routine['result'])and(routine['kind'].lower()=="function"): routine['result']=routine['name'] while 1: - (jline,comments,lines)=stream.nextFortranLine() + (jline,comment_list,lines)=stream.nextFortranLine() + comments = '\n'.join(_ for _ in comment_list if _) if len(lines)==0: break if lines[0].lower().startswith("#include"): break if not ignoreRe.match(jline): @@ -227,7 +248,7 @@ def parseRoutine(inFile): istart=lines interfaceDeclFile=StringIO() while 1: - (jline,comments,lines)=stream.nextFortranLine() + (jline,_,lines)=stream.nextFortranLine() if interfaceEndRe.match(jline): iend=lines break @@ -281,7 +302,7 @@ def parseRoutine(inFile): subF=file(m.group('file')) subStream = InputStream(subF) while 1: - (subjline,subcomments,sublines)=subStream.nextFortranLine() + (subjline,_,sublines)=subStream.nextFortranLine() if not sublines: break routine['strippedCore'].append(subjline) @@ -291,7 +312,7 @@ def parseRoutine(inFile): print("error trying to follow include ",m.group('file')) print("warning this might lead to the removal of used variables") traceback.print_exc() - (jline,comments,lines)=stream.nextFortranLine() + (jline,_,lines)=stream.nextFortranLine() return routine def findWord(word,text,options=re.IGNORECASE): @@ -746,7 +767,8 @@ def parseUse(inFile): commonUses="" stream = InputStream(inFile) while 1: - (jline,comments,lines)=stream.nextFortranLine() + (jline,comment_list,lines)=stream.nextFortranLine() + comments = '\n'.join(_ for _ in comment_list if _) lineNr=lineNr+len(lines) if not lines: break origLines.append("".join(lines)) diff --git a/tools/formatting/normalizeExtended.py b/tools/formatting/reformatFortranFile.py similarity index 74% rename from tools/formatting/normalizeExtended.py rename to tools/formatting/reformatFortranFile.py index 817ba42fee..1330ff3217 100644 --- a/tools/formatting/normalizeExtended.py +++ b/tools/formatting/reformatFortranFile.py @@ -2,21 +2,18 @@ Impose white space conventions and indentation based on scopes / subunits normalization of white spaces supported for following operators: - - relational operators (convention ' op '): + - relational operators: .EQ. .NE. .LT. .LE. .GT. .GE. == /= < <= > >= - - logical operators (convention ' op ', exception '.NOT. '): + - logical operators: .AND. .OR. .EQV. .NEQV. .NOT. - - bracket delimiters (context dependent convention) - - commas and semicolons (convention ', '): - - arithmetic operators convention 'op': - * / ** - - arithmetic operators convention ' op ': - + - - - other operators convention 'op': + - bracket delimiters + - commas and semicolons: + - arithmetic operators: + * / ** + - + - other operators: % - (sign) = (function argument) - - other operators convention ' op ': = (assignment) => (pointer assignment) supported criteria for alignment / indentation: @@ -41,22 +38,21 @@ import re import sys -from formatting.normalizeFortranFile import useParseRe, typeRe, InputStream, CharFilter +from formatting.normalizeFortranFile import useParseRe, typeRe, InputStream, CharFilter, ompRe, ompDirRe #========================================================================= # constants, mostly regular expressions RE_FLAGS = re.IGNORECASE # all regex should be case insensitive -DEFAULT_ERROR_MESSAGE = " Syntax error - this script can not handle invalid Fortran files." +FORTRAN_DEFAULT_ERROR_MESSAGE = " Syntax error - this script can not handle invalid Fortran files." +FORMATTER_ERROR_MESSAGE = " Wrong usage of formatting-specific directives '&', '!&', '!&<' or '!&>'." EOL_STR = r"\s*;?\s*$" # end of fortran line EOL_SC = r"\s*;\s*$" # whether line is ended with semicolon SOL_STR = r"^\s*" # start of fortran line -# special cases (f77 constructs and omp statements not formatted) F77_STYLE = re.compile(r"^\s*\d", RE_FLAGS) -OMP_RE = re.compile(r"^\s*!\$", RE_FLAGS) # regular expressions for parsing statements that start, continue or end a # subunit: @@ -106,13 +102,11 @@ PUBLIC_RE = re.compile(SOL_STR + r"PUBLIC\s*::") # intrinsic statements with parenthesis notation that are not functions INTR_STMTS_PAR = "(ALLOCATE|DEALLOCATE|REWIND|BACKSPACE|INQUIRE|OPEN|CLOSE|WRITE|READ|FORALL|WHERE|NULLIFY)" -# regular expressions for parsing linebreaks (it implicitly ignores -# ampersands in strings but does not work for comments with ampersand) +# regular expressions for parsing linebreaks LINEBREAK_STR = r"(&)[\s]*(?:!.*)?$" -# regular expressions for parsing operators: - -# Note: exclusion of +/- in real literals and sign operator +# regular expressions for parsing operators +# Note: +/- in real literals and sign operator is ignored PLUSMINUS_RE = re.compile( r"(?<=[\w\)\]])(? 2 or self._level: raise SyntaxError(self._filename + ':' + str(self._line_nr) + - ':' + DEFAULT_ERROR_MESSAGE) + ':' + FORTRAN_DEFAULT_ERROR_MESSAGE) def get_lines_indent(self): """ @@ -312,7 +312,7 @@ class F90Aligner(object): """ return self._line_indents - def __align_linebreaks(self, line, is_decl, indent_size, line_nr): + def __align_line_continuations(self, line, is_decl, indent_size, line_nr): indent_list = self._br_indent_list level = self._level @@ -354,7 +354,7 @@ class F90Aligner(object): indent_list.pop() if level < 0: raise SyntaxError(filename + ':' + str(line_nr) + - ':' + DEFAULT_ERROR_MESSAGE) + ':' + FORTRAN_DEFAULT_ERROR_MESSAGE) if pos_ldelim: pos_ldelim.pop() what_del_open = ldelim.pop() @@ -367,7 +367,7 @@ class F90Aligner(object): valid = what_del_close == r"]" if not valid: raise SyntaxError( - filename + ':' + str(line_nr) + ':' + DEFAULT_ERROR_MESSAGE) + filename + ':' + str(line_nr) + ':' + FORTRAN_DEFAULT_ERROR_MESSAGE) else: pos_rdelim.append(pos) rdelim.append(what_del_close) @@ -376,7 +376,7 @@ class F90Aligner(object): if not REL_OP_RE.match(line[max(0, pos - 1):min(pos + 2, len(line))]): if pos_eq > 0: raise SyntaxError( - filename + ':' + str(line_nr) + ':' + DEFAULT_ERROR_MESSAGE) + filename + ':' + str(line_nr) + ':' + FORTRAN_DEFAULT_ERROR_MESSAGE) is_pointer = line[pos + 1] == '>' pos_eq = pos + 1 # don't align if assignment operator directly before @@ -402,10 +402,12 @@ class F90Aligner(object): #========================================================================= + def inspect_ffile_format(infile, indent_size): """ - Get some structural information of the Fortran file (original indentation, - check if it has f77 constructs). + Determine indentation by inspecting original Fortran file (mainly for finding + aligned blocks of DO/IF statements). Also check if + it has f77 constructs. """ adopt = indent_size <= 0 @@ -434,13 +436,18 @@ def inspect_ffile_format(infile, indent_size): #========================================================================= -def format_single_fline(f_line, whitespace, linebreak_pos, filename, line_nr): + +def format_single_fline(f_line, whitespace, linebreak_pos, ampersand_sep, filename, line_nr, auto_format=True): """ - format a single Fortran line. Takes a logical Fortran line as input - as well as the positions of the linebreaks. Filename and line_nr just - for error messages. Imposes white space formatting and inserts linebreaks. - The higher whitespace, the more white space characters inserted. Right now - whitespace = 0, 1, 2 are supported. + format a single Fortran line - imposes white space formatting + and inserts linebreaks. + Takes a logical Fortran line `f_line` as input as well as the positions + of the linebreaks (`linebreak_pos`), and the number of separating whitespace + characters before ampersand (`ampersand_sep`). + `filename` and `line_nr` just for error messages. + The higher `whitespace`, the more white space characters inserted - + whitespace = 0, 1, 2 are currently supported. + auto formatting can be turned off by setting `auto_format` to False. """ # define whether to put whitespaces around operators: @@ -448,15 +455,15 @@ def format_single_fline(f_line, whitespace, linebreak_pos, filename, line_nr): # 1: assignment operators # 2: relational operators # 3: logical operators - # 4: arithm. operators plus and minus - if whitespace==0: - spacey = [0,0,0,0,0] - elif whitespace==1: - spacey = [1,1,1,1,0] - elif whitespace==2: - spacey = [1,1,1,1,1] + # 4: arithm. operators plus and minus + if whitespace == 0: + spacey = [0, 0, 0, 0, 0] + elif whitespace == 1: + spacey = [1, 1, 1, 1, 0] + elif whitespace == 2: + spacey = [1, 1, 1, 1, 1] else: - raise NotImplementedError("unknown value for whitespace") + raise NotImplementedError("unknown value for whitespace") level = 0 lines_out = [] @@ -536,7 +543,8 @@ def format_single_fline(f_line, whitespace, linebreak_pos, filename, line_nr): if char == ',' or char == ';': lhs = line_ftd[:pos + offset] rhs = line_ftd[pos + 1 + offset:] - line_ftd = lhs.rstrip(' ') + char + ' '*spacey[0] + rhs.lstrip(' ') + line_ftd = lhs.rstrip(' ') + char + ' ' * \ + spacey[0] + rhs.lstrip(' ') line_ftd = line_ftd.rstrip(' ') # format .NOT. @@ -544,7 +552,7 @@ def format_single_fline(f_line, whitespace, linebreak_pos, filename, line_nr): lhs = line_ftd[:pos + offset] rhs = line_ftd[pos + 5 + offset:] line_ftd = lhs.rstrip( - ' ') + line[pos:pos + 5] + ' '*spacey[3] + rhs.lstrip(' ') + ' ') + line[pos:pos + 5] + ' ' * spacey[3] + rhs.lstrip(' ') # strip whitespaces from '=' and prepare assignment operator # formatting @@ -568,7 +576,8 @@ def format_single_fline(f_line, whitespace, linebreak_pos, filename, line_nr): assign_op = '=>' # pointer assignment else: assign_op = '=' # assignment - line_ftd = lhs.rstrip(' ') + ' '*spacey[1] + assign_op + ' '*spacey[1] + rhs.lstrip(' ') + line_ftd = lhs.rstrip( + ' ') + ' ' * spacey[1] + assign_op + ' ' * spacey[1] + rhs.lstrip(' ') # offset w.r.t. unformatted line line = line_ftd @@ -599,10 +608,13 @@ def format_single_fline(f_line, whitespace, linebreak_pos, filename, line_nr): for pos, part in enumerate(line_parts): if not re.match(r"['\"!]", part, RE_FLAGS): # exclude comments, strings partsplit = lr_re.split(part) - line_parts[pos] = (' '*spacey[n_op+2]).join(partsplit) + line_parts[pos] = (' ' * spacey[n_op + 2]).join(partsplit) line = ''.join(line_parts) + if not auto_format: + line = line_orig + # Now it gets messy - we need to shift line break positions from original # to reformatted line pos_new = 0 @@ -636,26 +648,28 @@ def format_single_fline(f_line, whitespace, linebreak_pos, filename, line_nr): linebreak_pos_ftd.insert(0, 0) # We do not insert ampersands in empty lines and comments lines - lines_out = [line[l:r].rstrip(' ') + ' &' * min(1, r - l) - for l, r in zip(linebreak_pos_ftd[0:-1], linebreak_pos_ftd[1:])] + lines_out = [line[l:r].rstrip(' ') + ' ' * ampersand_sep[pos] + '&' * min(1, r - l) + for pos, (l, r) in enumerate(zip(linebreak_pos_ftd[0:-1], linebreak_pos_ftd[1:]))] lines_out.append(line[linebreak_pos_ftd[-1]:]) if level != 0: raise SyntaxError(filename + ':' + str(line_nr) + - ':' + DEFAULT_ERROR_MESSAGE) + ':' + FORTRAN_DEFAULT_ERROR_MESSAGE) return lines_out #========================================================================= -def format_extended_ffile(infile, outfile, logFile=sys.stdout, indent_size=2, whitespace=2, orig_filename=None): + +def reformat_ffile(infile, outfile, logFile=sys.stdout, indent_size=2, whitespace=2, orig_filename=None): """ - main method to be invoked for formatting a Fortran file + main method to be invoked for formatting a Fortran file. """ debug = False - adopt_indents = indent_size <= 0 # don't change original indentation if rel-indents set to 0 + # don't change original indentation if rel-indents set to 0 + adopt_indents = indent_size <= 0 if not orig_filename: orig_filename = infile.name @@ -676,47 +690,69 @@ def format_extended_ffile(infile, outfile, logFile=sys.stdout, indent_size=2, wh do_indent = True stream = InputStream(infile) skip_blank = False + in_manual_block = False + while 1: f_line, comments, lines = stream.nextFortranLine() if not lines: break - # FIXME here we reconstruct comment positions from lines - this is a bad hack and should be - # fixed in nextFortranLine - - if not comments: - comments = '' - - comments_tmp = comments.split('\n') - comments_tmp = comments_tmp[::-1] comment_lines = [] - for line in lines: - if comments_tmp: - comment = comments_tmp[-1] - else: - comment = '' - comment_len = len(comment) - if comment and comment == line[-1 - comment_len:-1]: - sep = not comment.strip(' \n') == line.strip(' \n') - comment_lines.append(' ' * sep + comments_tmp.pop()) - elif line.strip(' \n'): - comment_lines.append('') + for line, comment in zip(lines, comments): + has_comment = bool(comment.strip()) + sep = has_comment and not comment.strip() == line.strip() + if line.strip(): # empty lines between linebreaks are ignored + comment_lines.append(' ' * sep + comment.rstrip(' \n')) orig_lines = lines nfl += 1 + auto_align = not any(NO_ALIGN_RE.search(_) for _ in lines) + auto_format = not (in_manual_block or any( + _.lstrip().startswith('!&') for _ in comment_lines)) + if not auto_format: + auto_align = False + if (len(lines)) == 1: + valid_directive = True + if lines[0].strip().startswith('!&<'): + if in_manual_block: + valid_directive = False + else: + in_manual_block = True + if lines[0].strip().startswith('!&>'): + if not in_manual_block: + valid_directive = False + else: + in_manual_block = False + if not valid_directive: + raise SyntaxError(orig_filename + ':' + str(stream.line_nr) + + ':' + FORMATTER_ERROR_MESSAGE) + indent = [0] * len(lines) + is_omp_conditional = False + + if ompRe.match(f_line) and not ompDirRe.match(f_line): + # convert OMP-conditional fortran statements into normal fortran statements + # but remember to convert them back + f_line = ompRe.sub(' ', f_line, count=1) + lines = [ompRe.sub(' ', l, count=1) for l in lines] + is_omp_conditional = True + is_empty = EMPTY_RE.search(f_line) # blank line or comment only line + if useParseRe.match(f_line): pass # do not touch use statements cleaned up by existing prettify - elif OMP_RE.match(f_line): - pass # do not touch OMP statements! + elif ompDirRe.match(f_line): + # move '!$OMP' to line start, otherwise don't format omp directives + lines = ['!$OMP' + (len(l) - len(l.lstrip())) * + ' ' + ompDirRe.sub('', l, count=1) for l in lines] + pass elif lines[0].startswith('#'): # preprocessor macros assert len(lines) == 1 elif EMPTY_RE.search(f_line): # empty lines including comment lines assert len(lines) == 1 - if comments: + if any(comments): if lines[0].startswith('!'): pass # don't indent unindented comment lines else: @@ -726,8 +762,37 @@ def format_extended_ffile(infile, outfile, logFile=sys.stdout, indent_size=2, wh lines = [l.strip(' ') for l in lines] else: - # remove whitespaces on both sides, and '&' on left of line - lines = [l.strip(' ').lstrip('&') for l in lines] + + manual_lines_indent = [] + if not auto_align: + manual_lines_indent = [ + len(l) - len(l.lstrip(' ').lstrip('&')) for l in lines] + manual_lines_indent = [ind - manual_lines_indent[0] + for ind in manual_lines_indent] + + # ampersands at line starts are remembered (pre_ampersand) and recovered later; + # define the desired number of separating whitespaces before ampersand at line end (ampersand_sep): + # - insert one whitespace character before ampersand as default formatting + # - don't do this if next line starts with an ampersand but remember the original formatting + # this "special rule" is necessary since ampersands starting a line can be used to break literals, + # so inserting a whitespace in this case leads to invalid syntax. + + pre_ampersand = [] + ampersand_sep = [] + sep_next = None + for pos, line in enumerate(lines): + m = re.search(SOL_STR + r'(&\s*)', line) + if m: + pre_ampersand.append(m.group(1)) + sep = len( + re.search(r'(\s*)&[\s]*(?:!.*)?$', lines[pos - 1]).group(1)) + ampersand_sep.append(sep) + else: + pre_ampersand.append('') + if pos > 0: + ampersand_sep.append(1) + + lines = [l.strip(' ').strip('&') for l in lines] f_line = f_line.strip(' ') # find linebreak positions @@ -746,7 +811,7 @@ def format_extended_ffile(infile, outfile, logFile=sys.stdout, indent_size=2, wh 1 for _ in range(0, len(linebreak_pos))] lines = format_single_fline( - f_line, whitespace, linebreak_pos, orig_filename, stream.line_nr) + f_line, whitespace, linebreak_pos, ampersand_sep, orig_filename, stream.line_nr, auto_format) # we need to insert comments in formatted lines for pos, (line, comment) in enumerate(zip(lines, comment_lines)): @@ -761,9 +826,17 @@ def format_extended_ffile(infile, outfile, logFile=sys.stdout, indent_size=2, wh except IndexError: rel_indent = 0 - indenter.process_lines_of_fline(f_line, lines, rel_indent, indent_size, stream.line_nr) + indenter.process_lines_of_fline( + f_line, lines, rel_indent, indent_size, stream.line_nr, manual_lines_indent) indent = indenter.get_lines_indent() + # recover ampersands at line start + for pos, line in enumerate(lines): + amp_insert = pre_ampersand[pos] + if amp_insert: + indent[pos] += -1 + lines[pos] = amp_insert + line + lines = [re.sub(r"\s+$", '\n', l, RE_FLAGS) for l in lines] # deleting trailing whitespaces @@ -773,16 +846,19 @@ def format_extended_ffile(infile, outfile, logFile=sys.stdout, indent_size=2, wh else: ind_use = 1 if ind_use + len(line) <= 133: - outfile.write(' ' * ind_use + line) + outfile.write('!$' * is_omp_conditional + ' ' * + (ind_use - 2 * is_omp_conditional + + len(line) - len(line.lstrip(' '))) + line.lstrip(' ')) elif len(line) <= 133: - outfile.write(' ' * (133 - len(line)) + line) + outfile.write('!$' * is_omp_conditional + ' ' * + (133 - 2 * is_omp_conditional - + len(line.lstrip(' '))) + line.lstrip(' ')) logFile.write("*** " + orig_filename + ":" + str(stream.line_nr) + - ": auto indentation failed due to 132 chars limit, please insert line break. ***\n") + ": auto indentation failed due to 132 chars limit. ***\n") else: outfile.write(orig_line) logFile.write("*** " + orig_filename + ":" + str(stream.line_nr) + - (": auto indentation and whitespace formatting failed due to 132 chars limit, " - "please insert line break. ***\n")) + (": auto indentation and whitespace formatting failed due to 132 chars limit. ***\n")) if debug: print(' ' * ind_use + line) # no indentation of blank lines @@ -792,6 +868,6 @@ def format_extended_ffile(infile, outfile, logFile=sys.stdout, indent_size=2, wh do_indent = True # rm subsequent blank lines - skip_blank = is_empty and not comments + skip_blank = is_empty and not any(comments) -#EOF +# EOF diff --git a/tools/prettify.py b/tools/prettify.py index b3f180ee85..b4d7eda21a 100755 --- a/tools/prettify.py +++ b/tools/prettify.py @@ -6,7 +6,7 @@ import os, os.path from formatting import normalizeFortranFile from formatting import replacer from formatting import addSynopsis -from formatting import normalizeExtended +from formatting import reformatFortranFile from sys import argv operatorsStr=r"\.(?:and|eqv?|false|g[et]|l[et]|n(?:e(?:|qv)|ot)|or|true)\." @@ -15,9 +15,19 @@ keywordsStr="(?:a(?:llocat(?:able|e)|ssign(?:|ment))|c(?:a(?:ll|se)|haracter|los intrinsic_procStr=r"(?:a(?:bs|c(?:har|os)|djust[lr]|i(?:mag|nt)|ll(?:|ocated)|n(?:int|y)|s(?:in|sociated)|tan2?)|b(?:it_size|test)|c(?:eiling|har|mplx|o(?:njg|sh?|unt)|shift)|d(?:ate_and_time|ble|i(?:gits|m)|ot_product|prod)|e(?:oshift|psilon|xp(?:|onent))|f(?:loor|raction)|huge|i(?:a(?:char|nd)|b(?:clr|its|set)|char|eor|n(?:dex|t)|or|shftc?)|kind|l(?:bound|en(?:|_trim)|g[et]|l[et]|og(?:|10|ical))|m(?:a(?:tmul|x(?:|exponent|loc|val))|erge|in(?:|exponent|loc|val)|od(?:|ulo)|vbits)|n(?:earest|int|ot)|p(?:ack|r(?:e(?:cision|sent)|oduct))|r(?:a(?:dix|n(?:dom_(?:number|seed)|ge))|e(?:peat|shape)|rspacing)|s(?:ca(?:le|n)|e(?:lected_(?:int_kind|real_kind)|t_exponent)|hape|i(?:gn|nh?|ze)|p(?:acing|read)|qrt|um|ystem_clock)|t(?:anh?|iny|r(?:ans(?:fer|pose)|im))|u(?:bound|npack)|verify)(?= *\()" +ompDir=r"(?:atomic|barrier|c(?:apture|ritical)|do|end|flush|if|master|num_threads|ordered|parallel|read|s(?:ection(?:|s)|ingle)|t(?:ask(?:|wait|yield)|hreadprivate)|update|w(?:orkshare|rite)|!\$omp)" + +ompClause=r"(?:a|co(?:llapse|py(?:in|private))|default|fi(?:nal|rstprivate)|i(?:and|eor|or)|lastprivate|m(?:ax|ergeable|in)|n(?:one|owait)|ordered|private|reduction|shared|untied|\.(?:and|eqv|neqv|or)\.)" + +ompEnv=r"omp_(?:dynamic|max_active_levels|n(?:ested|um_threads)|proc_bind|s(?:tacksize|chedule)|thread_limit|wait_policy)" + +# FIXME: does not correctly match operator '.op.' if it is not separated by whitespaces. toUpcaseRe=re.compile("(?"+operatorsStr+ "|"+ keywordsStr +"|"+ intrinsic_procStr + ")(?![A-Za-z0-9_%])",flags=re.IGNORECASE) +toUpcaseOMPRe=re.compile("(?" + +ompDir+"|"+ompClause+"|"+ompEnv + + ")(?![A-Za-z0-9_%])",flags=re.IGNORECASE) linePartsRe=re.compile("(?P[^\"'!]*)(?P!.*)?"+ "(?P(?P[\"']).*?(?P=qchar))?") @@ -38,15 +48,23 @@ def upcaseStringKeywords(line): start=start+m.end() return res -def upcaseKeywords(infile,outfile,logFile=sys.stdout): +def upcaseOMP(line): + """Upcases OpenMP stuff.""" + return toUpcaseOMPRe.sub(lambda match: match.group("toUpcase").upper(),line) + +def upcaseKeywords(infile,outfile,upcase_omp,logFile=sys.stdout): """Writes infile to outfile with all the fortran keywords upcased""" while 1: line=infile.readline() if not line: break - outfile.write(upcaseStringKeywords(line)) + line = upcaseStringKeywords(line) + if upcase_omp: + if normalizeFortranFile.ompDirRe.match(line): + line=upcaseOMP(line) + outfile.write(line) -def prettifyFile(infile, normalize_use=1, normalize_extended=0, indent=2, whitespace=2, upcase_keywords=1, - interfaces_dir=None,replace=None,logFile=sys.stdout): +def prettifyFile(infile, normalize_use=1, reformat=0, indent=2, whitespace=2, upcase_keywords=1, + upcase_omp=0, interfaces_dir=None,replace=None,logFile=sys.stdout): """prettifyes the fortran source in infile into a temporary file that is returned. It can be the same as infile. if normalize_use normalizes the use statements (defaults to true) @@ -69,9 +87,9 @@ def prettifyFile(infile, normalize_use=1, normalize_extended=0, indent=2, whites tmpfile.close() tmpfile=tmpfile2 ifile=tmpfile - if normalize_extended: # extended needs to be done first + if reformat: # reformat needs to be done first tmpfile2=os.tmpfile() - normalizeExtended.format_extended_ffile(ifile,tmpfile2,logFile=logFile, + reformatFortranFile.reformat_ffile(ifile,tmpfile2,logFile=logFile, indent_size=indent,whitespace=whitespace, orig_filename=orig_filename) tmpfile2.seek(0) @@ -90,7 +108,7 @@ def prettifyFile(infile, normalize_use=1, normalize_extended=0, indent=2, whites ifile=tmpfile if upcase_keywords: tmpfile2=os.tmpfile() - upcaseKeywords(ifile,tmpfile2,logFile) + upcaseKeywords(ifile,tmpfile2,upcase_omp,logFile) tmpfile2.seek(0) if tmpfile: tmpfile.close() @@ -123,8 +141,8 @@ def prettifyFile(infile, normalize_use=1, normalize_extended=0, indent=2, whites raise def prettfyInplace(fileName,bkDir="preprettify",normalize_use=1, - normalize_extended=0,indent=2,whitespace=2, - upcase_keywords=1, interfaces_dir=None, + reformat=0,indent=2,whitespace=2, + upcase_keywords=1, upcase_omp=0, interfaces_dir=None, replace=None,logFile=sys.stdout): """Same as prettify, but inplace, replaces only if needed""" if not os.path.exists(bkDir): @@ -132,8 +150,9 @@ def prettfyInplace(fileName,bkDir="preprettify",normalize_use=1, if not os.path.isdir(bkDir): raise Error("bk-dir must be a directory, was "+bkDir) infile=open(fileName,'r') - outfile=prettifyFile(infile, normalize_use, normalize_extended, - indent, whitespace, upcase_keywords, interfaces_dir, replace) + outfile=prettifyFile(infile, normalize_use, reformat, + indent, whitespace, upcase_keywords, upcase_omp, + interfaces_dir, replace) if (infile==outfile): return infile.seek(0) @@ -171,22 +190,41 @@ def prettfyInplace(fileName,bkDir="preprettify",normalize_use=1, infile.close() outfile.close() +# FIXME: 'use' statements means not only use, but also variable declarations, add this to docu def main(): - defaultsDict={'upcase':1,'normalize-use':1, - 'extended':0,'indent':2, 'whitespace':2, - 'replace':1, - 'interface-dir':None, + # future defaults + defaultsDict={'upcase':1,'normalize-use':1,'omp-upcase':1, + 'reformat':1, 'indent':3, 'whitespace':1, + 'replace':1, 'interface-dir':None, 'backup-dir':'preprettify'} + + # current defaults (FIXME: change to future defaults) + defaultsDict={'upcase':1,'normalize-use':1,'omp-upcase':0, + 'reformat':0, 'indent':2, 'whitespace':1, + 'replace':1, 'interface-dir':None, + 'backup-dir':'preprettify'} + usageDesc=("usage:\n"+sys.argv[0]+ """ - [--[no-]upcase] [--[no-]normalize-use] [--[no-]replace] - [--[no-]extended] --indent=2 --whitespace=2 [--interface-dir=~/cp2k/obj/platform/target] [--help] + [--[no-]upcase] [--[no-]normalize-use] [--[no-]omp-upcase] [--[no-]replace] + [--[no-]reformat] --indent=3 --whitespace=1 [--interface-dir=~/cp2k/obj/platform/target] [--help] [--backup-dir=bk_dir] file1 [file2 ...] - replaces file1,... with their prettified version after performing on - them upcase of the fortran keywords, and normalizion the use statements. - If the interface direcory is given updates also the synopsis. - If requested the replacements performed by the replacer.py script - are also preformed. + Auto-format F90 source file1, file2, ...: + - normalization of use statements (--normalize-use) (FIXME: give more detailed description) + - upcasing fortran keywords (--upcase) + - auto-indentation, auto-alignment and whitespace formatting (--reformat). + Amount of whitespace controlled by --whitespace = 0, 1, 2. + For indenting with a relative width of n columns specify --indent=n. + For manual formatting of specific lines: + - disable auto-alignment by starting line continuation with an ampersand '&' + - completely disable reformatting by adding a comment '!&' + For manual formatting of a code block, use: + - start a manually formatted block with a '!&<' comment and close it with a '!&>' comment + - upcasing OMP directives (--omp-upcase) + - If requested the replacements performed by the replacer.py script are also performed (--replace) (FIXME: ???) + - If the interface direcory is given updates also the synopsis. (--interface-dir) (FIXME: ???) + + Defaults: """+str(defaultsDict)) replace=None @@ -195,7 +233,7 @@ def main(): sys.exit(0) args=[] for arg in sys.argv[1:]: - m=re.match(r"--(no-)?(normalize-use|upcase|replace|extended)",arg) + m=re.match(r"--(no-)?(normalize-use|upcase|omp-upcase|replace|reformat)",arg) if m: defaultsDict[m.groups()[1]]=not m.groups()[0] else: @@ -208,7 +246,10 @@ def main(): path=os.path.abspath(os.path.expanduser(m.groups()[1])) defaultsDict[m.groups()[0]]=path else: - args.append(arg) + if arg.startswith('--'): + print('unknown option',arg) + else: + args.append(arg) if len(args)<1: print(usageDesc) else: @@ -231,10 +272,11 @@ def main(): try: prettfyInplace(fileName,bkDir, normalize_use=defaultsDict['normalize-use'], - normalize_extended=defaultsDict['extended'], + reformat=defaultsDict['reformat'], indent=defaultsDict['indent'], whitespace=defaultsDict['whitespace'], upcase_keywords=defaultsDict['upcase'], + upcase_omp=defaultsDict['omp-upcase'], interfaces_dir=defaultsDict['interface-dir'], replace=defaultsDict['replace']) except: