diff options
Diffstat (limited to 'codegen/vulkan/scripts/reflow.py')
| -rwxr-xr-x | codegen/vulkan/scripts/reflow.py | 912 |
1 files changed, 0 insertions, 912 deletions
diff --git a/codegen/vulkan/scripts/reflow.py b/codegen/vulkan/scripts/reflow.py deleted file mode 100755 index 1897b05c..00000000 --- a/codegen/vulkan/scripts/reflow.py +++ /dev/null @@ -1,912 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright 2016-2021 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Used for automatic reflow of spec sources to satisfy the agreed layout to -minimize git churn. Most of the logic has to do with detecting asciidoc -markup or block types that *shouldn't* be reflowed (tables, code) and -ignoring them. It's very likely there are many asciidoc constructs not yet -accounted for in the script, our usage of asciidoc markup is intentionally -somewhat limited. - -Also used to insert identifying tags on explicit Valid Usage statements. - -Usage: `reflow.py [-noflow] [-tagvu] [-nextvu #] [-overwrite] [-out dir] [-suffix str] files` - -- `-noflow` acts as a passthrough, instead of reflowing text. Other - processing may occur. -- `-tagvu` generates explicit VUID tag for Valid Usage statements which - don't already have them. -- `-nextvu #` starts VUID tag generation at the specified # instead of - the value wired into the `reflow.py` script. -- `-overwrite` updates in place (can be risky, make sure there are backups) -- `-check FAIL|WARN` runs some simple sanity checks on markup. If the checks - fail and the WARN option is given, the script will simply print a warning - message. If the checks fail and the FAIL option is given, the script will - exit with an error code. FAIL is for use with continuous integration - scripts enforcing the checks. -- `-out` specifies directory to create output file in, default 'out' -- `-suffix` specifies suffix to add to output files, default '' -- `files` are asciidoc source files from the spec to reflow. -""" -# For error and file-loading interfaces only -import argparse -import os -import re -import sys -from reflib import loadFile, logDiag, logWarn, logErr, setLogFile, getBranch - -# Vulkan-specific - will consolidate into scripts/ like OpenXR soon -sys.path.insert(0, 'xml') - -from vkconventions import VulkanConventions as APIConventions -conventions = APIConventions() - -# Markup that always ends a paragraph -# empty line or whitespace -# [block options] -# [[anchor]] -# // comment -# <<<< page break -# :attribute-setting -# macro-directive::terms -# + standalone list item continuation -# label:: labelled list - label must be standalone -endPara = re.compile(r'^( *|\[.*\]|//.*|<<<<|:.*|[a-z]+::.*|\+|.*::)$') - -# Special case of markup ending a paragraph, used to track the current -# command/structure. This allows for either OpenXR or Vulkan API path -# conventions. Nominally it should use the file suffix defined by the API -# conventions (conventions.file_suffix), except that XR uses '.txt' for -# generated API include files, not '.adoc' like its other includes. -includePat = re.compile( - r'include::(?P<directory_traverse>((../){1,4}|\{INCS-VAR\}/|\{generated\}/)(generated/)?)(?P<generated_type>[\w]+)/(?P<category>\w+)/(?P<entity_name>[^./]+).txt[\[][\]]') - -# Find the first pname: or code: pattern in a Valid Usage statement -pnamePat = re.compile(r'pname:(?P<param>\{?\w+\}?)') -codePat = re.compile(r'code:(?P<param>\w+)') - -# Markup that's OK in a contiguous paragraph but otherwise passed through -# .anything (except .., which indicates a literal block) -# === Section Titles -endParaContinue = re.compile(r'^(\.[^.].*|=+ .*)$') - -# Markup for block delimiters whose contents *should* be reformatted -# -- (exactly two) (open block) -# **** (4 or more) (sidebar block - why do we have these?!) -# ==== (4 or more) (example block) -# ____ (4 or more) (quote block) -blockReflow = re.compile(r'^(--|[*=_]{4,})$') - -# Fake block delimiters for "common" VU statements -blockCommonReflow = '// Common Valid Usage\n' - -# Markup for block delimiters whose contents should *not* be reformatted -# |=== (3 or more) (table) -# ++++ (4 or more) (passthrough block) -# .... (4 or more) (literal block) -# //// (4 or more) (comment block) -# ---- (4 or more) (listing block) -# ``` (3 or more) (listing block) -# **** (4 or more) (sidebar block) -blockPassthrough = re.compile(r'^(\|={3,}|[`]{3}|[\-+./~]{4,})$') - -# Markup for introducing lists (hanging paragraphs) -# * bullet -# ** bullet -# -- bullet -# . bullet -# :: bullet (no longer supported by asciidoctor 2) -# {empty}:: bullet -# 1. list item -beginBullet = re.compile(r'^ *([*\-.]+|\{empty\}::|::|[0-9]+[.]) ') - -# Start of an asciidoctor conditional -# ifdef:: -# ifndef:: -conditionalStart = re.compile(r'^(ifdef|ifndef)::') - -# Text that (may) not end sentences - -# A single letter followed by a period, typically a middle initial. -endInitial = re.compile(r'^[A-Z]\.$') -# An abbreviation, which doesn't (usually) end a line. -endAbbrev = re.compile(r'(e\.g|i\.e|c\.f|vs)\.$', re.IGNORECASE) - -class ReflowState: - """State machine for reflowing. - - Represents the state of the reflow operation""" - def __init__(self, - filename, - margin = 76, - file = sys.stdout, - breakPeriod = True, - reflow = True, - nextvu = None, - maxvu = None): - - self.blockStack = [ None ] - """The last element is a line with the asciidoc block delimiter that's currently in effect, - such as '--', '----', '****', '======', or '+++++++++'. - This affects whether or not the block contents should be formatted.""" - - self.reflowStack = [ True ] - """The last element is True or False if the current blockStack contents - should be reflowed.""" - self.vuStack = [ False ] - """the last element is True or False if the current blockStack contents - are an explicit Valid Usage block.""" - - self.margin = margin - """margin to reflow text to.""" - - self.para = [] - """list of lines in the paragraph being accumulated. - When this is non-empty, there is a current paragraph.""" - - self.lastTitle = False - """true if the previous line was a document title line - (e.g. :leveloffset: 0 - no attempt to track changes to this is made).""" - - self.leadIndent = 0 - """indent level (in spaces) of the first line of a paragraph.""" - - self.hangIndent = 0 - """indent level of the remaining lines of a paragraph.""" - - self.file = file - """file handle to write to.""" - - self.filename = filename - """base name of file being read from.""" - - self.lineNumber = 0 - """line number being read from the input file.""" - - self.breakPeriod = breakPeriod - """True if justification should break to a new line after the end of a sentence.""" - - self.breakInitial = True - """True if justification should break to a new line after - something that appears to be an initial in someone's name. **TBD**""" - - self.reflow = reflow - """True if text should be reflowed, False to pass through unchanged.""" - - self.vuPrefix = 'VUID' - """Prefix of generated Valid Usage tags""" - - self.vuFormat = '{0}-{1}-{2}-{3:0>5d}' - """Format string for generating Valid Usage tags. - First argument is vuPrefix, second is command/struct name, third is parameter name, fourth is the tag number.""" - - self.nextvu = nextvu - """Integer to start tagging un-numbered Valid Usage statements with, - or None if no tagging should be done.""" - - self.maxvu = maxvu - """Maximum tag to use for Valid Usage statements, or None if no - tagging should be done.""" - - self.defaultApiName = '{refpage}' - self.apiName = self.defaultApiName - """String name of a Vulkan structure or command for VUID tag - generation, or {refpage} if one hasn't been included in this file - yet.""" - - def incrLineNumber(self): - self.lineNumber = self.lineNumber + 1 - - def printLines(self, lines): - """Print an array of lines with newlines already present""" - if len(lines) > 0: - logDiag(':: printLines:', len(lines), 'lines: ', lines[0], end='') - - if self.file is not None: - for line in lines: - print(line, file=self.file, end='') - - def endSentence(self, word): - """Return True if word ends with a sentence-period, False otherwise. - - Allows for contraction cases which won't end a line: - - - A single letter (if breakInitial is True) - - Abbreviations: 'c.f.', 'e.g.', 'i.e.' (or mixed-case versions)""" - if (word[-1:] != '.' or - endAbbrev.search(word) or - (self.breakInitial and endInitial.match(word))): - return False - - return True - - def vuidAnchor(self, word): - """Return True if word is a Valid Usage ID Tag anchor.""" - return (word[0:7] == '[[VUID-') - - def isOpenBlockDelimiter(self, line): - """Returns True if line is an open block delimiter.""" - return line[0:2] == '--' - - def reflowPara(self): - """Reflow the current paragraph, respecting the paragraph lead and - hanging indentation levels. - - The algorithm also respects trailing '+' signs that indicate embedded newlines, - and will not reflow a very long word immediately after a bullet point. - - Just return the paragraph unchanged if the -noflow argument was - given.""" - if not self.reflow: - return self.para - - logDiag('reflowPara lead indent = ', self.leadIndent, - 'hangIndent =', self.hangIndent, - 'para:', self.para[0], end='') - - # Total words processed (we care about the *first* word vs. others) - wordCount = 0 - - # Tracks the *previous* word processed. It must not be empty. - prevWord = ' ' - - # Track the previous line and paragraph being indented, if any - outLine = None - outPara = [] - - for line in self.para: - line = line.rstrip() - words = line.split() - - # logDiag('reflowPara: input line =', line) - numWords = len(words) - 1 - - for i in range(0, numWords + 1): - word = words[i] - wordLen = len(word) - wordCount += 1 - - endEscape = False - if i == numWords and word == '+': - # Trailing ' +' must stay on the same line - endEscape = word - # logDiag('reflowPara last word of line =', word, 'prevWord =', prevWord, 'endEscape =', endEscape) - else: - pass - # logDiag('reflowPara wordCount =', wordCount, 'word =', word, 'prevWord =', prevWord) - - if wordCount == 1: - # The first word of the paragraph is treated specially. - # The loop logic becomes trickier if all this code is - # done prior to looping over lines and words, so all the - # setup logic is done here. - - outPara = [] - outLine = ''.ljust(self.leadIndent) + word - outLineLen = self.leadIndent + wordLen - - # If the paragraph begins with a bullet point, generate - # a hanging indent level if there isn't one already. - if beginBullet.match(self.para[0]): - bulletPoint = True - if len(self.para) > 1: - logDiag('reflowPara first line matches bullet point', - 'but indent already hanging @ input line', - self.lineNumber) - else: - logDiag('reflowPara first line matches bullet point -' - 'single line, assuming hangIndent @ input line', - self.lineNumber) - self.hangIndent = outLineLen + 1 - else: - bulletPoint = False - else: - # Possible actions to take with this word - # - # addWord - add word to current line - # closeLine - append line and start a new (null) one - # startLine - add word to a new line - - # Default behavior if all the tests below fail is to add - # this word to the current line, and keep accumulating - # that line. - (addWord, closeLine, startLine) = (True, False, False) - - # How long would this line be if the word were added? - newLen = outLineLen + 1 + wordLen - - # Are we on the first word following a bullet point? - firstBullet = (wordCount == 2 and bulletPoint) - - if endEscape: - # If the new word ends the input line with ' +', - # add it to the current line. - - (addWord, closeLine, startLine) = (True, True, False) - elif self.vuidAnchor(word): - # If the new word is a Valid Usage anchor, break the - # line afterwards. Note that this should only happen - # immediately after a bullet point, but we don't - # currently check for this. - (addWord, closeLine, startLine) = (True, True, False) - elif newLen > self.margin: - if firstBullet: - # If the word follows a bullet point, add it to - # the current line no matter its length. - - (addWord, closeLine, startLine) = (True, True, False) - elif beginBullet.match(word + ' '): - # If the word *is* a bullet point, add it to - # the current line no matter its length. - # This avoids an innocent inline '-' or '*' - # turning into a bogus bullet point. - - (addWord, closeLine, startLine) = (True, True, False) - else: - # The word overflows, so add it to a new line. - - (addWord, closeLine, startLine) = (False, True, True) - elif (self.breakPeriod and - (wordCount > 2 or not firstBullet) and - self.endSentence(prevWord)): - # If the previous word ends a sentence and - # breakPeriod is set, start a new line. - # The complicated logic allows for leading bullet - # points which are periods (implicitly numbered lists). - # @@@ But not yet for explicitly numbered lists. - - (addWord, closeLine, startLine) = (False, True, True) - - # Add a word to the current line - if addWord: - if outLine: - outLine += ' ' + word - outLineLen = newLen - else: - # Fall through to startLine case if there's no - # current line yet. - startLine = True - - # Add current line to the output paragraph. Force - # starting a new line, although we don't yet know if it - # will ever have contents. - if closeLine: - if outLine: - outPara.append(outLine + '\n') - outLine = None - - # Start a new line and add a word to it - if startLine: - outLine = ''.ljust(self.hangIndent) + word - outLineLen = self.hangIndent + wordLen - - # Track the previous word, for use in breaking at end of - # a sentence - prevWord = word - - # Add this line to the output paragraph. - if outLine: - outPara.append(outLine + '\n') - - return outPara - - def emitPara(self): - """Emit a paragraph, possibly reflowing it depending on the block context. - - Resets the paragraph accumulator.""" - if self.para != []: - if self.vuStack[-1] and self.nextvu is not None: - # If: - # - this paragraph is in a Valid Usage block, - # - VUID tags are being assigned, - # Try to assign VUIDs - - if nestedVuPat.search(self.para[0]): - # Check for nested bullet points. These should not be - # assigned VUIDs, nor present at all, because they break - # the VU extractor. - logWarn(self.filename + ': Invalid nested bullet point in VU block:', self.para[0]) - elif self.vuPrefix not in self.para[0]: - # If: - # - a tag is not already present, and - # - the paragraph is a properly marked-up list item - # Then add a VUID tag starting with the next free ID. - - # Split the first line after the bullet point - matches = vuPat.search(self.para[0]) - if matches is not None: - logDiag('findRefs: Matched vuPat on line:', self.para[0], end='') - head = matches.group('head') - tail = matches.group('tail') - - # Use the first pname: or code: tag in the paragraph as - # the parameter name in the VUID tag. This won't always - # be correct, but should be highly reliable. - for vuLine in self.para: - matches = pnamePat.search(vuLine) - if matches is not None: - break - matches = codePat.search(vuLine) - if matches is not None: - break - - if matches is not None: - paramName = matches.group('param') - else: - paramName = 'None' - logWarn(self.filename, - 'No param name found for VUID tag on line:', - self.para[0]) - - newline = (head + ' [[' + - self.vuFormat.format(self.vuPrefix, - self.apiName, - paramName, - self.nextvu) + ']] ' + tail) - - logDiag('Assigning', self.vuPrefix, self.apiName, self.nextvu, - ' on line:', self.para[0], '->', newline, 'END') - - # Don't actually assign the VUID unless it's in the reserved range - if self.nextvu <= self.maxvu: - if self.nextvu == self.maxvu: - logWarn('Skipping VUID assignment, no more VUIDs available') - self.para[0] = newline - self.nextvu = self.nextvu + 1 - # else: - # There are only a few cases of this, and they're all - # legitimate. Leave detecting this case to another tool - # or hand inspection. - # logWarn(self.filename + ': Unexpected non-bullet item in VU block (harmless if following an ifdef):', - # self.para[0]) - - if self.reflowStack[-1]: - self.printLines(self.reflowPara()) - else: - self.printLines(self.para) - - # Reset the paragraph, including its indentation level - self.para = [] - self.leadIndent = 0 - self.hangIndent = 0 - - def endPara(self, line): - """'line' ends a paragraph and should itself be emitted. - line may be None to indicate EOF or other exception.""" - logDiag('endPara line', self.lineNumber, ': emitting paragraph') - - # Emit current paragraph, this line, and reset tracker - self.emitPara() - - if line: - self.printLines( [ line ] ) - - def endParaContinue(self, line): - """'line' ends a paragraph (unless there's already a paragraph being - accumulated, e.g. len(para) > 0 - currently not implemented)""" - self.endPara(line) - - def endBlock(self, line, reflow = False, vuBlock = False): - """'line' begins or ends a block. - - If beginning a block, tag whether or not to reflow the contents. - - vuBlock is True if the previous line indicates this is a Valid Usage block.""" - self.endPara(line) - - if self.blockStack[-1] == line: - logDiag('endBlock line', self.lineNumber, - ': popping block end depth:', len(self.blockStack), - ':', line, end='') - - # Reset apiName at the end of an open block. - # Open blocks cannot be nested (at present), so this is safe. - if self.isOpenBlockDelimiter(line): - logDiag('reset apiName to empty at line', self.lineNumber) - self.apiName = self.defaultApiName - else: - logDiag('NOT resetting apiName to default at line', self.lineNumber) - - self.blockStack.pop() - self.reflowStack.pop() - self.vuStack.pop() - else: - # Start a block - self.blockStack.append(line) - self.reflowStack.append(reflow) - self.vuStack.append(vuBlock) - - logDiag('endBlock reflow =', reflow, ' line', self.lineNumber, - ': pushing block start depth', len(self.blockStack), - ':', line, end='') - - def endParaBlockReflow(self, line, vuBlock): - """'line' begins or ends a block. The paragraphs in the block *should* be - reformatted (e.g. a NOTE).""" - self.endBlock(line, reflow = True, vuBlock = vuBlock) - - def endParaBlockPassthrough(self, line): - """'line' begins or ends a block. The paragraphs in the block should - *not* be reformatted (e.g. a code listing).""" - self.endBlock(line, reflow = False) - - def addLine(self, line): - """'line' starts or continues a paragraph. - - Paragraphs may have "hanging indent", e.g. - - ``` - * Bullet point... - ... continued - ``` - - In this case, when the higher indentation level ends, so does the - paragraph.""" - logDiag('addLine line', self.lineNumber, ':', line, end='') - - # See https://stackoverflow.com/questions/13648813/what-is-the-pythonic-way-to-count-the-leading-spaces-in-a-string - indent = len(line) - len(line.lstrip()) - - # A hanging paragraph ends due to a less-indented line. - if self.para != [] and indent < self.hangIndent: - logDiag('addLine: line reduces indentation, emit paragraph') - self.emitPara() - - # A bullet point (or something that looks like one) always ends the - # current paragraph. - if beginBullet.match(line): - logDiag('addLine: line matches beginBullet, emit paragraph') - self.emitPara() - - if self.para == []: - # Begin a new paragraph - self.para = [ line ] - self.leadIndent = indent - self.hangIndent = indent - else: - # Add a line to a paragraph. Increase the hanging indentation - # level - once. - if self.hangIndent == self.leadIndent: - self.hangIndent = indent - self.para.append(line) - -def apiMatch(oldname, newname): - """Returns whether oldname and newname match, up to an API suffix. - This should use the API map instead of this heuristic, since aliases - like VkPhysicalDeviceVariablePointerFeatures -> - VkPhysicalDeviceVariablePointersFeatures are not recognized.""" - upper = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - return oldname.rstrip(upper) == newname.rstrip(upper) - -def reflowFile(filename, args): - logDiag('reflow: filename', filename) - - lines = loadFile(filename) - if lines is None: - return - - # Output file handle and reflow object for this file. There are no race - # conditions on overwriting the input, but it's not recommended unless - # you have backing store such as git. - - if args.overwrite: - outFilename = filename - else: - outFilename = args.outDir + '/' + os.path.basename(filename) + args.suffix - - if args.nowrite: - fp = None - else: - try: - fp = open(outFilename, 'w', encoding='utf8') - except: - logWarn('Cannot open output file', outFilename, ':', sys.exc_info()[0]) - return - - state = ReflowState(filename, - margin = args.margin, - file = fp, - reflow = not args.noflow, - nextvu = args.nextvu, - maxvu = args.maxvu) - - for line in lines: - state.incrLineNumber() - - # Is this a title line (leading '= ' followed by text)? - thisTitle = False - - matches = vuidPat.search(line) - if matches is not None: - # If we found a VUID pattern, add the (filename,line) it was - # found at to a list for that VUID, to find duplicates. - vuid = matches.group('vuid') - if vuid not in args.vuidDict: - args.vuidDict[vuid] = [] - args.vuidDict[vuid].append([filename, line]) - - # The logic here is broken. If we're in a non-reflowable block and - # this line *doesn't* end the block, it should always be - # accumulated. - - # Test for a blockCommonReflow delimiter comment first, to avoid - # treating it solely as a end-Paragraph marker comment. - if line == blockCommonReflow: - # Starting or ending a pseudo-block for "common" VU statements. - state.endParaBlockReflow(line, vuBlock = True) - - elif blockReflow.match(line): - # Starting or ending a block whose contents may be reflowed. - # Blocks cannot be nested. - - # Is this is an explicit Valid Usage block? - vuBlock = (state.lineNumber > 1 and - lines[state.lineNumber-2] == '.Valid Usage\n') - - state.endParaBlockReflow(line, vuBlock) - - elif endPara.match(line): - # Ending a paragraph. Emit the current paragraph, if any, and - # prepare to begin a new paragraph. - - state.endPara(line) - - # If this is an include:: line starting the definition of a - # structure or command, track that for use in VUID generation. - - matches = includePat.search(line) - if matches is not None: - generated_type = matches.group('generated_type') - include_type = matches.group('category') - if generated_type == 'api' and include_type in ('protos', 'structs', 'funcpointers'): - apiName = matches.group('entity_name') - if state.apiName != state.defaultApiName: - # This happens when there are multiple API include - # lines in a single block. The style guideline is to - # always place the API which others are promoted to - # first. In virtually all cases, the promoted API - # will differ solely in the vendor suffix (or - # absence of it), which is benign. - if not apiMatch(state.apiName, apiName): - logDiag(f'Promoted API name mismatch at line {state.lineNumber}: {apiName} does not match state.apiName (this is OK if it is just a spelling alias)') - else: - state.apiName = apiName - - elif endParaContinue.match(line): - # For now, always just end the paragraph. - # Could check see if len(para) > 0 to accumulate. - - state.endParaContinue(line) - - # If it's a title line, track that - if line[0:2] == '= ': - thisTitle = True - - elif blockPassthrough.match(line): - # Starting or ending a block whose contents must not be reflowed. - # These are tables, etc. Blocks cannot be nested. - - state.endParaBlockPassthrough(line) - elif state.lastTitle: - # The previous line was a document title line. This line - # is the author / credits line and must not be reflowed. - - state.endPara(line) - else: - # Just accumulate a line to the current paragraph. Watch out for - # hanging indents / bullet-points and track that indent level. - - state.addLine(line) - - # This test looks for disallowed conditionals inside Valid Usage - # blocks, by checking if (a) this line does not start a new VU - # (bullet point) and (b) the previous line starts an asciidoctor - # conditional (ifdef:: or ifndef::). - - if (args.check - and state.vuStack[-1] - and not beginBullet.match(line) - and conditionalStart.match(lines[state.lineNumber-2])): - - logWarn('Detected embedded Valid Usage conditional: {}:{}'.format( - filename, state.lineNumber - 1)) - # Keep track of warning check count - args.warnCount = args.warnCount + 1 - - state.lastTitle = thisTitle - - # Cleanup at end of file - state.endPara(None) - - # Check for sensible block nesting - if len(state.blockStack) > 1: - logWarn('file', filename, - 'mismatched asciidoc block delimiters at EOF:', - state.blockStack[-1]) - - if fp is not None: - fp.close() - - # Update the 'nextvu' value - if args.nextvu != state.nextvu: - logWarn('Updated nextvu to', state.nextvu, 'after file', filename) - args.nextvu = state.nextvu - -def reflowAllAdocFiles(folder_to_reflow, args): - for root, subdirs, files in os.walk(folder_to_reflow): - for file in files: - if file.endswith(conventions.file_suffix): - file_path = os.path.join(root, file) - reflowFile(file_path, args) - for subdir in subdirs: - sub_folder = os.path.join(root, subdir) - print('Sub-folder = %s' % sub_folder) - if subdir.lower() not in conventions.spec_no_reflow_dirs: - print(' Parsing = %s' % sub_folder) - reflowAllAdocFiles(sub_folder, args) - else: - print(' Skipping = %s' % sub_folder) - -# Patterns used to recognize interesting lines in an asciidoc source file. -# These patterns are only compiled once. - -# Explicit Valid Usage list item with one or more leading asterisks -# The re.DOTALL is needed to prevent vuPat.search() from stripping -# the trailing newline. -vuPat = re.compile(r'^(?P<head> [*]+)( *)(?P<tail>.*)', re.DOTALL) - -# VUID with the numeric portion captured in the match object -vuidPat = re.compile(r'VUID-[^-]+-[^-]+-(?P<vuid>[0-9]+)') - -# Pattern matching leading nested bullet points -global nestedVuPat -nestedVuPat = re.compile(r'^ \*\*') - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument('-diag', action='store', dest='diagFile', - help='Set the diagnostic file') - parser.add_argument('-warn', action='store', dest='warnFile', - help='Set the warning file') - parser.add_argument('-log', action='store', dest='logFile', - help='Set the log file for both diagnostics and warnings') - parser.add_argument('-overwrite', action='store_true', - help='Overwrite input filenames instead of writing different output filenames') - parser.add_argument('-out', action='store', dest='outDir', - default='out', - help='Set the output directory in which updated files are generated (default: out)') - parser.add_argument('-nowrite', action='store_true', - help='Do not write output files, for use with -check') - parser.add_argument('-check', action='store', dest='check', - help='Run markup checks and warn if WARN option is given, error exit if FAIL option is given') - parser.add_argument('-checkVUID', action='store', dest='checkVUID', - help='Detect duplicated VUID numbers and warn if WARN option is given, error exit if FAIL option is given') - parser.add_argument('-tagvu', action='store_true', - help='Tag un-tagged Valid Usage statements starting at the value wired into reflow.py') - parser.add_argument('-nextvu', action='store', dest='nextvu', type=int, - default=None, - help='Specify start VUID to use instead of the value wired into vuidCounts.py') - parser.add_argument('-maxvu', action='store', dest='maxvu', type=int, - default=None, - help='Specify maximum VUID instead of the value wired into vuidCounts.py') - parser.add_argument('-branch', action='store', dest='branch', - help='Specify branch to assign VUIDs for') - parser.add_argument('-noflow', action='store_true', dest='noflow', - help='Do not reflow text. Other actions may apply') - parser.add_argument('-margin', action='store', type=int, dest='margin', - default='76', - help='Width to reflow text, defaults to 76 characters') - parser.add_argument('-suffix', action='store', dest='suffix', - default='', - help='Set the suffix added to updated file names (default: none)') - parser.add_argument('files', metavar='filename', nargs='*', - help='a filename to reflow text in') - parser.add_argument('--version', action='version', version='%(prog)s 1.0') - - args = parser.parse_args() - - setLogFile(True, True, args.logFile) - setLogFile(True, False, args.diagFile) - setLogFile(False, True, args.warnFile) - - print('args.margin = ', args.margin) - - if args.overwrite: - logWarn("reflow.py: will overwrite all input files") - - errors = '' - if args.branch is None: - (args.branch, errors) = getBranch() - if args.branch is None: - # This is not fatal unless VUID assignment is required - if args.tagvu: - logErr('Cannot determine current git branch, so cannot assign VUIDs:', errors) - - if args.tagvu and args.nextvu is None: - # Moved here since vuidCounts is only needed in the internal - # repository - from vuidCounts import vuidCounts - - if args.branch not in vuidCounts: - logErr('Branch', args.branch, 'not in vuidCounts, cannot continue') - maxVUID = vuidCounts[args.branch][1] - startVUID = vuidCounts[args.branch][2] - args.nextvu = startVUID - args.maxvu = maxVUID - - if args.nextvu is not None: - logWarn('Tagging untagged Valid Usage statements starting at', args.nextvu) - - # Count of markup check warnings encountered - # This is added to the argparse structure - args.warnCount = 0 - - # Dictionary of VUID numbers found, containing a list of (file, line) on - # which that number was found - # This is added to the argparse structure - args.vuidDict = {} - - # If no files are specified, reflow the entire specification chapters folder - if not args.files: - folder_to_reflow = conventions.spec_reflow_path - logWarn('Reflowing all asciidoc files under', folder_to_reflow) - reflowAllAdocFiles(folder_to_reflow, args) - else: - for file in args.files: - reflowFile(file, args) - - if args.warnCount > 0: - if args.check == 'FAIL': - logErr('Failed with', args.warnCount, 'markup errors detected.\n' + - 'To fix these, you can take actions such as:\n' + - ' * Moving conditionals outside VU start / end without changing VU meaning\n' + - ' * Refactor conditional text using terminology defined conditionally outside the VU itself\n' + - ' * Remove the conditional (allowable when this just affects command / structure / enum names)\n') - else: - logWarn('Total warning count for markup issues is', args.warnCount) - - # Look for duplicated VUID numbers - if args.checkVUID: - dupVUIDs = 0 - for vuid in sorted(args.vuidDict): - found = args.vuidDict[vuid] - if len(found) > 1: - logWarn('Duplicate VUID number {} found in files:'.format(vuid)) - for (file, line) in found: - logWarn(' {}: {}'.format(file, line)) - dupVUIDs = dupVUIDs + 1 - - if dupVUIDs > 0: - if args.checkVUID == 'FAIL': - logErr('Failed with', dupVUIDs, 'duplicated VUID numbers found.\n' + - 'To fix this, either convert these to commonvalidity VUs if possible, or strip\n' + - 'the VUIDs from all but one of the duplicates and regenerate new ones.') - else: - logWarn('Total number of duplicated VUID numbers is', dupVUIDs) - - if args.nextvu is not None and args.nextvu != startVUID: - # Update next free VUID to assign - vuidCounts[args.branch][2] = args.nextvu - try: - reflow_count_file_path = os.path.dirname(os.path.realpath(__file__)) - reflow_count_file_path += '/vuidCounts.py' - reflow_count_file = open(reflow_count_file_path, 'w', encoding='utf8') - print('# Do not edit this file!', file=reflow_count_file) - print('# VUID ranges reserved for branches', file=reflow_count_file) - print('# Key is branch name, value is [ start, end, nextfree ]', file=reflow_count_file) - print('vuidCounts = {', file=reflow_count_file) - for key in sorted(vuidCounts): - print(" '{}': [ {}, {}, {} ],".format( - key, - vuidCounts[key][0], - vuidCounts[key][1], - vuidCounts[key][2]), - file=reflow_count_file) - print('}', file=reflow_count_file) - reflow_count_file.close() - except: - logWarn('Cannot open output count file vuidCounts.py', ':', sys.exc_info()[0]) |