Prepare for move to hardware/google/gfxstream

Move codegen into what will become
hardware/google/gfxstream/codegen/vulkan
and move generated vulkan headers into what will become
hardware/google/gfxstream/common/vulkan.

Bug: 271464937
Test: presubmit
Test: python android/build/python/cmake.py --gfxstream
Change-Id: Ie0465ac72b4a2f7444b595cd1421e2e3cb1d0997

1 files changed, 912 insertions, 0 deletions

diff --git a/codegen/vulkan/scripts/reflow.py b/codegen/vulkan/scripts/reflow.py
new file mode 100755
index 00000000..1897b05c
--- /dev/null
+++ b/codegen/vulkan/scripts/reflow.py
@@ -0,0 +1,912 @@
+#!/usr/bin/python3
+#
+# Copyright 2016-2021 The Khronos Group Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Used for automatic reflow of spec sources to satisfy the agreed layout to
+minimize git churn. Most of the logic has to do with detecting asciidoc
+markup or block types that *shouldn't* be reflowed (tables, code) and
+ignoring them. It's very likely there are many asciidoc constructs not yet
+accounted for in the script, our usage of asciidoc markup is intentionally
+somewhat limited.
+
+Also used to insert identifying tags on explicit Valid Usage statements.
+
+Usage: `reflow.py [-noflow] [-tagvu] [-nextvu #] [-overwrite] [-out dir] [-suffix str] files`
+
+- `-noflow` acts as a passthrough, instead of reflowing text. Other
+  processing may occur.
+- `-tagvu` generates explicit VUID tag for Valid Usage statements which
+  don't already have them.
+- `-nextvu #` starts VUID tag generation at the specified # instead of
+  the value wired into the `reflow.py` script.
+- `-overwrite` updates in place (can be risky, make sure there are backups)
+- `-check FAIL|WARN` runs some simple sanity checks on markup. If the checks
+  fail and the WARN option is given, the script will simply print a warning
+  message. If the checks fail and the FAIL option is given, the script will
+  exit with an error code. FAIL is for use with continuous integration
+  scripts enforcing the checks.
+- `-out` specifies directory to create output file in, default 'out'
+- `-suffix` specifies suffix to add to output files, default ''
+- `files` are asciidoc source files from the spec to reflow.
+"""
+# For error and file-loading interfaces only
+import argparse
+import os
+import re
+import sys
+from reflib import loadFile, logDiag, logWarn, logErr, setLogFile, getBranch
+
+# Vulkan-specific - will consolidate into scripts/ like OpenXR soon
+sys.path.insert(0, 'xml')
+
+from vkconventions import VulkanConventions as APIConventions
+conventions = APIConventions()
+
+# Markup that always ends a paragraph
+#   empty line or whitespace
+#   [block options]
+#   [[anchor]]
+#   //                  comment
+#   <<<<                page break
+#   :attribute-setting
+#   macro-directive::terms
+#   +                   standalone list item continuation
+#   label::             labelled list - label must be standalone
+endPara = re.compile(r'^( *|\[.*\]|//.*|<<<<|:.*|[a-z]+::.*|\+|.*::)$')
+
+# Special case of markup ending a paragraph, used to track the current
+# command/structure. This allows for either OpenXR or Vulkan API path
+# conventions. Nominally it should use the file suffix defined by the API
+# conventions (conventions.file_suffix), except that XR uses '.txt' for
+# generated API include files, not '.adoc' like its other includes.
+includePat = re.compile(
+        r'include::(?P<directory_traverse>((../){1,4}|\{INCS-VAR\}/|\{generated\}/)(generated/)?)(?P<generated_type>[\w]+)/(?P<category>\w+)/(?P<entity_name>[^./]+).txt[\[][\]]')
+
+# Find the first pname: or code: pattern in a Valid Usage statement
+pnamePat = re.compile(r'pname:(?P<param>\{?\w+\}?)')
+codePat = re.compile(r'code:(?P<param>\w+)')
+
+# Markup that's OK in a contiguous paragraph but otherwise passed through
+#   .anything (except .., which indicates a literal block)
+#   === Section Titles
+endParaContinue = re.compile(r'^(\.[^.].*|=+ .*)$')
+
+# Markup for block delimiters whose contents *should* be reformatted
+#   --   (exactly two)  (open block)
+#   **** (4 or more)    (sidebar block - why do we have these?!)
+#   ==== (4 or more)    (example block)
+#   ____ (4 or more)    (quote block)
+blockReflow = re.compile(r'^(--|[*=_]{4,})$')
+
+# Fake block delimiters for "common" VU statements
+blockCommonReflow = '// Common Valid Usage\n'
+
+# Markup for block delimiters whose contents should *not* be reformatted
+#   |=== (3 or more)  (table)
+#   ++++ (4 or more)  (passthrough block)
+#   .... (4 or more)  (literal block)
+#   //// (4 or more)  (comment block)
+#   ---- (4 or more)  (listing block)
+#   ```  (3 or more)  (listing block)
+#   **** (4 or more)  (sidebar block)
+blockPassthrough = re.compile(r'^(\|={3,}|[`]{3}|[\-+./~]{4,})$')
+
+# Markup for introducing lists (hanging paragraphs)
+#   * bullet
+#     ** bullet
+#     -- bullet
+#   . bullet
+#   :: bullet (no longer supported by asciidoctor 2)
+#   {empty}:: bullet
+#   1. list item
+beginBullet = re.compile(r'^ *([*\-.]+|\{empty\}::|::|[0-9]+[.]) ')
+
+# Start of an asciidoctor conditional
+#   ifdef::
+#   ifndef::
+conditionalStart = re.compile(r'^(ifdef|ifndef)::')
+
+# Text that (may) not end sentences
+
+# A single letter followed by a period, typically a middle initial.
+endInitial = re.compile(r'^[A-Z]\.$')
+# An abbreviation, which doesn't (usually) end a line.
+endAbbrev = re.compile(r'(e\.g|i\.e|c\.f|vs)\.$', re.IGNORECASE)
+
+class ReflowState:
+    """State machine for reflowing.
+
+    Represents the state of the reflow operation"""
+    def __init__(self,
+                 filename,
+                 margin = 76,
+                 file = sys.stdout,
+                 breakPeriod = True,
+                 reflow = True,
+                 nextvu = None,
+                 maxvu = None):
+
+        self.blockStack = [ None ]
+        """The last element is a line with the asciidoc block delimiter that's currently in effect,
+        such as '--', '----', '****', '======', or '+++++++++'.
+        This affects whether or not the block contents should be formatted."""
+
+        self.reflowStack = [ True ]
+        """The last element is True or False if the current blockStack contents
+        should be reflowed."""
+        self.vuStack = [ False ]
+        """the last element is True or False if the current blockStack contents
+        are an explicit Valid Usage block."""
+
+        self.margin = margin
+        """margin to reflow text to."""
+
+        self.para = []
+        """list of lines in the paragraph being accumulated.
+        When this is non-empty, there is a current paragraph."""
+
+        self.lastTitle = False
+        """true if the previous line was a document title line
+        (e.g. :leveloffset: 0 - no attempt to track changes to this is made)."""
+
+        self.leadIndent = 0
+        """indent level (in spaces) of the first line of a paragraph."""
+
+        self.hangIndent = 0
+        """indent level of the remaining lines of a paragraph."""
+
+        self.file = file
+        """file handle to write to."""
+
+        self.filename = filename
+        """base name of file being read from."""
+
+        self.lineNumber = 0
+        """line number being read from the input file."""
+
+        self.breakPeriod = breakPeriod
+        """True if justification should break to a new line after the end of a sentence."""
+
+        self.breakInitial = True
+        """True if justification should break to a new line after
+        something that appears to be an initial in someone's name. **TBD**"""
+
+        self.reflow = reflow
+        """True if text should be reflowed, False to pass through unchanged."""
+
+        self.vuPrefix = 'VUID'
+        """Prefix of generated Valid Usage tags"""
+
+        self.vuFormat = '{0}-{1}-{2}-{3:0>5d}'
+        """Format string for generating Valid Usage tags.
+        First argument is vuPrefix, second is command/struct name, third is parameter name, fourth is the tag number."""
+
+        self.nextvu = nextvu
+        """Integer to start tagging un-numbered Valid Usage statements with,
+        or None if no tagging should be done."""
+
+        self.maxvu = maxvu
+        """Maximum tag to use for Valid Usage statements, or None if no
+        tagging should be done."""
+
+        self.defaultApiName = '{refpage}'
+        self.apiName = self.defaultApiName
+        """String name of a Vulkan structure or command for VUID tag
+        generation, or {refpage} if one hasn't been included in this file
+        yet."""
+
+    def incrLineNumber(self):
+        self.lineNumber = self.lineNumber + 1
+
+    def printLines(self, lines):
+        """Print an array of lines with newlines already present"""
+        if len(lines) > 0:
+            logDiag(':: printLines:', len(lines), 'lines: ', lines[0], end='')
+
+        if self.file is not None:
+            for line in lines:
+                print(line, file=self.file, end='')
+
+    def endSentence(self, word):
+        """Return True if word ends with a sentence-period, False otherwise.
+
+        Allows for contraction cases which won't end a line:
+
+         - A single letter (if breakInitial is True)
+         - Abbreviations: 'c.f.', 'e.g.', 'i.e.' (or mixed-case versions)"""
+        if (word[-1:] != '.' or
+            endAbbrev.search(word) or
+                (self.breakInitial and endInitial.match(word))):
+            return False
+
+        return True
+
+    def vuidAnchor(self, word):
+        """Return True if word is a Valid Usage ID Tag anchor."""
+        return (word[0:7] == '[[VUID-')
+
+    def isOpenBlockDelimiter(self, line):
+        """Returns True if line is an open block delimiter."""
+        return line[0:2] == '--'
+
+    def reflowPara(self):
+        """Reflow the current paragraph, respecting the paragraph lead and
+        hanging indentation levels.
+
+        The algorithm also respects trailing '+' signs that indicate embedded newlines,
+        and will not reflow a very long word immediately after a bullet point.
+
+        Just return the paragraph unchanged if the -noflow argument was
+        given."""
+        if not self.reflow:
+            return self.para
+
+        logDiag('reflowPara lead indent = ', self.leadIndent,
+                'hangIndent =', self.hangIndent,
+                'para:', self.para[0], end='')
+
+        # Total words processed (we care about the *first* word vs. others)
+        wordCount = 0
+
+        # Tracks the *previous* word processed. It must not be empty.
+        prevWord = ' '
+
+        # Track the previous line and paragraph being indented, if any
+        outLine = None
+        outPara = []
+
+        for line in self.para:
+            line = line.rstrip()
+            words = line.split()
+
+            # logDiag('reflowPara: input line =', line)
+            numWords = len(words) - 1
+
+            for i in range(0, numWords + 1):
+                word = words[i]
+                wordLen = len(word)
+                wordCount += 1
+
+                endEscape = False
+                if i == numWords and word == '+':
+                    # Trailing ' +' must stay on the same line
+                    endEscape = word
+                    # logDiag('reflowPara last word of line =', word, 'prevWord =', prevWord, 'endEscape =', endEscape)
+                else:
+                    pass
+                    # logDiag('reflowPara wordCount =', wordCount, 'word =', word, 'prevWord =', prevWord)
+
+                if wordCount == 1:
+                    # The first word of the paragraph is treated specially.
+                    # The loop logic becomes trickier if all this code is
+                    # done prior to looping over lines and words, so all the
+                    # setup logic is done here.
+
+                    outPara = []
+                    outLine = ''.ljust(self.leadIndent) + word
+                    outLineLen = self.leadIndent + wordLen
+
+                    # If the paragraph begins with a bullet point, generate
+                    # a hanging indent level if there isn't one already.
+                    if beginBullet.match(self.para[0]):
+                        bulletPoint = True
+                        if len(self.para) > 1:
+                            logDiag('reflowPara first line matches bullet point',
+                                    'but indent already hanging @ input line',
+                                    self.lineNumber)
+                        else:
+                            logDiag('reflowPara first line matches bullet point -'
+                                    'single line, assuming hangIndent @ input line',
+                                    self.lineNumber)
+                            self.hangIndent = outLineLen + 1
+                    else:
+                        bulletPoint = False
+                else:
+                    # Possible actions to take with this word
+                    #
+                    # addWord - add word to current line
+                    # closeLine - append line and start a new (null) one
+                    # startLine - add word to a new line
+
+                    # Default behavior if all the tests below fail is to add
+                    # this word to the current line, and keep accumulating
+                    # that line.
+                    (addWord, closeLine, startLine) = (True, False, False)
+
+                    # How long would this line be if the word were added?
+                    newLen = outLineLen + 1 + wordLen
+
+                    # Are we on the first word following a bullet point?
+                    firstBullet = (wordCount == 2 and bulletPoint)
+
+                    if endEscape:
+                        # If the new word ends the input line with ' +',
+                        # add it to the current line.
+
+                        (addWord, closeLine, startLine) = (True, True, False)
+                    elif self.vuidAnchor(word):
+                        # If the new word is a Valid Usage anchor, break the
+                        # line afterwards. Note that this should only happen
+                        # immediately after a bullet point, but we don't
+                        # currently check for this.
+                        (addWord, closeLine, startLine) = (True, True, False)
+                    elif newLen > self.margin:
+                        if firstBullet:
+                            # If the word follows a bullet point, add it to
+                            # the current line no matter its length.
+
+                            (addWord, closeLine, startLine) = (True, True, False)
+                        elif beginBullet.match(word + ' '):
+                            # If the word *is* a bullet point, add it to
+                            # the current line no matter its length.
+                            # This avoids an innocent inline '-' or '*'
+                            # turning into a bogus bullet point.
+
+                            (addWord, closeLine, startLine) = (True, True, False)
+                        else:
+                            # The word overflows, so add it to a new line.
+
+                            (addWord, closeLine, startLine) = (False, True, True)
+                    elif (self.breakPeriod and
+                          (wordCount > 2 or not firstBullet) and
+                          self.endSentence(prevWord)):
+                        # If the previous word ends a sentence and
+                        # breakPeriod is set, start a new line.
+                        # The complicated logic allows for leading bullet
+                        # points which are periods (implicitly numbered lists).
+                        # @@@ But not yet for explicitly numbered lists.
+
+                        (addWord, closeLine, startLine) = (False, True, True)
+
+                    # Add a word to the current line
+                    if addWord:
+                        if outLine:
+                            outLine += ' ' + word
+                            outLineLen = newLen
+                        else:
+                            # Fall through to startLine case if there's no
+                            # current line yet.
+                            startLine = True
+
+                    # Add current line to the output paragraph. Force
+                    # starting a new line, although we don't yet know if it
+                    # will ever have contents.
+                    if closeLine:
+                        if outLine:
+                            outPara.append(outLine + '\n')
+                            outLine = None
+
+                    # Start a new line and add a word to it
+                    if startLine:
+                        outLine = ''.ljust(self.hangIndent) + word
+                        outLineLen = self.hangIndent + wordLen
+
+                # Track the previous word, for use in breaking at end of
+                # a sentence
+                prevWord = word
+
+        # Add this line to the output paragraph.
+        if outLine:
+            outPara.append(outLine + '\n')
+
+        return outPara
+
+    def emitPara(self):
+        """Emit a paragraph, possibly reflowing it depending on the block context.
+
+        Resets the paragraph accumulator."""
+        if self.para != []:
+            if self.vuStack[-1] and self.nextvu is not None:
+                # If:
+                #   - this paragraph is in a Valid Usage block,
+                #   - VUID tags are being assigned,
+                # Try to assign VUIDs
+
+                if nestedVuPat.search(self.para[0]):
+                    # Check for nested bullet points. These should not be
+                    # assigned VUIDs, nor present at all, because they break
+                    # the VU extractor.
+                    logWarn(self.filename + ': Invalid nested bullet point in VU block:', self.para[0])
+                elif self.vuPrefix not in self.para[0]:
+                    # If:
+                    #   - a tag is not already present, and
+                    #   - the paragraph is a properly marked-up list item
+                    # Then add a VUID tag starting with the next free ID.
+
+                    # Split the first line after the bullet point
+                    matches = vuPat.search(self.para[0])
+                    if matches is not None:
+                        logDiag('findRefs: Matched vuPat on line:', self.para[0], end='')
+                        head = matches.group('head')
+                        tail = matches.group('tail')
+
+                        # Use the first pname: or code: tag in the paragraph as
+                        # the parameter name in the VUID tag. This won't always
+                        # be correct, but should be highly reliable.
+                        for vuLine in self.para:
+                            matches = pnamePat.search(vuLine)
+                            if matches is not None:
+                                break
+                            matches = codePat.search(vuLine)
+                            if matches is not None:
+                                break
+
+                        if matches is not None:
+                            paramName = matches.group('param')
+                        else:
+                            paramName = 'None'
+                            logWarn(self.filename,
+                                    'No param name found for VUID tag on line:',
+                                    self.para[0])
+
+                        newline = (head + ' [[' +
+                                   self.vuFormat.format(self.vuPrefix,
+                                                        self.apiName,
+                                                        paramName,
+                                                        self.nextvu) + ']] ' + tail)
+
+                        logDiag('Assigning', self.vuPrefix, self.apiName, self.nextvu,
+                                ' on line:', self.para[0], '->', newline, 'END')
+
+                        # Don't actually assign the VUID unless it's in the reserved range
+                        if self.nextvu <= self.maxvu:
+                            if self.nextvu == self.maxvu:
+                                logWarn('Skipping VUID assignment, no more VUIDs available')
+                            self.para[0] = newline
+                            self.nextvu = self.nextvu + 1
+                # else:
+                #     There are only a few cases of this, and they're all
+                #     legitimate. Leave detecting this case to another tool
+                #     or hand inspection.
+                #     logWarn(self.filename + ': Unexpected non-bullet item in VU block (harmless if following an ifdef):',
+                #             self.para[0])
+
+            if self.reflowStack[-1]:
+                self.printLines(self.reflowPara())
+            else:
+                self.printLines(self.para)
+
+        # Reset the paragraph, including its indentation level
+        self.para = []
+        self.leadIndent = 0
+        self.hangIndent = 0
+
+    def endPara(self, line):
+        """'line' ends a paragraph and should itself be emitted.
+        line may be None to indicate EOF or other exception."""
+        logDiag('endPara line', self.lineNumber, ': emitting paragraph')
+
+        # Emit current paragraph, this line, and reset tracker
+        self.emitPara()
+
+        if line:
+            self.printLines( [ line ] )
+
+    def endParaContinue(self, line):
+        """'line' ends a paragraph (unless there's already a paragraph being
+        accumulated, e.g. len(para) > 0 - currently not implemented)"""
+        self.endPara(line)
+
+    def endBlock(self, line, reflow = False, vuBlock = False):
+        """'line' begins or ends a block.
+
+        If beginning a block, tag whether or not to reflow the contents.
+
+        vuBlock is True if the previous line indicates this is a Valid Usage block."""
+        self.endPara(line)
+
+        if self.blockStack[-1] == line:
+            logDiag('endBlock line', self.lineNumber,
+                    ': popping block end depth:', len(self.blockStack),
+                    ':', line, end='')
+
+            # Reset apiName at the end of an open block.
+            # Open blocks cannot be nested (at present), so this is safe.
+            if self.isOpenBlockDelimiter(line):
+                logDiag('reset apiName to empty at line', self.lineNumber)
+                self.apiName = self.defaultApiName
+            else:
+                logDiag('NOT resetting apiName to default at line', self.lineNumber)
+
+            self.blockStack.pop()
+            self.reflowStack.pop()
+            self.vuStack.pop()
+        else:
+            # Start a block
+            self.blockStack.append(line)
+            self.reflowStack.append(reflow)
+            self.vuStack.append(vuBlock)
+
+            logDiag('endBlock reflow =', reflow, ' line', self.lineNumber,
+                    ': pushing block start depth', len(self.blockStack),
+                    ':', line, end='')
+
+    def endParaBlockReflow(self, line, vuBlock):
+        """'line' begins or ends a block. The paragraphs in the block *should* be
+        reformatted (e.g. a NOTE)."""
+        self.endBlock(line, reflow = True, vuBlock = vuBlock)
+
+    def endParaBlockPassthrough(self, line):
+        """'line' begins or ends a block. The paragraphs in the block should
+        *not* be reformatted (e.g. a code listing)."""
+        self.endBlock(line, reflow = False)
+
+    def addLine(self, line):
+        """'line' starts or continues a paragraph.
+
+        Paragraphs may have "hanging indent", e.g.
+
+        ```
+          * Bullet point...
+            ... continued
+        ```
+
+        In this case, when the higher indentation level ends, so does the
+        paragraph."""
+        logDiag('addLine line', self.lineNumber, ':', line, end='')
+
+        # See https://stackoverflow.com/questions/13648813/what-is-the-pythonic-way-to-count-the-leading-spaces-in-a-string
+        indent = len(line) - len(line.lstrip())
+
+        # A hanging paragraph ends due to a less-indented line.
+        if self.para != [] and indent < self.hangIndent:
+            logDiag('addLine: line reduces indentation, emit paragraph')
+            self.emitPara()
+
+        # A bullet point (or something that looks like one) always ends the
+        # current paragraph.
+        if beginBullet.match(line):
+            logDiag('addLine: line matches beginBullet, emit paragraph')
+            self.emitPara()
+
+        if self.para == []:
+            # Begin a new paragraph
+            self.para = [ line ]
+            self.leadIndent = indent
+            self.hangIndent = indent
+        else:
+            # Add a line to a paragraph. Increase the hanging indentation
+            # level - once.
+            if self.hangIndent == self.leadIndent:
+                self.hangIndent = indent
+            self.para.append(line)
+
+def apiMatch(oldname, newname):
+    """Returns whether oldname and newname match, up to an API suffix.
+       This should use the API map instead of this heuristic, since aliases
+       like VkPhysicalDeviceVariablePointerFeatures ->
+       VkPhysicalDeviceVariablePointersFeatures are not recognized."""
+    upper = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+    return oldname.rstrip(upper) == newname.rstrip(upper)
+
+def reflowFile(filename, args):
+    logDiag('reflow: filename', filename)
+
+    lines = loadFile(filename)
+    if lines is None:
+        return
+
+    # Output file handle and reflow object for this file. There are no race
+    # conditions on overwriting the input, but it's not recommended unless
+    # you have backing store such as git.
+
+    if args.overwrite:
+        outFilename = filename
+    else:
+        outFilename = args.outDir + '/' + os.path.basename(filename) + args.suffix
+
+    if args.nowrite:
+        fp = None
+    else:
+        try:
+            fp = open(outFilename, 'w', encoding='utf8')
+        except:
+            logWarn('Cannot open output file', outFilename, ':', sys.exc_info()[0])
+            return
+
+    state = ReflowState(filename,
+                        margin = args.margin,
+                        file = fp,
+                        reflow = not args.noflow,
+                        nextvu = args.nextvu,
+                        maxvu = args.maxvu)
+
+    for line in lines:
+        state.incrLineNumber()
+
+        # Is this a title line (leading '= ' followed by text)?
+        thisTitle = False
+
+        matches = vuidPat.search(line)
+        if matches is not None:
+            # If we found a VUID pattern, add the (filename,line) it was
+            # found at to a list for that VUID, to find duplicates.
+            vuid = matches.group('vuid')
+            if vuid not in args.vuidDict:
+                args.vuidDict[vuid] = []
+            args.vuidDict[vuid].append([filename, line])
+
+        # The logic here is broken. If we're in a non-reflowable block and
+        # this line *doesn't* end the block, it should always be
+        # accumulated.
+
+        # Test for a blockCommonReflow delimiter comment first, to avoid
+        # treating it solely as a end-Paragraph marker comment.
+        if line == blockCommonReflow:
+            # Starting or ending a pseudo-block for "common" VU statements.
+            state.endParaBlockReflow(line, vuBlock = True)
+
+        elif blockReflow.match(line):
+            # Starting or ending a block whose contents may be reflowed.
+            # Blocks cannot be nested.
+
+            # Is this is an explicit Valid Usage block?
+            vuBlock = (state.lineNumber > 1 and
+                       lines[state.lineNumber-2] == '.Valid Usage\n')
+
+            state.endParaBlockReflow(line, vuBlock)
+
+        elif endPara.match(line):
+            # Ending a paragraph. Emit the current paragraph, if any, and
+            # prepare to begin a new paragraph.
+
+            state.endPara(line)
+
+            # If this is an include:: line starting the definition of a
+            # structure or command, track that for use in VUID generation.
+
+            matches = includePat.search(line)
+            if matches is not None:
+                generated_type = matches.group('generated_type')
+                include_type = matches.group('category')
+                if generated_type == 'api' and include_type in ('protos', 'structs', 'funcpointers'):
+                    apiName = matches.group('entity_name')
+                    if state.apiName != state.defaultApiName:
+                        # This happens when there are multiple API include
+                        # lines in a single block. The style guideline is to
+                        # always place the API which others are promoted to
+                        # first. In virtually all cases, the promoted API
+                        # will differ solely in the vendor suffix (or
+                        # absence of it), which is benign.
+                        if not apiMatch(state.apiName, apiName):
+                            logDiag(f'Promoted API name mismatch at line {state.lineNumber}: {apiName} does not match state.apiName (this is OK if it is just a spelling alias)')
+                    else:
+                        state.apiName = apiName
+
+        elif endParaContinue.match(line):
+            # For now, always just end the paragraph.
+            # Could check see if len(para) > 0 to accumulate.
+
+            state.endParaContinue(line)
+
+            # If it's a title line, track that
+            if line[0:2] == '= ':
+                thisTitle = True
+
+        elif blockPassthrough.match(line):
+            # Starting or ending a block whose contents must not be reflowed.
+            # These are tables, etc. Blocks cannot be nested.
+
+            state.endParaBlockPassthrough(line)
+        elif state.lastTitle:
+            # The previous line was a document title line. This line
+            # is the author / credits line and must not be reflowed.
+
+            state.endPara(line)
+        else:
+            # Just accumulate a line to the current paragraph. Watch out for
+            # hanging indents / bullet-points and track that indent level.
+
+            state.addLine(line)
+
+            # This test looks for disallowed conditionals inside Valid Usage
+            # blocks, by checking if (a) this line does not start a new VU
+            # (bullet point) and (b) the previous line starts an asciidoctor
+            # conditional (ifdef:: or ifndef::).
+
+            if (args.check
+                and state.vuStack[-1]
+                and not beginBullet.match(line)
+                and conditionalStart.match(lines[state.lineNumber-2])):
+
+                logWarn('Detected embedded Valid Usage conditional: {}:{}'.format(
+                        filename, state.lineNumber - 1))
+                # Keep track of warning check count
+                args.warnCount = args.warnCount + 1
+
+        state.lastTitle = thisTitle
+
+    # Cleanup at end of file
+    state.endPara(None)
+
+    # Check for sensible block nesting
+    if len(state.blockStack) > 1:
+        logWarn('file', filename,
+                'mismatched asciidoc block delimiters at EOF:',
+                state.blockStack[-1])
+
+    if fp is not None:
+        fp.close()
+
+    # Update the 'nextvu' value
+    if args.nextvu != state.nextvu:
+        logWarn('Updated nextvu to', state.nextvu, 'after file', filename)
+        args.nextvu = state.nextvu
+
+def reflowAllAdocFiles(folder_to_reflow, args):
+    for root, subdirs, files in os.walk(folder_to_reflow):
+        for file in files:
+            if file.endswith(conventions.file_suffix):
+                file_path = os.path.join(root, file)
+                reflowFile(file_path, args)
+        for subdir in subdirs:
+            sub_folder = os.path.join(root, subdir)
+            print('Sub-folder = %s' % sub_folder)
+            if subdir.lower() not in conventions.spec_no_reflow_dirs:
+                print('   Parsing = %s' % sub_folder)
+                reflowAllAdocFiles(sub_folder, args)
+            else:
+                print('   Skipping = %s' % sub_folder)
+
+# Patterns used to recognize interesting lines in an asciidoc source file.
+# These patterns are only compiled once.
+
+# Explicit Valid Usage list item with one or more leading asterisks
+# The re.DOTALL is needed to prevent vuPat.search() from stripping
+# the trailing newline.
+vuPat = re.compile(r'^(?P<head>  [*]+)( *)(?P<tail>.*)', re.DOTALL)
+
+# VUID with the numeric portion captured in the match object
+vuidPat = re.compile(r'VUID-[^-]+-[^-]+-(?P<vuid>[0-9]+)')
+
+# Pattern matching leading nested bullet points
+global nestedVuPat
+nestedVuPat = re.compile(r'^  \*\*')
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('-diag', action='store', dest='diagFile',
+                        help='Set the diagnostic file')
+    parser.add_argument('-warn', action='store', dest='warnFile',
+                        help='Set the warning file')
+    parser.add_argument('-log', action='store', dest='logFile',
+                        help='Set the log file for both diagnostics and warnings')
+    parser.add_argument('-overwrite', action='store_true',
+                        help='Overwrite input filenames instead of writing different output filenames')
+    parser.add_argument('-out', action='store', dest='outDir',
+                        default='out',
+                        help='Set the output directory in which updated files are generated (default: out)')
+    parser.add_argument('-nowrite', action='store_true',
+                        help='Do not write output files, for use with -check')
+    parser.add_argument('-check', action='store', dest='check',
+                        help='Run markup checks and warn if WARN option is given, error exit if FAIL option is given')
+    parser.add_argument('-checkVUID', action='store', dest='checkVUID',
+                        help='Detect duplicated VUID numbers and warn if WARN option is given, error exit if FAIL option is given')
+    parser.add_argument('-tagvu', action='store_true',
+                        help='Tag un-tagged Valid Usage statements starting at the value wired into reflow.py')
+    parser.add_argument('-nextvu', action='store', dest='nextvu', type=int,
+                        default=None,
+                        help='Specify start VUID to use instead of the value wired into vuidCounts.py')
+    parser.add_argument('-maxvu', action='store', dest='maxvu', type=int,
+                        default=None,
+                        help='Specify maximum VUID instead of the value wired into vuidCounts.py')
+    parser.add_argument('-branch', action='store', dest='branch',
+                        help='Specify branch to assign VUIDs for')
+    parser.add_argument('-noflow', action='store_true', dest='noflow',
+                        help='Do not reflow text. Other actions may apply')
+    parser.add_argument('-margin', action='store', type=int, dest='margin',
+                        default='76',
+                        help='Width to reflow text, defaults to 76 characters')
+    parser.add_argument('-suffix', action='store', dest='suffix',
+                        default='',
+                        help='Set the suffix added to updated file names (default: none)')
+    parser.add_argument('files', metavar='filename', nargs='*',
+                        help='a filename to reflow text in')
+    parser.add_argument('--version', action='version', version='%(prog)s 1.0')
+
+    args = parser.parse_args()
+
+    setLogFile(True,  True, args.logFile)
+    setLogFile(True, False, args.diagFile)
+    setLogFile(False, True, args.warnFile)
+
+    print('args.margin = ', args.margin)
+
+    if args.overwrite:
+        logWarn("reflow.py: will overwrite all input files")
+
+    errors = ''
+    if args.branch is None:
+        (args.branch, errors) = getBranch()
+    if args.branch is None:
+        # This is not fatal unless VUID assignment is required
+        if args.tagvu:
+            logErr('Cannot determine current git branch, so cannot assign VUIDs:', errors)
+
+    if args.tagvu and args.nextvu is None:
+        # Moved here since vuidCounts is only needed in the internal
+        # repository
+        from vuidCounts import vuidCounts
+
+        if args.branch not in vuidCounts:
+            logErr('Branch', args.branch, 'not in vuidCounts, cannot continue')
+        maxVUID = vuidCounts[args.branch][1]
+        startVUID = vuidCounts[args.branch][2]
+        args.nextvu = startVUID
+        args.maxvu = maxVUID
+
+    if args.nextvu is not None:
+        logWarn('Tagging untagged Valid Usage statements starting at', args.nextvu)
+
+    # Count of markup check warnings encountered
+    # This is added to the argparse structure
+    args.warnCount = 0
+
+    # Dictionary of VUID numbers found, containing a list of (file, line) on
+    # which that number was found
+    # This is added to the argparse structure
+    args.vuidDict = {}
+
+    # If no files are specified, reflow the entire specification chapters folder
+    if not args.files:
+        folder_to_reflow = conventions.spec_reflow_path
+        logWarn('Reflowing all asciidoc files under', folder_to_reflow)
+        reflowAllAdocFiles(folder_to_reflow, args)
+    else:
+        for file in args.files:
+            reflowFile(file, args)
+
+    if args.warnCount > 0:
+        if args.check == 'FAIL':
+            logErr('Failed with', args.warnCount, 'markup errors detected.\n' +
+                   'To fix these, you can take actions such as:\n' +
+                   '  * Moving conditionals outside VU start / end without changing VU meaning\n' +
+                   '  * Refactor conditional text using terminology defined conditionally outside the VU itself\n' +
+                   '  * Remove the conditional (allowable when this just affects command / structure / enum names)\n')
+        else:
+            logWarn('Total warning count for markup issues is', args.warnCount)
+
+    # Look for duplicated VUID numbers
+    if args.checkVUID:
+        dupVUIDs = 0
+        for vuid in sorted(args.vuidDict):
+            found = args.vuidDict[vuid]
+            if len(found) > 1:
+                logWarn('Duplicate VUID number {} found in files:'.format(vuid))
+                for (file, line) in found:
+                    logWarn('    {}: {}'.format(file, line))
+                dupVUIDs = dupVUIDs + 1
+
+        if dupVUIDs > 0:
+            if args.checkVUID == 'FAIL':
+                logErr('Failed with', dupVUIDs, 'duplicated VUID numbers found.\n' +
+                       'To fix this, either convert these to commonvalidity VUs if possible, or strip\n' +
+                       'the VUIDs from all but one of the duplicates and regenerate new ones.')
+            else:
+                logWarn('Total number of duplicated VUID numbers is', dupVUIDs)
+
+    if args.nextvu is not None and args.nextvu != startVUID:
+        # Update next free VUID to assign
+        vuidCounts[args.branch][2] = args.nextvu
+        try:
+            reflow_count_file_path = os.path.dirname(os.path.realpath(__file__))
+            reflow_count_file_path += '/vuidCounts.py'
+            reflow_count_file = open(reflow_count_file_path, 'w', encoding='utf8')
+            print('# Do not edit this file!', file=reflow_count_file)
+            print('# VUID ranges reserved for branches', file=reflow_count_file)
+            print('# Key is branch name, value is [ start, end, nextfree ]', file=reflow_count_file)
+            print('vuidCounts = {', file=reflow_count_file)
+            for key in sorted(vuidCounts):
+                print("    '{}': [ {}, {}, {} ],".format(
+                    key,
+                    vuidCounts[key][0],
+                    vuidCounts[key][1],
+                    vuidCounts[key][2]),
+                    file=reflow_count_file)
+            print('}', file=reflow_count_file)
+            reflow_count_file.close()
+        except:
+            logWarn('Cannot open output count file vuidCounts.py', ':', sys.exc_info()[0])