diff options
Diffstat (limited to 'codegen/vulkan/scripts/spec_tools')
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/__init__.py | 7 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/algo.py | 69 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/attributes.py | 114 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/base_printer.py | 213 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/consistency_tools.py | 697 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/console_printer.py | 274 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/data_structures.py | 58 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/entity_db.py | 659 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/file_process.py | 119 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/html_printer.py | 436 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/macro_checker.py | 220 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/macro_checker_file.py | 1592 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/main.py | 244 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/shared.py | 257 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/util.py | 58 | ||||
| -rw-r--r-- | codegen/vulkan/scripts/spec_tools/validity.py | 216 |
16 files changed, 0 insertions, 5233 deletions
diff --git a/codegen/vulkan/scripts/spec_tools/__init__.py b/codegen/vulkan/scripts/spec_tools/__init__.py deleted file mode 100644 index 34c01f39..00000000 --- a/codegen/vulkan/scripts/spec_tools/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright (c) 2018-2019 Collabora, Ltd. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> diff --git a/codegen/vulkan/scripts/spec_tools/algo.py b/codegen/vulkan/scripts/spec_tools/algo.py deleted file mode 100644 index 3b4c81f4..00000000 --- a/codegen/vulkan/scripts/spec_tools/algo.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright (c) 2019 Collabora, Ltd. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> -"""RecursiveMemoize serves as a base class for a function modeled -as a dictionary computed on-the-fly.""" - - -class RecursiveMemoize: - """Base class for functions that are recursive. - - Derive and implement `def compute(self, key):` to perform the computation: - you may use __getitem__ (aka self[otherkey]) to access the results for - another key. Each value will be computed at most once. Your - function should never return None, since it is used as a sentinel here. - - """ - - def __init__(self, func, key_iterable=None, permit_cycles=False): - """Initialize data structures, and optionally compute/cache the answer - for all elements of an iterable. - - If permit_cycles is False, then __getitem__ on something that's - currently being computed raises an exception. - If permit_cycles is True, then __getitem__ on something that's - currently being computed returns None. - """ - self._compute = func - self.permit_cycles = permit_cycles - self.d = {} - if key_iterable: - # If we were given an iterable, let's populate those. - for key in key_iterable: - _ = self[key] - - def __getitem__(self, key): - """Access the result of computing the function on the input. - - Performed lazily and cached. - Implement `def compute(self, key):` with the actual function, - which will be called on demand.""" - if key in self.d: - ret = self.d[key] - # Detect "we're computing this" sentinel and - # fail if cycles not permitted - if ret is None and not self.permit_cycles: - raise RuntimeError("Cycle detected when computing function: " + - "f({}) depends on itself".format(key)) - # return the memoized value - # (which might be None if we're in a cycle that's permitted) - return ret - - # Set sentinel for "we're computing this" - self.d[key] = None - # Delegate to function to actually compute - ret = self._compute(key) - # Memoize - self.d[key] = ret - - return ret - - def get_dict(self): - """Return the dictionary where memoized results are stored. - - DO NOT MODIFY!""" - return self.d diff --git a/codegen/vulkan/scripts/spec_tools/attributes.py b/codegen/vulkan/scripts/spec_tools/attributes.py deleted file mode 100644 index ef771811..00000000 --- a/codegen/vulkan/scripts/spec_tools/attributes.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright 2013-2021 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 -"""Utilities for working with attributes of the XML registry.""" - -import re - -_PARAM_REF_NAME_RE = re.compile( - r"(?P<name>[\w]+)(?P<brackets>\[\])?(?P<delim>\.|::|->)?") - - -def _split_param_ref(val): - return [name for name, _, _ in _PARAM_REF_NAME_RE.findall(val)] - - -def _human_readable_deref(val, make_param_name=None): - """Turn the "name[].member[]" notation into plain English.""" - parts = [] - matches = _PARAM_REF_NAME_RE.findall(val) - for name, brackets, delim in reversed(matches): - if make_param_name: - name = make_param_name(name) - if delim: - parts.append('member of') - if brackets: - parts.append('each element of') - parts.append('the') - parts.append(name) - parts.append('parameter') - return ' '.join(parts) - - -class LengthEntry: - """An entry in a (comma-separated) len attribute""" - NULL_TERMINATED_STRING = 'null-terminated' - MATH_STRING = 'latexmath:' - - def __init__(self, val): - self.full_reference = val - self.other_param_name = None - self.null_terminated = False - self.number = None - self.math = None - self.param_ref_parts = None - if val == LengthEntry.NULL_TERMINATED_STRING: - self.null_terminated = True - return - - if val.startswith(LengthEntry.MATH_STRING): - self.math = val.replace(LengthEntry.MATH_STRING, '')[1:-1] - return - - if val.isdigit(): - self.number = int(val) - return - - # Must be another param name. - self.param_ref_parts = _split_param_ref(val) - self.other_param_name = self.param_ref_parts[0] - - def __str__(self): - return self.full_reference - - def get_human_readable(self, make_param_name=None): - assert(self.other_param_name) - return _human_readable_deref(self.full_reference, make_param_name=make_param_name) - - def __repr__(self): - "Formats an object for repr(), debugger display, etc." - return 'spec_tools.attributes.LengthEntry("{}")'.format(self.full_reference) - - @staticmethod - def parse_len_from_param(param): - """Get a list of LengthEntry.""" - len_str = param.get('len') - if len_str is None: - return None - return [LengthEntry(elt) for elt in len_str.split(',')] - - -class ExternSyncEntry: - """An entry in a (comma-separated) externsync attribute""" - - TRUE_STRING = 'true' - TRUE_WITH_CHILDREN_STRING = 'true_with_children' - - def __init__(self, val): - self.full_reference = val - self.entirely_extern_sync = (val in (ExternSyncEntry.TRUE_STRING, ExternSyncEntry.TRUE_WITH_CHILDREN_STRING)) - self.children_extern_sync = (val == ExternSyncEntry.TRUE_WITH_CHILDREN_STRING) - if self.entirely_extern_sync: - return - - self.param_ref_parts = _split_param_ref(val) - self.member = self.param_ref_parts[0] - - def get_human_readable(self, make_param_name=None): - assert(not self.entirely_extern_sync) - return _human_readable_deref(self.full_reference, make_param_name=make_param_name) - - @staticmethod - def parse_externsync_from_param(param): - """Get a list of ExternSyncEntry.""" - sync_str = param.get('externsync') - if sync_str is None: - return None - return [ExternSyncEntry(elt) for elt in sync_str.split(',')] - - def __repr__(self): - "Formats an object for repr(), debugger display, etc." - return 'spec_tools.attributes.ExternSyncEntry("{}")'.format(self.full_reference) - diff --git a/codegen/vulkan/scripts/spec_tools/base_printer.py b/codegen/vulkan/scripts/spec_tools/base_printer.py deleted file mode 100644 index f48905ac..00000000 --- a/codegen/vulkan/scripts/spec_tools/base_printer.py +++ /dev/null @@ -1,213 +0,0 @@ -"""Provides the BasePrinter base class for MacroChecker/Message output techniques.""" - -# Copyright (c) 2018-2019 Collabora, Ltd. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> - -from abc import ABC, abstractmethod -from pathlib import Path - -from .macro_checker import MacroChecker -from .macro_checker_file import MacroCheckerFile -from .shared import EntityData, Message, MessageContext, MessageType - - -def getColumn(message_context): - """Return the (zero-based) column number of the message context. - - If a group is specified: returns the column of the start of the group. - If no group, but a match is specified: returns the column of the start of - the match. - If no match: returns column 0 (whole line). - """ - if not message_context.match: - # whole line - return 0 - if message_context.group is not None: - return message_context.match.start(message_context.group) - return message_context.match.start() - - -class BasePrinter(ABC): - """Base class for a way of outputting results of a checker execution.""" - - def __init__(self): - """Constructor.""" - self._cwd = None - - def close(self): - """Write the tail end of the output and close it, if applicable. - - Override if you want to print a summary or are writing to a file. - """ - pass - - ### - # Output methods: these should all print/output directly. - def output(self, obj): - """Output any object. - - Delegates to other output* methods, if type known, - otherwise uses self.outputFallback(). - """ - if isinstance(obj, Message): - self.outputMessage(obj) - elif isinstance(obj, MacroCheckerFile): - self.outputCheckerFile(obj) - elif isinstance(obj, MacroChecker): - self.outputChecker(obj) - else: - self.outputFallback(self.formatBrief(obj)) - - @abstractmethod - def outputResults(self, checker, broken_links=True, - missing_includes=False): - """Output the full results of a checker run. - - Must be implemented. - - Typically will call self.output() on the MacroChecker, - as well as calling self.outputBrokenAndMissing() - """ - raise NotImplementedError - - @abstractmethod - def outputBrokenLinks(self, checker, broken): - """Output the collection of broken links. - - `broken` is a dictionary of entity names: usage contexts. - - Must be implemented. - - Called by self.outputBrokenAndMissing() if requested. - """ - raise NotImplementedError - - @abstractmethod - def outputMissingIncludes(self, checker, missing): - """Output a table of missing includes. - - `missing` is a iterable entity names. - - Must be implemented. - - Called by self.outputBrokenAndMissing() if requested. - """ - raise NotImplementedError - - def outputChecker(self, checker): - """Output the contents of a MacroChecker object. - - Default implementation calls self.output() on every MacroCheckerFile. - """ - for f in checker.files: - self.output(f) - - def outputCheckerFile(self, fileChecker): - """Output the contents of a MacroCheckerFile object. - - Default implementation calls self.output() on every Message. - """ - for m in fileChecker.messages: - self.output(m) - - def outputBrokenAndMissing(self, checker, broken_links=True, - missing_includes=False): - """Outputs broken links and missing includes, if desired. - - Delegates to self.outputBrokenLinks() (if broken_links==True) - and self.outputMissingIncludes() (if missing_includes==True). - """ - if broken_links: - broken = checker.getBrokenLinks() - if broken: - self.outputBrokenLinks(checker, broken) - if missing_includes: - missing = checker.getMissingUnreferencedApiIncludes() - if missing: - self.outputMissingIncludes(checker, missing) - - @abstractmethod - def outputMessage(self, msg): - """Output a Message. - - Must be implemented. - """ - raise NotImplementedError - - @abstractmethod - def outputFallback(self, msg): - """Output some text in a general way. - - Must be implemented. - """ - raise NotImplementedError - - ### - # Format methods: these should all return a string. - def formatContext(self, context, _message_type=None): - """Format a message context in a verbose way, if applicable. - - May override, default implementation delegates to - self.formatContextBrief(). - """ - return self.formatContextBrief(context) - - def formatContextBrief(self, context, _with_color=True): - """Format a message context in a brief way. - - May override, default is relativeFilename:line:column - """ - return '{}:{}:{}'.format(self.getRelativeFilename(context.filename), - context.lineNum, getColumn(context)) - - def formatMessageTypeBrief(self, message_type, _with_color=True): - """Format a message type in a brief way. - - May override, default is message_type: - """ - return '{}:'.format(message_type) - - def formatEntityBrief(self, entity_data, _with_color=True): - """Format an entity in a brief way. - - May override, default is macro:entity. - """ - return '{}:{}'.format(entity_data.macro, entity_data.entity) - - def formatBrief(self, obj, with_color=True): - """Format any object in a brief way. - - Delegates to other format*Brief methods, if known, - otherwise uses str(). - """ - if isinstance(obj, MessageContext): - return self.formatContextBrief(obj, with_color) - if isinstance(obj, MessageType): - return self.formatMessageTypeBrief(obj, with_color) - if isinstance(obj, EntityData): - return self.formatEntityBrief(obj, with_color) - return str(obj) - - @property - def cwd(self): - """Get the current working directory, fully resolved. - - Lazy initialized. - """ - if not self._cwd: - self._cwd = Path('.').resolve() - return self._cwd - - ### - # Helper function - def getRelativeFilename(self, fn): - """Return the given filename relative to the current directory, - if possible. - """ - try: - return str(Path(fn).relative_to(self.cwd)) - except ValueError: - return str(Path(fn)) diff --git a/codegen/vulkan/scripts/spec_tools/consistency_tools.py b/codegen/vulkan/scripts/spec_tools/consistency_tools.py deleted file mode 100644 index c256a724..00000000 --- a/codegen/vulkan/scripts/spec_tools/consistency_tools.py +++ /dev/null @@ -1,697 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright (c) 2019 Collabora, Ltd. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> -"""Provides utilities to write a script to verify XML registry consistency.""" - -import re - -import networkx as nx - -from .algo import RecursiveMemoize -from .attributes import ExternSyncEntry, LengthEntry -from .data_structures import DictOfStringSets -from .util import findNamedElem, getElemName - - -class XMLChecker: - def __init__(self, entity_db, conventions, manual_types_to_codes=None, - forward_only_types_to_codes=None, - reverse_only_types_to_codes=None, - suppressions=None): - """Set up data structures. - - May extend - call: - `super().__init__(db, conventions, manual_types_to_codes)` - as the last statement in your function. - - manual_types_to_codes is a dictionary of hard-coded - "manual" return codes: - the codes of the value are available for a command if-and-only-if - the key type is passed as an input. - - forward_only_types_to_codes is additional entries to the above - that should only be used in the "forward" direction - (arg type implies return code) - - reverse_only_types_to_codes is additional entries to - manual_types_to_codes that should only be used in the - "reverse" direction - (return code implies arg type) - """ - self.fail = False - self.entity = None - self.errors = DictOfStringSets() - self.warnings = DictOfStringSets() - self.db = entity_db - self.reg = entity_db.registry - self.handle_data = HandleData(self.reg) - self.conventions = conventions - - self.CONST_RE = re.compile(r"\bconst\b") - self.ARRAY_RE = re.compile(r"\[[^]]+\]") - - # Init memoized properties - self._handle_data = None - - if not manual_types_to_codes: - manual_types_to_codes = {} - if not reverse_only_types_to_codes: - reverse_only_types_to_codes = {} - if not forward_only_types_to_codes: - forward_only_types_to_codes = {} - - reverse_codes = DictOfStringSets(reverse_only_types_to_codes) - forward_codes = DictOfStringSets(forward_only_types_to_codes) - for k, v in manual_types_to_codes.items(): - forward_codes.add(k, v) - reverse_codes.add(k, v) - - self.forward_only_manual_types_to_codes = forward_codes.get_dict() - self.reverse_only_manual_types_to_codes = reverse_codes.get_dict() - - # The presence of some types as input to a function imply the - # availability of some return codes. - self.input_type_to_codes = compute_type_to_codes( - self.handle_data, - forward_codes, - extra_op=self.add_extra_codes) - - # Some return codes require a type (or its child) in the input. - self.codes_requiring_input_type = compute_codes_requiring_type( - self.handle_data, - reverse_codes - ) - - specified_codes = set(self.codes_requiring_input_type.keys()) - for codes in self.forward_only_manual_types_to_codes.values(): - specified_codes.update(codes) - for codes in self.reverse_only_manual_types_to_codes.values(): - specified_codes.update(codes) - for codes in self.input_type_to_codes.values(): - specified_codes.update(codes) - - unrecognized = specified_codes - self.return_codes - if unrecognized: - raise RuntimeError("Return code mentioned in script that isn't in the registry: " + - ', '.join(unrecognized)) - - self.referenced_input_types = ReferencedTypes(self.db, self.is_input) - self.referenced_api_types = ReferencedTypes(self.db, self.is_api_type) - if not suppressions: - suppressions = {} - self.suppressions = DictOfStringSets(suppressions) - - def is_api_type(self, member_elem): - """Return true if the member/parameter ElementTree passed is from this API. - - May override or extend.""" - membertext = "".join(member_elem.itertext()) - - return self.conventions.type_prefix in membertext - - def is_input(self, member_elem): - """Return true if the member/parameter ElementTree passed is - considered "input". - - May override or extend.""" - membertext = "".join(member_elem.itertext()) - - if self.conventions.type_prefix not in membertext: - return False - - ret = True - # Const is always input. - if self.CONST_RE.search(membertext): - ret = True - - # Arrays and pointers that aren't const are always output. - elif "*" in membertext: - ret = False - elif self.ARRAY_RE.search(membertext): - ret = False - - return ret - - def add_extra_codes(self, types_to_codes): - """Add any desired entries to the types-to-codes DictOfStringSets - before performing "ancestor propagation". - - Passed to compute_type_to_codes as the extra_op. - - May override.""" - pass - - def should_skip_checking_codes(self, name): - """Return True if more than the basic validation of return codes should - be skipped for a command. - - May override.""" - - return self.conventions.should_skip_checking_codes - - def get_codes_for_command_and_type(self, cmd_name, type_name): - """Return a set of error codes expected due to having - an input argument of type type_name. - - The cmd_name is passed for use by extending methods. - - May extend.""" - return self.input_type_to_codes.get(type_name, set()) - - def check(self): - """Iterate through the registry, looking for consistency problems. - - Outputs error messages at the end.""" - # Iterate through commands, looking for consistency problems. - for name, info in self.reg.cmddict.items(): - self.set_error_context(entity=name, elem=info.elem) - - self.check_command(name, info) - - for name, info in self.reg.typedict.items(): - cat = info.elem.get('category') - if not cat: - # This is an external thing, skip it. - continue - self.set_error_context(entity=name, elem=info.elem) - - self.check_type(name, info, cat) - - # check_extension is called for all extensions, even 'disabled' - # ones, but some checks may be skipped depending on extension - # status. - for name, info in self.reg.extdict.items(): - self.set_error_context(entity=name, elem=info.elem) - self.check_extension(name, info) - - entities_with_messages = set( - self.errors.keys()).union(self.warnings.keys()) - if entities_with_messages: - print('xml_consistency/consistency_tools error and warning messages follow.') - - for entity in entities_with_messages: - print() - print('-------------------') - print('Messages for', entity) - print() - messages = self.errors.get(entity) - if messages: - for m in messages: - print('Error:', m) - - messages = self.warnings.get(entity) - if messages: - for m in messages: - print('Warning:', m) - - def check_param(self, param): - """Check a member of a struct or a param of a function. - - Called from check_params. - - May extend.""" - param_name = getElemName(param) - externsyncs = ExternSyncEntry.parse_externsync_from_param(param) - if externsyncs: - for entry in externsyncs: - if entry.entirely_extern_sync: - if len(externsyncs) > 1: - self.record_error("Comma-separated list in externsync attribute includes 'true' for", - param_name) - else: - # member name - # TODO only looking at the superficial feature here, - # not entry.param_ref_parts - if entry.member != param_name: - self.record_error("externsync attribute for", param_name, - "refers to some other member/parameter:", entry.member) - - def check_params(self, params): - """Check the members of a struct or params of a function. - - Called from check_type and check_command. - - May extend.""" - for param in params: - self.check_param(param) - - # Check for parameters referenced by len= attribute - lengths = LengthEntry.parse_len_from_param(param) - if lengths: - for entry in lengths: - if not entry.other_param_name: - continue - # TODO only looking at the superficial feature here, - # not entry.param_ref_parts - other_param = findNamedElem(params, entry.other_param_name) - if other_param is None: - self.record_error("References a non-existent parameter/member in the length of", - getElemName(param), ":", entry.other_param_name) - - def check_type(self, name, info, category): - """Check a type's XML data for consistency. - - Called from check. - - May extend.""" - if category == 'struct': - if not name.startswith(self.conventions.type_prefix): - self.record_error("Name does not start with", - self.conventions.type_prefix) - members = info.elem.findall('member') - self.check_params(members) - - # Check the structure type member, if present. - type_member = findNamedElem( - members, self.conventions.structtype_member_name) - if type_member is not None: - val = type_member.get('values') - if val: - expected = self.conventions.generate_structure_type_from_name( - name) - if val != expected: - self.record_error("Type has incorrect type-member value: expected", - expected, "got", val) - - elif category == "bitmask": - if 'Flags' not in name: - self.record_error("Name of bitmask doesn't include 'Flags'") - - def check_extension(self, name, info): - """Check an extension's XML data for consistency. - - Called from check. - - May extend.""" - pass - - def check_command(self, name, info): - """Check a command's XML data for consistency. - - Called from check. - - May extend.""" - elem = info.elem - - self.check_params(elem.findall('param')) - - # Some minimal return code checking - errorcodes = elem.get("errorcodes") - if errorcodes: - errorcodes = errorcodes.split(",") - else: - errorcodes = [] - - successcodes = elem.get("successcodes") - if successcodes: - successcodes = successcodes.split(",") - else: - successcodes = [] - - if not successcodes and not errorcodes: - # Early out if no return codes. - return - - # Create a set for each group of codes, and check that - # they aren't duplicated within or between groups. - errorcodes_set = set(errorcodes) - if len(errorcodes) != len(errorcodes_set): - self.record_error("Contains a duplicate in errorcodes") - - successcodes_set = set(successcodes) - if len(successcodes) != len(successcodes_set): - self.record_error("Contains a duplicate in successcodes") - - if not successcodes_set.isdisjoint(errorcodes_set): - self.record_error("Has errorcodes and successcodes that overlap") - - self.check_command_return_codes_basic( - name, info, successcodes_set, errorcodes_set) - - # Continue to further return code checking if not "complicated" - if not self.should_skip_checking_codes(name): - codes_set = successcodes_set.union(errorcodes_set) - self.check_command_return_codes( - name, info, successcodes_set, errorcodes_set, codes_set) - - def check_command_return_codes_basic(self, name, info, - successcodes, errorcodes): - """Check a command's return codes for consistency. - - Called from check_command on every command. - - May extend.""" - - # Check that all error codes include _ERROR_, - # and that no success codes do. - for code in errorcodes: - if "_ERROR_" not in code: - self.record_error( - code, "in errorcodes but doesn't contain _ERROR_") - - for code in successcodes: - if "_ERROR_" in code: - self.record_error(code, "in successcodes but contain _ERROR_") - - def check_command_return_codes(self, name, type_info, - successcodes, errorcodes, - codes): - """Check a command's return codes in-depth for consistency. - - Called from check_command, only if - `self.should_skip_checking_codes(name)` is False. - - May extend.""" - referenced_input = self.referenced_input_types[name] - referenced_types = self.referenced_api_types[name] - - # Check that we have all the codes we expect, based on input types. - for referenced_type in referenced_input: - required_codes = self.get_codes_for_command_and_type( - name, referenced_type) - missing_codes = required_codes - codes - if missing_codes: - path = self.referenced_input_types.shortest_path( - name, referenced_type) - path_str = " -> ".join(path) - self.record_error("Missing expected return code(s)", - ",".join(missing_codes), - "implied because of input of type", - referenced_type, - "found via path", - path_str) - - # Check that, for each code returned by this command that we can - # associate with a type, we have some type that can provide it. - # e.g. can't have INSTANCE_LOST without an Instance - # (or child of Instance). - for code in codes: - - required_types = self.codes_requiring_input_type.get(code) - if not required_types: - # This code doesn't have a known requirement - continue - - # TODO: do we look at referenced_types or referenced_input here? - # the latter is stricter - if not referenced_types.intersection(required_types): - self.record_error("Unexpected return code", code, - "- none of these types:", - required_types, - "found in the set of referenced types", - referenced_types) - - ### - # Utility properties/methods - ### - - def set_error_context(self, entity=None, elem=None): - """Set the entity and/or element for future record_error calls.""" - self.entity = entity - self.elem = elem - self.name = getElemName(elem) - self.entity_suppressions = self.suppressions.get(getElemName(elem)) - - def record_error(self, *args, **kwargs): - """Record failure and an error message for the current context.""" - message = " ".join((str(x) for x in args)) - - if self._is_message_suppressed(message): - return - - message = self._prepend_sourceline_to_message(message, **kwargs) - self.fail = True - self.errors.add(self.entity, message) - - def record_warning(self, *args, **kwargs): - """Record a warning message for the current context.""" - message = " ".join((str(x) for x in args)) - - if self._is_message_suppressed(message): - return - - message = self._prepend_sourceline_to_message(message, **kwargs) - self.warnings.add(self.entity, message) - - def _is_message_suppressed(self, message): - """Return True if the given message, for this entity, should be suppressed.""" - if not self.entity_suppressions: - return False - for suppress in self.entity_suppressions: - if suppress in message: - return True - - return False - - def _prepend_sourceline_to_message(self, message, **kwargs): - """Prepend a file and/or line reference to the message, if possible. - - If filename is given as a keyword argument, it is used on its own. - - If filename is not given, this will attempt to retrieve the filename and line from an XML element. - If 'elem' is given as a keyword argument and is not None, it is used to find the line. - If 'elem' is given as None, no XML elements are looked at. - If 'elem' is not supplied, the error context element is used. - - If using XML, the filename, if available, is retrieved from the Registry class. - If using XML and python-lxml is installed, the source line is retrieved from whatever element is chosen.""" - fn = kwargs.get('filename') - sourceline = None - - if fn is None: - elem = kwargs.get('elem', self.elem) - if elem is not None: - sourceline = getattr(elem, 'sourceline', None) - if self.reg.filename: - fn = self.reg.filename - - if fn is None and sourceline is None: - return message - - if fn is None: - return "Line {}: {}".format(sourceline, message) - - if sourceline is None: - return "{}: {}".format(fn, message) - - return "{}:{}: {}".format(fn, sourceline, message) - - -class HandleParents(RecursiveMemoize): - def __init__(self, handle_types): - self.handle_types = handle_types - - def compute(handle_type): - immediate_parent = self.handle_types[handle_type].elem.get( - 'parent') - - if immediate_parent is None: - # No parents, no need to recurse - return [] - - # Support multiple (alternate) parents - immediate_parents = immediate_parent.split(',') - - # Recurse, combine, and return - all_parents = immediate_parents[:] - for parent in immediate_parents: - all_parents.extend(self[parent]) - return all_parents - - super().__init__(compute, handle_types.keys()) - - -def _always_true(x): - return True - - -class ReferencedTypes(RecursiveMemoize): - """Find all types(optionally matching a predicate) that are referenced - by a struct or function, recursively.""" - - def __init__(self, db, predicate=None): - """Initialize. - - Provide an EntityDB object and a predicate function.""" - self.db = db - - self.predicate = predicate - if not self.predicate: - # Default predicate is "anything goes" - self.predicate = _always_true - - self._directly_referenced = {} - self.graph = nx.DiGraph() - - def compute(type_name): - """Compute and return all types referenced by type_name, recursively, that satisfy the predicate. - - Called by the [] operator in the base class.""" - types = self.directly_referenced(type_name) - if not types: - return types - - all_types = set() - all_types.update(types) - for t in types: - referenced = self[t] - if referenced is not None: - # If not leading to a cycle - all_types.update(referenced) - return all_types - - # Initialize base class - super().__init__(compute, permit_cycles=True) - - def shortest_path(self, source, target): - """Get the shortest path between one type/function name and another.""" - # Trigger computation - _ = self[source] - - return nx.algorithms.shortest_path(self.graph, source=source, target=target) - - def directly_referenced(self, type_name): - """Get all types referenced directly by type_name that satisfy the predicate. - - Memoizes its results.""" - if type_name not in self._directly_referenced: - members = self.db.getMemberElems(type_name) - if members: - types = ((member, member.find("type")) for member in members) - self._directly_referenced[type_name] = set(type_elem.text for (member, type_elem) in types - if type_elem is not None and self.predicate(member)) - - else: - self._directly_referenced[type_name] = set() - - # Update graph - self.graph.add_node(type_name) - self.graph.add_edges_from((type_name, t) - for t in self._directly_referenced[type_name]) - - return self._directly_referenced[type_name] - - -class HandleData: - """Data about all the handle types available in an API specification.""" - - def __init__(self, registry): - self.reg = registry - self._handle_types = None - self._ancestors = None - self._descendants = None - - @property - def handle_types(self): - """Return a dictionary of handle type names to type info.""" - if not self._handle_types: - # First time requested - compute it. - self._handle_types = { - type_name: type_info - for type_name, type_info in self.reg.typedict.items() - if type_info.elem.get('category') == 'handle' - } - return self._handle_types - - @property - def ancestors_dict(self): - """Return a dictionary of handle type names to sets of ancestors.""" - if not self._ancestors: - # First time requested - compute it. - self._ancestors = HandleParents(self.handle_types).get_dict() - return self._ancestors - - @property - def descendants_dict(self): - """Return a dictionary of handle type names to sets of descendants.""" - if not self._descendants: - # First time requested - compute it. - - handle_parents = self.ancestors_dict - - def get_descendants(handle): - return set(h for h in handle_parents.keys() - if handle in handle_parents[h]) - - self._descendants = { - h: get_descendants(h) - for h in handle_parents.keys() - } - return self._descendants - - -def compute_type_to_codes(handle_data, types_to_codes, extra_op=None): - """Compute a DictOfStringSets of input type to required return codes. - - - handle_data is a HandleData instance. - - d is a dictionary of type names to strings or string collections of - return codes. - - extra_op, if any, is called after populating the output from the input - dictionary, but before propagation of parent codes to child types. - extra_op is called with the in-progress DictOfStringSets. - - Returns a DictOfStringSets of input type name to set of required return - code names. - """ - # Initialize with the supplied "manual" codes - types_to_codes = DictOfStringSets(types_to_codes) - - # Dynamically generate more codes, if desired - if extra_op: - extra_op(types_to_codes) - - # Final post-processing - - # Any handle can result in its parent handle's codes too. - - handle_ancestors = handle_data.ancestors_dict - - extra_handle_codes = {} - for handle_type, ancestors in handle_ancestors.items(): - codes = set() - # The sets of return codes corresponding to each ancestor type. - ancestors_codes = (types_to_codes.get(ancestor, set()) - for ancestor in ancestors) - codes.union(*ancestors_codes) - # for parent_codes in ancestors_codes: - # codes.update(parent_codes) - extra_handle_codes[handle_type] = codes - - for handle_type, extras in extra_handle_codes.items(): - types_to_codes.add(handle_type, extras) - - return types_to_codes - - -def compute_codes_requiring_type(handle_data, types_to_codes, registry=None): - """Compute a DictOfStringSets of return codes to a set of input types able - to provide the ability to generate that code. - - handle_data is a HandleData instance. - d is a dictionary of input types to associated return codes(same format - as for input to compute_type_to_codes, may use same dict). - This will invert that relationship, and also permit any "child handles" - to satisfy a requirement for a parent in producing a code. - - Returns a DictOfStringSets of return code name to the set of parameter - types that would allow that return code. - """ - # Use DictOfStringSets to normalize the input into a dict with values - # that are sets of strings - in_dict = DictOfStringSets(types_to_codes) - - handle_descendants = handle_data.descendants_dict - - out = DictOfStringSets() - for in_type, code_set in in_dict.items(): - descendants = handle_descendants.get(in_type) - for code in code_set: - out.add(code, in_type) - if descendants: - out.add(code, descendants) - - return out diff --git a/codegen/vulkan/scripts/spec_tools/console_printer.py b/codegen/vulkan/scripts/spec_tools/console_printer.py deleted file mode 100644 index 18acabfd..00000000 --- a/codegen/vulkan/scripts/spec_tools/console_printer.py +++ /dev/null @@ -1,274 +0,0 @@ -"""Defines ConsolePrinter, a BasePrinter subclass for appealing console output.""" - -# Copyright (c) 2018-2019 Collabora, Ltd. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> - -from sys import stdout - -from .base_printer import BasePrinter -from .shared import (colored, getHighlightedRange, getInterestedRange, - toNameAndLine) - -try: - from tabulate import tabulate_impl - HAVE_TABULATE = True -except ImportError: - HAVE_TABULATE = False - - -def colWidth(collection, columnNum): - """Compute the required width of a column in a collection of row-tuples.""" - MIN_PADDING = 5 - return MIN_PADDING + max((len(row[columnNum]) for row in collection)) - - -def alternateTabulate(collection, headers=None): - """Minimal re-implementation of the tabulate module.""" - # We need a list, not a generator or anything else. - if not isinstance(collection, list): - collection = list(collection) - - # Empty collection means no table - if not collection: - return None - - if headers is None: - fullTable = collection - else: - underline = ['-' * len(header) for header in headers] - fullTable = [headers, underline] + collection - widths = [colWidth(collection, colNum) - for colNum in range(len(fullTable[0]))] - widths[-1] = None - - lines = [] - for row in fullTable: - fields = [] - for data, width in zip(row, widths): - if width: - spaces = ' ' * (width - len(data)) - fields.append(data + spaces) - else: - fields.append(data) - lines.append(''.join(fields)) - return '\n'.join(lines) - - -def printTabulated(collection, headers=None): - """Call either tabulate.tabulate(), or our internal alternateTabulate().""" - if HAVE_TABULATE: - tabulated = tabulate_impl(collection, headers=headers) - else: - tabulated = alternateTabulate(collection, headers=headers) - if tabulated: - print(tabulated) - - -def printLineSubsetWithHighlighting( - line, start, end, highlightStart=None, highlightEnd=None, maxLen=120, replacement=None): - """Print a (potential subset of a) line, with highlighting/underline and optional replacement. - - Will print at least the characters line[start:end], and potentially more if possible - to do so without making the output too wide. - Will highlight (underline) line[highlightStart:highlightEnd], where the default - value for highlightStart is simply start, and the default value for highlightEnd is simply end. - Replacment, if supplied, will be aligned with the highlighted range. - - Output is intended to look like part of a Clang compile error/warning message. - """ - # Fill in missing start/end with start/end of range. - if highlightStart is None: - highlightStart = start - if highlightEnd is None: - highlightEnd = end - - # Expand interested range start/end. - start = min(start, highlightStart) - end = max(end, highlightEnd) - - tildeLength = highlightEnd - highlightStart - 1 - caretLoc = highlightStart - continuation = '[...]' - - if len(line) > maxLen: - # Too long - - # the max is to handle -1 from .find() (which indicates "not found") - followingSpaceIndex = max(end, line.find(' ', min(len(line), end + 1))) - - # Maximum length has decreased by at least - # the length of a single continuation we absolutely need. - maxLen -= len(continuation) - - if followingSpaceIndex <= maxLen: - # We can grab the whole beginning of the line, - # and not adjust caretLoc - line = line[:maxLen] + continuation - - elif (len(line) - followingSpaceIndex) < 5: - # We need to truncate the beginning, - # but we're close to the end of line. - newBeginning = len(line) - maxLen - - caretLoc += len(continuation) - caretLoc -= newBeginning - line = continuation + line[newBeginning:] - else: - # Need to truncate the beginning of the string too. - newEnd = followingSpaceIndex - - # Now we need two continuations - # (and to adjust caret to the right accordingly) - maxLen -= len(continuation) - caretLoc += len(continuation) - - newBeginning = newEnd - maxLen - caretLoc -= newBeginning - - line = continuation + line[newBeginning:newEnd] + continuation - - stdout.buffer.write(line.encode('utf-8')) - print() - - spaces = ' ' * caretLoc - tildes = '~' * tildeLength - print(spaces + colored('^' + tildes, 'green')) - if replacement is not None: - print(spaces + colored(replacement, 'green')) - - -class ConsolePrinter(BasePrinter): - """Implementation of BasePrinter for generating diagnostic reports in colored, helpful console output.""" - - def __init__(self): - self.show_script_location = False - super().__init__() - - ### - # Output methods: these all print directly. - def outputResults(self, checker, broken_links=True, - missing_includes=False): - """Output the full results of a checker run. - - Includes the diagnostics, broken links (if desired), - and missing includes (if desired). - """ - self.output(checker) - if broken_links: - broken = checker.getBrokenLinks() - if broken: - self.outputBrokenLinks(checker, broken) - if missing_includes: - missing = checker.getMissingUnreferencedApiIncludes() - if missing: - self.outputMissingIncludes(checker, missing) - - def outputBrokenLinks(self, checker, broken): - """Output a table of broken links. - - Called by self.outputBrokenAndMissing() if requested. - """ - print('Missing API includes that are referenced by a linking macro: these result in broken links in the spec!') - - def makeRowOfBroken(entity, uses): - fn = checker.findEntity(entity).filename - anchor = '[[{}]]'.format(entity) - locations = ', '.join((toNameAndLine(context, root_path=checker.root_path) - for context in uses)) - return (fn, anchor, locations) - printTabulated((makeRowOfBroken(entity, uses) - for entity, uses in sorted(broken.items())), - headers=['Include File', 'Anchor in lieu of include', 'Links to this entity']) - - def outputMissingIncludes(self, checker, missing): - """Output a table of missing includes. - - Called by self.outputBrokenAndMissing() if requested. - """ - missing = list(sorted(missing)) - if not missing: - # Exit if none - return - print( - 'Missing, but unreferenced, API includes/anchors - potentially not-documented entities:') - - def makeRowOfMissing(entity): - fn = checker.findEntity(entity).filename - anchor = '[[{}]]'.format(entity) - return (fn, anchor) - printTabulated((makeRowOfMissing(entity) for entity in missing), - headers=['Include File', 'Anchor in lieu of include']) - - def outputMessage(self, msg): - """Output a Message, with highlighted range and replacement, if appropriate.""" - highlightStart, highlightEnd = getHighlightedRange(msg.context) - - if '\n' in msg.context.filename: - # This is a multi-line string "filename". - # Extra blank line and delimiter line for readability: - print() - print('--------------------------------------------------------------------') - - fileAndLine = colored('{}:'.format( - self.formatBrief(msg.context)), attrs=['bold']) - - headingSize = len('{context}: {mtype}: '.format( - context=self.formatBrief(msg.context), - mtype=self.formatBrief(msg.message_type, False))) - indent = ' ' * headingSize - printedHeading = False - - lines = msg.message[:] - if msg.see_also: - lines.append('See also:') - lines.extend((' {}'.format(self.formatBrief(see)) - for see in msg.see_also)) - - if msg.fix: - lines.append('Note: Auto-fix available') - - for line in msg.message: - if not printedHeading: - scriptloc = '' - if msg.script_location and self.show_script_location: - scriptloc = ', ' + msg.script_location - print('{fileLine} {mtype} {msg} (-{arg}{loc})'.format( - fileLine=fileAndLine, mtype=msg.message_type.formattedWithColon(), - msg=colored(line, attrs=['bold']), arg=msg.message_id.enable_arg(), loc=scriptloc)) - printedHeading = True - else: - print(colored(indent + line, attrs=['bold'])) - - if len(msg.message) > 1: - # extra blank line after multiline message - print('') - - start, end = getInterestedRange(msg.context) - printLineSubsetWithHighlighting( - msg.context.line, - start, end, - highlightStart, highlightEnd, - replacement=msg.replacement) - - def outputFallback(self, obj): - """Output by calling print.""" - print(obj) - - ### - # Format methods: these all return a string. - def formatFilename(self, fn, _with_color=True): - """Format a local filename, as a relative path if possible.""" - return self.getRelativeFilename(fn) - - def formatMessageTypeBrief(self, message_type, with_color=True): - """Format a message type briefly, applying color if desired and possible. - - Delegates to the superclass if not formatting with color. - """ - if with_color: - return message_type.formattedWithColon() - return super(ConsolePrinter, self).formatMessageTypeBrief( - message_type, with_color) diff --git a/codegen/vulkan/scripts/spec_tools/data_structures.py b/codegen/vulkan/scripts/spec_tools/data_structures.py deleted file mode 100644 index f2808cf1..00000000 --- a/codegen/vulkan/scripts/spec_tools/data_structures.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright (c) 2019 Collabora, Ltd. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> -"""Provides general-purpose data structures.""" - - -class DictOfStringSets: - """A dictionary where the values are sets of strings. - - Has some convenience functions to allow easier maintenance via - the .add method.""" - - def __init__(self, d=None): - self.d = {} - if d: - for k, v in d.items(): - self.add(k, v) - - def __getitem__(self, k): - return self.d[k] - - def __contains__(self, k): - return k in self.d - - def get(self, k, default=None): - return self.d.get(k, default) - - def get_dict(self): - return self.d - - def items(self): - """Return an iterator like dict().items().""" - return self.d.items() - - def keys(self): - """Return an iterator over keys.""" - return self.d.keys() - - def values(self): - """Return an iterator over values.""" - return self.d.values() - - def add_key(self, k): - """Ensure the set for the given key exists.""" - if k not in self.d: - self.d[k] = set() - - def add(self, k, v): - self.add_key(k) - if isinstance(v, str): - v = (v, ) - if not isinstance(v, set): - v = set(v) - self.d[k].update(v) diff --git a/codegen/vulkan/scripts/spec_tools/entity_db.py b/codegen/vulkan/scripts/spec_tools/entity_db.py deleted file mode 100644 index 9a8dcfb1..00000000 --- a/codegen/vulkan/scripts/spec_tools/entity_db.py +++ /dev/null @@ -1,659 +0,0 @@ -"""Provides EntityDatabase, a class that keeps track of spec-defined entities and associated macros.""" - -# Copyright (c) 2018-2019 Collabora, Ltd. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> - -from abc import ABC, abstractmethod - -from .shared import (CATEGORIES_WITH_VALIDITY, EXTENSION_CATEGORY, - NON_EXISTENT_MACROS, EntityData) -from .util import getElemName - - -def _entityToDict(data): - return { - 'macro': data.macro, - 'filename': data.filename, - 'category': data.category, - 'directory': data.directory - } - - -class EntityDatabase(ABC): - """Parsed and processed information from the registry XML. - - Must be subclasses for each specific API. - """ - - ### - # Methods that must be implemented in subclasses. - ### - @abstractmethod - def makeRegistry(self): - """Return a Registry object that has already had loadFile() and parseTree() called. - - Called only once during construction. - """ - raise NotImplementedError - - @abstractmethod - def getNamePrefix(self): - """Return the (two-letter) prefix of all entity names for this API. - - Called only once during construction. - """ - raise NotImplementedError - - @abstractmethod - def getPlatformRequires(self): - """Return the 'requires' string associated with external/platform definitions. - - This is the string found in the requires attribute of the XML for entities that - are externally defined in a platform include file, like the question marks in: - - <type requires="???" name="int8_t"/> - - In Vulkan, this is 'vk_platform'. - - Called only once during construction. - """ - raise NotImplementedError - - ### - # Methods that it is optional to **override** - ### - def getSystemTypes(self): - """Return an enumerable of strings that name system types. - - System types use the macro `code`, and they do not generate API/validity includes. - - Called only once during construction. - """ - return [] - - def getGeneratedDirs(self): - """Return a sequence of strings that are the subdirectories of generates API includes. - - Called only once during construction. - """ - return ['basetypes', - 'defines', - 'enums', - 'flags', - 'funcpointers', - 'handles', - 'protos', - 'structs'] - - def populateMacros(self): - """Perform API-specific calls, if any, to self.addMacro() and self.addMacros(). - - It is recommended to implement/override this and call - self.addMacros(..., ..., [..., "flags"]), - since the base implementation, in _basicPopulateMacros(), - does not add any macros as pertaining to the category "flags". - - Called only once during construction. - """ - pass - - def populateEntities(self): - """Perform API-specific calls, if any, to self.addEntity().""" - pass - - def getEntitiesWithoutValidity(self): - """Return an enumerable of entity names that do not generate validity includes.""" - return [self.mixed_case_name_prefix + - x for x in ['BaseInStructure', 'BaseOutStructure']] - - def getExclusionSet(self): - """Return a set of "support=" attribute strings that should not be included in the database. - - Called only during construction.""" - return set(('disabled',)) - - ### - # Methods that it is optional to **extend** - ### - def handleType(self, name, info, requires): - """Add entities, if appropriate, for an item in registry.typedict. - - Called at construction for every name, info in registry.typedict.items() - not immediately skipped, - to perform the correct associated addEntity() call, if applicable. - The contents of the requires attribute, if any, is passed in requires. - - May be extended by API-specific code to handle some cases preferentially, - then calling the super implementation to handle the rest. - """ - if requires == self.platform_requires: - # Ah, no, don't skip this, it's just in the platform header file. - # TODO are these code or basetype? - self.addEntity(name, 'code', elem=info.elem, generates=False) - return - - protect = info.elem.get('protect') - if protect: - self.addEntity(protect, 'dlink', - category='configdefines', generates=False) - - alias = info.elem.get('alias') - if alias: - self.addAlias(name, alias) - - cat = info.elem.get('category') - if cat == 'struct': - self.addEntity(name, 'slink', elem=info.elem) - - elif cat == 'union': - # TODO: is this right? - self.addEntity(name, 'slink', elem=info.elem) - - elif cat == 'enum': - self.addEntity( - name, 'elink', elem=info.elem) - - elif cat == 'handle': - self.addEntity(name, 'slink', elem=info.elem, - category='handles') - - elif cat == 'bitmask': - self.addEntity( - name, 'tlink', elem=info.elem, category='flags') - - elif cat == 'basetype': - self.addEntity(name, 'basetype', - elem=info.elem) - - elif cat == 'define': - self.addEntity(name, 'dlink', elem=info.elem) - - elif cat == 'funcpointer': - self.addEntity(name, 'tlink', elem=info.elem) - - elif cat == 'include': - # skip - return - - elif cat is None: - self.addEntity(name, 'code', elem=info.elem, generates=False) - - else: - raise RuntimeError('unrecognized category {}'.format(cat)) - - def handleCommand(self, name, info): - """Add entities, if appropriate, for an item in registry.cmddict. - - Called at construction for every name, info in registry.cmddict.items(). - Calls self.addEntity() accordingly. - """ - self.addEntity(name, 'flink', elem=info.elem, - category='commands', directory='protos') - - def handleExtension(self, name, info): - """Add entities, if appropriate, for an item in registry.extdict. - - Called at construction for every name, info in registry.extdict.items(). - Calls self.addEntity() accordingly. - """ - if info.supported in self._supportExclusionSet: - # Don't populate with disabled extensions. - return - - # Only get the protect strings and name from extensions - - self.addEntity(name, None, category=EXTENSION_CATEGORY, - generates=False) - protect = info.elem.get('protect') - if protect: - self.addEntity(protect, 'dlink', - category='configdefines', generates=False) - - def handleEnumValue(self, name, info): - """Add entities, if appropriate, for an item in registry.enumdict. - - Called at construction for every name, info in registry.enumdict.items(). - Calls self.addEntity() accordingly. - """ - self.addEntity(name, 'ename', elem=info.elem, - category='enumvalues', generates=False) - - ### - # END of methods intended to be implemented, overridden, or extended in child classes! - ### - - ### - # Accessors - ### - def findMacroAndEntity(self, macro, entity): - """Look up EntityData by macro and entity pair. - - Does **not** resolve aliases.""" - return self._byMacroAndEntity.get((macro, entity)) - - def findEntity(self, entity): - """Look up EntityData by entity name (case-sensitive). - - If it fails, it will try resolving aliases. - """ - result = self._byEntity.get(entity) - if result: - return result - - alias_set = self._aliasSetsByEntity.get(entity) - if alias_set: - for alias in alias_set: - if alias in self._byEntity: - return self.findEntity(alias) - - assert(not "Alias without main entry!") - - return None - - def findEntityCaseInsensitive(self, entity): - """Look up EntityData by entity name (case-insensitive). - - Does **not** resolve aliases.""" - return self._byLowercaseEntity.get(entity.lower()) - - def getMemberElems(self, commandOrStruct): - """Given a command or struct name, retrieve the ETree elements for each member/param. - - Returns None if the entity is not found or doesn't have members/params. - """ - data = self.findEntity(commandOrStruct) - - if not data: - return None - if data.elem is None: - return None - if data.macro == 'slink': - tag = 'member' - else: - tag = 'param' - return data.elem.findall('.//{}'.format(tag)) - - def getMemberNames(self, commandOrStruct): - """Given a command or struct name, retrieve the names of each member/param. - - Returns an empty list if the entity is not found or doesn't have members/params. - """ - members = self.getMemberElems(commandOrStruct) - if not members: - return [] - ret = [] - for member in members: - name_tag = member.find('name') - if name_tag: - ret.append(name_tag.text) - return ret - - def getEntityJson(self): - """Dump the internal entity dictionary to JSON for debugging.""" - import json - d = {entity: _entityToDict(data) - for entity, data in self._byEntity.items()} - return json.dumps(d, sort_keys=True, indent=4) - - def entityHasValidity(self, entity): - """Estimate if we expect to see a validity include for an entity name. - - Returns None if the entity name is not known, - otherwise a boolean: True if a validity include is expected. - - Related to Generator.isStructAlwaysValid. - """ - data = self.findEntity(entity) - if not data: - return None - - if entity in self.entities_without_validity: - return False - - if data.category == 'protos': - # All protos have validity - return True - - if data.category not in CATEGORIES_WITH_VALIDITY: - return False - - # Handle structs here. - members = self.getMemberElems(entity) - if not members: - return None - for member in members: - member_name = getElemName(member) - member_type = member.find('type').text - member_category = member.get('category') - - if member_name in ('next', 'type'): - return True - - if member_type in ('void', 'char'): - return True - - if member.get('noautovalidity'): - # Not generating validity for this member, skip it - continue - - if member.get('len'): - # Array - return True - - typetail = member.find('type').tail - if typetail and '*' in typetail: - # Pointer - return True - - if member_category in ('handle', 'enum', 'bitmask'): - return True - - if member.get('category') in ('struct', 'union') \ - and self.entityHasValidity(member_type): - # struct or union member - recurse - return True - - # Got this far - no validity needed - return False - - def entityGenerates(self, entity_name): - """Return True if the named entity generates include file(s).""" - return entity_name in self._generating_entities - - @property - def generating_entities(self): - """Return a sequence of all generating entity names.""" - return self._generating_entities.keys() - - def shouldBeRecognized(self, macro, entity_name): - """Determine, based on the macro and the name provided, if we should expect to recognize the entity. - - True if it is linked. Specific APIs may also provide additional cases where it is True.""" - return self.isLinkedMacro(macro) - - def likelyRecognizedEntity(self, entity_name): - """Guess (based on name prefix alone) if an entity is likely to be recognized.""" - return entity_name.lower().startswith(self.name_prefix) - - def isLinkedMacro(self, macro): - """Identify if a macro is considered a "linked" macro.""" - return macro in self._linkedMacros - - def isValidMacro(self, macro): - """Identify if a macro is known and valid.""" - if macro not in self._categoriesByMacro: - return False - - return macro not in NON_EXISTENT_MACROS - - def getCategoriesForMacro(self, macro): - """Identify the categories associated with a (known, valid) macro.""" - if macro in self._categoriesByMacro: - return self._categoriesByMacro[macro] - return None - - def areAliases(self, first_entity_name, second_entity_name): - """Return true if the two entity names are equivalent (aliases of each other).""" - alias_set = self._aliasSetsByEntity.get(first_entity_name) - if not alias_set: - # If this assert fails, we have goofed in addAlias - assert(second_entity_name not in self._aliasSetsByEntity) - - return False - - return second_entity_name in alias_set - - @property - def macros(self): - """Return the collection of all known entity-related markup macros.""" - return self._categoriesByMacro.keys() - - ### - # Methods only used during initial setup/population of this data structure - ### - def addMacro(self, macro, categories, link=False): - """Add a single markup macro to the collection of categories by macro. - - Also adds the macro to the set of linked macros if link=True. - - If a macro has already been supplied to a call, later calls for that macro have no effect. - """ - if macro in self._categoriesByMacro: - return - self._categoriesByMacro[macro] = categories - if link: - self._linkedMacros.add(macro) - - def addMacros(self, letter, macroTypes, categories): - """Add markup macros associated with a leading letter to the collection of categories by macro. - - Also, those macros created using 'link' in macroTypes will also be added to the set of linked macros. - - Basically automates a number of calls to addMacro(). - """ - for macroType in macroTypes: - macro = letter + macroType - self.addMacro(macro, categories, link=(macroType == 'link')) - - def addAlias(self, entityName, aliasName): - """Record that entityName is an alias for aliasName.""" - # See if we already have something with this as the alias. - alias_set = self._aliasSetsByEntity.get(aliasName) - other_alias_set = self._aliasSetsByEntity.get(entityName) - if alias_set and other_alias_set: - # If this fails, we need to merge sets and update. - assert(alias_set is other_alias_set) - - if not alias_set: - # Try looking by the other name. - alias_set = other_alias_set - - if not alias_set: - # Nope, this is a new set. - alias_set = set() - self._aliasSets.append(alias_set) - - # Add both names to the set - alias_set.add(entityName) - alias_set.add(aliasName) - - # Associate the set with each name - self._aliasSetsByEntity[aliasName] = alias_set - self._aliasSetsByEntity[entityName] = alias_set - - def addEntity(self, entityName, macro, category=None, elem=None, - generates=None, directory=None, filename=None): - """Add an entity (command, structure type, enum, enum value, etc) in the database. - - If an entityName has already been supplied to a call, later calls for that entityName have no effect. - - Arguments: - entityName -- the name of the entity. - macro -- the macro (without the trailing colon) that should be used to refer to this entity. - - Optional keyword arguments: - category -- If not manually specified, looked up based on the macro. - elem -- The ETree element associated with the entity in the registry XML. - generates -- Indicates whether this entity generates api and validity include files. - Default depends on directory (or if not specified, category). - directory -- The directory that include files (under api/ and validity/) are generated in. - If not specified (and generates is True), the default is the same as the category, - which is almost always correct. - filename -- The relative filename (under api/ or validity/) where includes are generated for this. - This only matters if generates is True (default). If not specified and generates is True, - one will be generated based on directory and entityName. - """ - # Probably dealt with in handleType(), but just in case it wasn't. - if elem is not None: - alias = elem.get('alias') - if alias: - self.addAlias(entityName, alias) - - if entityName in self._byEntity: - # skip if already recorded. - return - - # Look up category based on the macro, if category isn't specified. - if category is None: - category = self._categoriesByMacro.get(macro)[0] - - if generates is None: - potential_dir = directory or category - generates = potential_dir in self._generated_dirs - - # If directory isn't specified and this entity generates, - # the directory is the same as the category. - if directory is None and generates: - directory = category - - # Don't generate a filename if this entity doesn't generate includes. - if filename is None and generates: - filename = '{}/{}.txt'.format(directory, entityName) - - data = EntityData( - entity=entityName, - macro=macro, - elem=elem, - filename=filename, - category=category, - directory=directory - ) - if entityName.lower() not in self._byLowercaseEntity: - self._byLowercaseEntity[entityName.lower()] = [] - - self._byEntity[entityName] = data - self._byLowercaseEntity[entityName.lower()].append(data) - self._byMacroAndEntity[(macro, entityName)] = data - if generates and filename is not None: - self._generating_entities[entityName] = data - - def __init__(self): - """Constructor: Do not extend or override. - - Changing the behavior of other parts of this logic should be done by - implementing, extending, or overriding (as documented): - - - Implement makeRegistry() - - Implement getNamePrefix() - - Implement getPlatformRequires() - - Override getSystemTypes() - - Override populateMacros() - - Override populateEntities() - - Extend handleType() - - Extend handleCommand() - - Extend handleExtension() - - Extend handleEnumValue() - """ - # Internal data that we don't want consumers of the class touching for fear of - # breaking invariants - self._byEntity = {} - self._byLowercaseEntity = {} - self._byMacroAndEntity = {} - self._categoriesByMacro = {} - self._linkedMacros = set() - self._aliasSetsByEntity = {} - self._aliasSets = [] - - self._registry = None - - # Retrieve from subclass, if overridden, then store locally. - self._supportExclusionSet = set(self.getExclusionSet()) - - # Entities that get a generated/api/category/entity.txt file. - self._generating_entities = {} - - # Name prefix members - self.name_prefix = self.getNamePrefix().lower() - self.mixed_case_name_prefix = self.name_prefix[:1].upper( - ) + self.name_prefix[1:] - # Regex string for the name prefix that is case-insensitive. - self.case_insensitive_name_prefix_pattern = ''.join( - ('[{}{}]'.format(c.upper(), c) for c in self.name_prefix)) - - self.platform_requires = self.getPlatformRequires() - - self._generated_dirs = set(self.getGeneratedDirs()) - - # Note: Default impl requires self.mixed_case_name_prefix - self.entities_without_validity = set(self.getEntitiesWithoutValidity()) - - # TODO: Where should flags actually go? Not mentioned in the style guide. - # TODO: What about flag wildcards? There are a few such uses... - - # Abstract method: subclass must implement to define macros for flags - self.populateMacros() - - # Now, do default macro population - self._basicPopulateMacros() - - # Abstract method: subclass must implement to add any "not from the registry" (and not system type) - # entities - self.populateEntities() - - # Now, do default entity population - self._basicPopulateEntities(self.registry) - - ### - # Methods only used internally during initial setup/population of this data structure - ### - @property - def registry(self): - """Return a Registry.""" - if not self._registry: - self._registry = self.makeRegistry() - return self._registry - - def _basicPopulateMacros(self): - """Contains calls to self.addMacro() and self.addMacros(). - - If you need to change any of these, do so in your override of populateMacros(), - which will be called first. - """ - self.addMacro('basetype', ['basetypes']) - self.addMacro('code', ['code']) - self.addMacros('f', ['link', 'name', 'text'], ['protos']) - self.addMacros('s', ['link', 'name', 'text'], ['structs', 'handles']) - self.addMacros('e', ['link', 'name', 'text'], ['enums']) - self.addMacros('p', ['name', 'text'], ['parameter', 'member']) - self.addMacros('t', ['link', 'name'], ['funcpointers']) - self.addMacros('d', ['link', 'name'], ['defines', 'configdefines']) - - for macro in NON_EXISTENT_MACROS: - # Still search for them - self.addMacro(macro, None) - - def _basicPopulateEntities(self, registry): - """Contains typical calls to self.addEntity(). - - If you need to change any of these, do so in your override of populateEntities(), - which will be called first. - """ - system_types = set(self.getSystemTypes()) - for t in system_types: - self.addEntity(t, 'code', generates=False) - - for name, info in registry.typedict.items(): - if name in system_types: - # We already added these. - continue - - requires = info.elem.get('requires') - - if requires and not requires.lower().startswith(self.name_prefix): - # This is an externally-defined type, will skip it. - continue - - # OK, we might actually add an entity here - self.handleType(name=name, info=info, requires=requires) - - for name, info in registry.enumdict.items(): - self.handleEnumValue(name, info) - - for name, info in registry.cmddict.items(): - self.handleCommand(name, info) - - for name, info in registry.extdict.items(): - self.handleExtension(name, info) diff --git a/codegen/vulkan/scripts/spec_tools/file_process.py b/codegen/vulkan/scripts/spec_tools/file_process.py deleted file mode 100644 index f0d4c608..00000000 --- a/codegen/vulkan/scripts/spec_tools/file_process.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright (c) 2018-2019 Collabora, Ltd. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> -"Utilities for processing files." - -from pathlib import Path - - -class LinewiseFileProcessor: - """A base class for code that processes an input file (or file handle) one line at a time.""" - - def __init__(self): - self._lines = [] - self._line_num = 0 - self._next_line = None - self._line = '' - self._filename = Path() - - @property - def filename(self): - """The Path object of the currently processed file""" - return self._filename - - @property - def relative_filename(self): - """The current file's Path relative to the current working directory""" - return self.filename.relative_to(Path('.').resolve()) - - @property - def line(self): - """The current line, including any trailing whitespace and the line ending.""" - return self._line - - @property - def line_number(self): - """Get 1-indexed line number.""" - return self._line_num - - @property - def line_rstripped(self): - """The current line without any trailing whitespace.""" - if self.line is None: - return None - return self.line.rstrip() - - @property - def trailing_whitespace(self): - """The trailing whitespace of the current line that gets removed when accessing rstrippedLine""" - non_whitespace_length = len(self.line_rstripped) - return self.line[non_whitespace_length:] - - @property - def next_line(self): - """Peek at the next line, if any.""" - return self._next_line - - @property - def next_line_rstripped(self): - """Peek at the next line, if any, without any trailing whitespace.""" - if self.next_line is None: - return None - return self.next_line.rstrip() - - def get_preceding_line(self, relative_index=-1): - """Retrieve the line at an line number at the given relative index, if one exists. Returns None if there is no line there.""" - if relative_index >= 0: - raise RuntimeError( - 'relativeIndex must be negative, to retrieve a preceding line.') - if relative_index + self.line_number <= 0: - # There is no line at this index - return None - return self._lines[self.line_number + relative_index - 1] - - def get_preceding_lines(self, num): - """Get *up to* the preceding num lines. Fewer may be returned if the requested number aren't available.""" - return self._lines[- (num + 1):-1] - - def process_line(self, line_num, line): - """Implement in your subclass to handle each new line.""" - raise NotImplementedError - - def _process_file_handle(self, file_handle): - # These are so we can process one line earlier than we're actually iterating thru. - processing_line_num = None - processing_line = None - - def do_process_line(): - self._line_num = processing_line_num - self._line = processing_line - if processing_line is not None: - self._lines.append(processing_line) - self.process_line(processing_line_num, processing_line) - - for line_num, line in enumerate(file_handle, 1): - self._next_line = line - do_process_line() - processing_line_num = line_num - processing_line = line - - # Finally process the left-over line - self._next_line = None - do_process_line() - - def process_file(self, filename, file_handle=None): - """Main entry point - call with a filename and optionally the file handle to read from.""" - if isinstance(filename, str): - filename = Path(filename).resolve() - - self._filename = filename - - if file_handle: - self._process_file_handle(file_handle) - else: - with self._filename.open('r', encoding='utf-8') as f: - self._process_file_handle(f) diff --git a/codegen/vulkan/scripts/spec_tools/html_printer.py b/codegen/vulkan/scripts/spec_tools/html_printer.py deleted file mode 100644 index eb1df406..00000000 --- a/codegen/vulkan/scripts/spec_tools/html_printer.py +++ /dev/null @@ -1,436 +0,0 @@ -"""Defines HTMLPrinter, a BasePrinter subclass for a single-page HTML results file.""" - -# Copyright (c) 2018-2019 Collabora, Ltd. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> - -import html -import re -from collections import namedtuple - -from .base_printer import BasePrinter, getColumn -from .shared import (MessageContext, MessageType, generateInclude, - getHighlightedRange) - -# Bootstrap styles (for constructing CSS class names) associated with MessageType values. -MESSAGE_TYPE_STYLES = { - MessageType.ERROR: 'danger', - MessageType.WARNING: 'warning', - MessageType.NOTE: 'secondary' -} - - -# HTML Entity for a little emoji-icon associated with MessageType values. -MESSAGE_TYPE_ICONS = { - MessageType.ERROR: '⊗', # makeIcon('times-circle'), - MessageType.WARNING: '⚠', # makeIcon('exclamation-triangle'), - MessageType.NOTE: 'ℹ' # makeIcon('info-circle') -} - -LINK_ICON = '🔗' # link icon - - -class HTMLPrinter(BasePrinter): - """Implementation of BasePrinter for generating diagnostic reports in HTML format. - - Generates a single file containing neatly-formatted messages. - - The HTML file loads Bootstrap 4 as well as 'prism' syntax highlighting from CDN. - """ - - def __init__(self, filename): - """Construct by opening the file.""" - self.f = open(filename, 'w', encoding='utf-8') - self.f.write("""<!doctype html> - <html lang="en"><head> - <meta charset="utf-8"> - <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> - <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.15.0/themes/prism.min.css" integrity="sha256-N1K43s+8twRa+tzzoF3V8EgssdDiZ6kd9r8Rfgg8kZU=" crossorigin="anonymous" /> - <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.15.0/plugins/line-numbers/prism-line-numbers.min.css" integrity="sha256-Afz2ZJtXw+OuaPX10lZHY7fN1+FuTE/KdCs+j7WZTGc=" crossorigin="anonymous" /> - <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.15.0/plugins/line-highlight/prism-line-highlight.min.css" integrity="sha256-FFGTaA49ZxFi2oUiWjxtTBqoda+t1Uw8GffYkdt9aco=" crossorigin="anonymous" /> - <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous"> - <style> - pre { - overflow-x: scroll; - white-space: nowrap; - } - </style> - <title>check_spec_links results</title> - </head> - <body> - <div class="container"> - <h1><code>check_spec_links.py</code> Scan Results</h1> - """) - # - self.filenameTransformer = re.compile(r'[^\w]+') - self.fileRange = {} - self.fileLines = {} - self.backLink = namedtuple( - 'BackLink', ['lineNum', 'col', 'end_col', 'target', 'tooltip', 'message_type']) - self.fileBackLinks = {} - - self.nextAnchor = 0 - super().__init__() - - def close(self): - """Write the tail end of the file and close it.""" - self.f.write(""" - </div> - <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.15.0/prism.min.js" integrity="sha256-jc6y1s/Y+F+78EgCT/lI2lyU7ys+PFYrRSJ6q8/R8+o=" crossorigin="anonymous"></script> - <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.15.0/plugins/keep-markup/prism-keep-markup.min.js" integrity="sha256-mP5i3m+wTxxOYkH+zXnKIG5oJhXLIPQYoiicCV1LpkM=" crossorigin="anonymous"></script> - <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.15.0/components/prism-asciidoc.min.js" integrity="sha256-NHPE1p3VBIdXkmfbkf/S0hMA6b4Ar4TAAUlR+Rlogoc=" crossorigin="anonymous"></script> - <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.15.0/plugins/line-numbers/prism-line-numbers.min.js" integrity="sha256-JfF9MVfGdRUxzT4pecjOZq6B+F5EylLQLwcQNg+6+Qk=" crossorigin="anonymous"></script> - <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.15.0/plugins/line-highlight/prism-line-highlight.min.js" integrity="sha256-DEl9ZQE+lseY13oqm2+mlUr+sVI18LG813P+kzzIm8o=" crossorigin="anonymous"></script> - <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.slim.min.js" integrity="sha256-3edrmyuQ0w65f8gfBsqowzjJe2iM6n0nKciPUp8y+7E=" crossorigin="anonymous"></script> - <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.6/esm/popper.min.js" integrity="sha256-T0gPN+ySsI9ixTd/1ciLl2gjdLJLfECKvkQjJn98lOs=" crossorigin="anonymous"></script> - <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script> - <script> - $(function () { - $('[data-toggle="tooltip"]').tooltip(); - function autoExpand() { - var hash = window.location.hash; - if (!hash) { - return; - } - $(hash).parents().filter('.collapse').collapse('show'); - } - window.addEventListener('hashchange', autoExpand); - $(document).ready(autoExpand); - $('.accordion').on('shown.bs.collapse', function(e) { - e.target.parentNode.scrollIntoView(); - }) - }) - </script> - </body></html> - """) - self.f.close() - - ### - # Output methods: these all write to the HTML file. - def outputResults(self, checker, broken_links=True, - missing_includes=False): - """Output the full results of a checker run. - - Includes the diagnostics, broken links (if desired), - missing includes (if desired), and excerpts of all files with diagnostics. - """ - self.output(checker) - self.outputBrokenAndMissing( - checker, broken_links=broken_links, missing_includes=missing_includes) - - self.f.write(""" - <div class="container"> - <h2>Excerpts of referenced files</h2>""") - for fn in self.fileRange: - self.outputFileExcerpt(fn) - self.f.write('</div><!-- .container -->\n') - - def outputChecker(self, checker): - """Output the contents of a MacroChecker object. - - Starts and ends the accordion populated by outputCheckerFile(). - """ - self.f.write( - '<div class="container"><h2>Per-File Warnings and Errors</h2>\n') - self.f.write('<div class="accordion" id="fileAccordion">\n') - super(HTMLPrinter, self).outputChecker(checker) - self.f.write("""</div><!-- #fileAccordion --> - </div><!-- .container -->\n""") - - def outputCheckerFile(self, fileChecker): - """Output the contents of a MacroCheckerFile object. - - Stashes the lines of the file for later excerpts, - and outputs any diagnostics in an accordion card. - """ - # Save lines for later - self.fileLines[fileChecker.filename] = fileChecker.lines - - if not fileChecker.numDiagnostics(): - return - - self.f.write(""" - <div class="card"> - <div class="card-header" id="{id}-file-heading"> - <div class="row"> - <div class="col"> - <button data-target="#collapse-{id}" class="btn btn-link btn-primary mb-0 collapsed" type="button" data-toggle="collapse" aria-expanded="false" aria-controls="collapse-{id}"> - {relativefn} - </button> - </div> - """.format(id=self.makeIdentifierFromFilename(fileChecker.filename), relativefn=html.escape(self.getRelativeFilename(fileChecker.filename)))) - self.f.write('<div class="col-1">') - warnings = fileChecker.numMessagesOfType(MessageType.WARNING) - if warnings > 0: - self.f.write("""<span class="badge badge-warning" data-toggle="tooltip" title="{num} warnings in this file"> - {icon} - {num}<span class="sr-only"> warnings</span></span>""".format(num=warnings, icon=MESSAGE_TYPE_ICONS[MessageType.WARNING])) - self.f.write('</div>\n<div class="col-1">') - errors = fileChecker.numMessagesOfType(MessageType.ERROR) - if errors > 0: - self.f.write("""<span class="badge badge-danger" data-toggle="tooltip" title="{num} errors in this file"> - {icon} - {num}<span class="sr-only"> errors</span></span>""".format(num=errors, icon=MESSAGE_TYPE_ICONS[MessageType.ERROR])) - self.f.write(""" - </div><!-- .col-1 --> - </div><!-- .row --> - </div><!-- .card-header --> - <div id="collapse-{id}" class="collapse" aria-labelledby="{id}-file-heading" data-parent="#fileAccordion"> - <div class="card-body"> - """.format(id=self.makeIdentifierFromFilename(fileChecker.filename))) - super(HTMLPrinter, self).outputCheckerFile(fileChecker) - - self.f.write(""" - </div><!-- .card-body --> - </div><!-- .collapse --> - </div><!-- .card --> - <!-- ..................................... --> - """.format(id=self.makeIdentifierFromFilename(fileChecker.filename))) - - def outputMessage(self, msg): - """Output a Message.""" - anchor = self.getUniqueAnchor() - - self.recordUsage(msg.context, - linkBackTarget=anchor, - linkBackTooltip='{}: {} [...]'.format( - msg.message_type, msg.message[0]), - linkBackType=msg.message_type) - - self.f.write(""" - <div class="card"> - <div class="card-body"> - <h5 class="card-header bg bg-{style}" id="{anchor}">{icon} {t} Line {lineNum}, Column {col} (-{arg})</h5> - <p class="card-text"> - """.format( - anchor=anchor, - icon=MESSAGE_TYPE_ICONS[msg.message_type], - style=MESSAGE_TYPE_STYLES[msg.message_type], - t=self.formatBrief(msg.message_type), - lineNum=msg.context.lineNum, - col=getColumn(msg.context), - arg=msg.message_id.enable_arg())) - self.f.write(self.formatContext(msg.context)) - self.f.write('<br/>') - for line in msg.message: - self.f.write(html.escape(line)) - self.f.write('<br />\n') - self.f.write('</p>\n') - if msg.see_also: - self.f.write('<p>See also:</p><ul>\n') - for see in msg.see_also: - if isinstance(see, MessageContext): - self.f.write( - '<li>{}</li>\n'.format(self.formatContext(see))) - self.recordUsage(see, - linkBackTarget=anchor, - linkBackType=MessageType.NOTE, - linkBackTooltip='see-also associated with {} at {}'.format(msg.message_type, self.formatContextBrief(see))) - else: - self.f.write('<li>{}</li>\n'.format(self.formatBrief(see))) - self.f.write('</ul>') - if msg.replacement is not None: - self.f.write( - '<div class="alert alert-primary">Hover the highlight text to view suggested replacement.</div>') - if msg.fix is not None: - self.f.write( - '<div class="alert alert-info">Note: Auto-fix available.</div>') - if msg.script_location: - self.f.write( - '<p>Message originated at <code>{}</code></p>'.format(msg.script_location)) - self.f.write('<pre class="line-numbers language-asciidoc" data-start="{}"><code>'.format( - msg.context.lineNum)) - highlightStart, highlightEnd = getHighlightedRange(msg.context) - self.f.write(html.escape(msg.context.line[:highlightStart])) - self.f.write( - '<span class="border border-{}"'.format(MESSAGE_TYPE_STYLES[msg.message_type])) - if msg.replacement is not None: - self.f.write( - ' data-toggle="tooltip" title="{}"'.format(msg.replacement)) - self.f.write('>') - self.f.write(html.escape( - msg.context.line[highlightStart:highlightEnd])) - self.f.write('</span>') - self.f.write(html.escape(msg.context.line[highlightEnd:])) - self.f.write('</code></pre></div></div>') - - def outputBrokenLinks(self, checker, broken): - """Output a table of broken links. - - Called by self.outputBrokenAndMissing() if requested. - """ - self.f.write(""" - <div class="container"> - <h2>Missing Referenced API Includes</h2> - <p>Items here have been referenced by a linking macro, so these are all broken links in the spec!</p> - <table class="table table-striped"> - <thead> - <th scope="col">Add line to include this file</th> - <th scope="col">or add this macro instead</th> - <th scope="col">Links to this entity</th></thead> - """) - - for entity_name, uses in sorted(broken.items()): - category = checker.findEntity(entity_name).category - anchor = self.getUniqueAnchor() - asciidocAnchor = '[[{}]]'.format(entity_name) - include = generateInclude(dir_traverse='../../generated/', - generated_type='api', - category=category, - entity=entity_name) - self.f.write(""" - <tr id={}> - <td><code class="text-dark language-asciidoc">{}</code></td> - <td><code class="text-dark">{}</code></td> - <td><ul class="list-inline"> - """.format(anchor, include, asciidocAnchor)) - for context in uses: - self.f.write( - '<li class="list-inline-item">{}</li>'.format(self.formatContext(context, MessageType.NOTE))) - self.recordUsage( - context, - linkBackTooltip='Link broken in spec: {} not seen'.format( - include), - linkBackTarget=anchor, - linkBackType=MessageType.NOTE) - self.f.write("""</ul></td></tr>""") - self.f.write("""</table></div>""") - - def outputMissingIncludes(self, checker, missing): - """Output a table of missing includes. - - Called by self.outputBrokenAndMissing() if requested. - """ - self.f.write(""" - <div class="container"> - <h2>Missing Unreferenced API Includes</h2> - <p>These items are expected to be generated in the spec build process, but aren't included. - However, as they also are not referenced by any linking macros, they aren't broken links - at worst they are undocumented entities, - at best they are errors in <code>check_spec_links.py</code> logic computing which entities get generated files.</p> - <table class="table table-striped"> - <thead> - <th scope="col">Add line to include this file</th> - <th scope="col">or add this macro instead</th> - """) - - for entity in sorted(missing): - fn = checker.findEntity(entity).filename - anchor = '[[{}]]'.format(entity) - self.f.write(""" - <tr> - <td><code class="text-dark">{filename}</code></td> - <td><code class="text-dark">{anchor}</code></td> - """.format(filename=fn, anchor=anchor)) - self.f.write("""</table></div>""") - - def outputFileExcerpt(self, filename): - """Output a card containing an excerpt of a file, sufficient to show locations of all diagnostics plus some context. - - Called by self.outputResults(). - """ - self.f.write("""<div class="card"> - <div class="card-header" id="heading-{id}"><h5 class="mb-0"> - <button class="btn btn-link" type="button"> - {fn} - </button></h5></div><!-- #heading-{id} --> - <div class="card-body"> - """.format(id=self.makeIdentifierFromFilename(filename), fn=self.getRelativeFilename(filename))) - lines = self.fileLines[filename] - r = self.fileRange[filename] - self.f.write("""<pre class="line-numbers language-asciidoc line-highlight" id="excerpt-{id}" data-start="{start}"><code>""".format( - id=self.makeIdentifierFromFilename(filename), - start=r.start)) - for lineNum, line in enumerate( - lines[(r.start - 1):(r.stop - 1)], r.start): - # self.f.write(line) - lineLinks = [x for x in self.fileBackLinks[filename] - if x.lineNum == lineNum] - for col, char in enumerate(line): - colLinks = (x for x in lineLinks if x.col == col) - for link in colLinks: - # TODO right now the syntax highlighting is interfering with the link! so the link-generation is commented out, - # only generating the emoji icon. - - # self.f.write('<a href="#{target}" title="{title}" data-toggle="tooltip" data-container="body">{icon}'.format( - # target=link.target, title=html.escape(link.tooltip), - # icon=MESSAGE_TYPE_ICONS[link.message_type])) - self.f.write(MESSAGE_TYPE_ICONS[link.message_type]) - self.f.write('<span class="sr-only">Cross reference: {t} {title}</span>'.format( - title=html.escape(link.tooltip, False), t=link.message_type)) - - # self.f.write('</a>') - - # Write the actual character - self.f.write(html.escape(char)) - self.f.write('\n') - - self.f.write('</code></pre>') - self.f.write('</div><!-- .card-body -->\n') - self.f.write('</div><!-- .card -->\n') - - def outputFallback(self, obj): - """Output some text in a general way.""" - self.f.write(obj) - - ### - # Format method: return a string. - def formatContext(self, context, message_type=None): - """Format a message context in a verbose way.""" - if message_type is None: - icon = LINK_ICON - else: - icon = MESSAGE_TYPE_ICONS[message_type] - return 'In context: <a href="{href}">{icon}{relative}:{lineNum}:{col}</a>'.format( - href=self.getAnchorLinkForContext(context), - icon=icon, - # id=self.makeIdentifierFromFilename(context.filename), - relative=self.getRelativeFilename(context.filename), - lineNum=context.lineNum, - col=getColumn(context)) - - ### - # Internal methods: not mandated by parent class. - def recordUsage(self, context, linkBackTooltip=None, - linkBackTarget=None, linkBackType=MessageType.NOTE): - """Internally record a 'usage' of something. - - Increases the range of lines that are included in the excerpts, - and records back-links if appropriate. - """ - BEFORE_CONTEXT = 6 - AFTER_CONTEXT = 3 - # Clamp because we need accurate start line number to make line number - # display right - start = max(1, context.lineNum - BEFORE_CONTEXT) - stop = context.lineNum + AFTER_CONTEXT + 1 - if context.filename not in self.fileRange: - self.fileRange[context.filename] = range(start, stop) - self.fileBackLinks[context.filename] = [] - else: - oldRange = self.fileRange[context.filename] - self.fileRange[context.filename] = range( - min(start, oldRange.start), max(stop, oldRange.stop)) - - if linkBackTarget is not None: - start_col, end_col = getHighlightedRange(context) - self.fileBackLinks[context.filename].append(self.backLink( - lineNum=context.lineNum, col=start_col, end_col=end_col, - target=linkBackTarget, tooltip=linkBackTooltip, - message_type=linkBackType)) - - def makeIdentifierFromFilename(self, fn): - """Compute an acceptable HTML anchor name from a filename.""" - return self.filenameTransformer.sub('_', self.getRelativeFilename(fn)) - - def getAnchorLinkForContext(self, context): - """Compute the anchor link to the excerpt for a MessageContext.""" - return '#excerpt-{}.{}'.format( - self.makeIdentifierFromFilename(context.filename), context.lineNum) - - def getUniqueAnchor(self): - """Create and return a new unique string usable as a link anchor.""" - anchor = 'anchor-{}'.format(self.nextAnchor) - self.nextAnchor += 1 - return anchor diff --git a/codegen/vulkan/scripts/spec_tools/macro_checker.py b/codegen/vulkan/scripts/spec_tools/macro_checker.py deleted file mode 100644 index a8a75aa8..00000000 --- a/codegen/vulkan/scripts/spec_tools/macro_checker.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Provides the MacroChecker class.""" - -# Copyright (c) 2018-2019 Collabora, Ltd. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> - -from io import StringIO -import re - - -class MacroChecker(object): - """Perform and track checking of one or more files in an API spec. - - This does not necessarily need to be subclassed per-API: it is sufficiently - parameterized in the constructor for expected usage. - """ - - def __init__(self, enabled_messages, entity_db, - macro_checker_file_type, root_path): - """Construct an object that tracks checking one or more files in an API spec. - - enabled_messages -- a set of MessageId that should be enabled. - entity_db -- an object of a EntityDatabase subclass for this API. - macro_checker_file_type -- Type to instantiate to create the right - MacroCheckerFile subclass for this API. - root_path -- A Path object for the root of this repository. - """ - self.enabled_messages = enabled_messages - self.entity_db = entity_db - self.macro_checker_file_type = macro_checker_file_type - self.root_path = root_path - - self.files = [] - - self.refpages = set() - - # keys: entity names. values: MessageContext - self.links = {} - self.apiIncludes = {} - self.validityIncludes = {} - self.headings = {} - - # Regexes that are members because they depend on the name prefix. - - # apiPrefix, followed by some word characters or * as many times as desired, - # NOT followed by >> and NOT preceded by one of the characters in that first character class. - # (which distinguish "names being used somewhere other than prose"). - self.suspected_missing_macro_re = re.compile( - r'\b(?<![-=:/[\.`+,])(?P<entity_name>{}[\w*]+)\b(?!>>)'.format( - self.entity_db.case_insensitive_name_prefix_pattern) - ) - self.heading_command_re = re.compile( - r'=+ (?P<command>{}[\w]+)'.format(self.entity_db.name_prefix) - ) - - macros_pattern = '|'.join((re.escape(macro) - for macro in self.entity_db.macros)) - # the "formatting" group is to strip matching */**/_/__ - # surrounding an entire macro. - self.macro_re = re.compile( - r'(?P<formatting>\**|_*)(?P<macro>{}):(?P<entity_name>[\w*]+((?P<subscript>[\[][^\]]*[\]]))?)(?P=formatting)'.format(macros_pattern)) - - def haveLinkTarget(self, entity): - """Report if we have parsed an API include (or heading) for an entity. - - None if there is no entity with that name. - """ - if not self.findEntity(entity): - return None - if entity in self.apiIncludes: - return True - return entity in self.headings - - def hasFixes(self): - """Report if any files have auto-fixes.""" - for f in self.files: - if f.hasFixes(): - return True - return False - - def addLinkToEntity(self, entity, context): - """Record seeing a link to an entity's docs from a context.""" - if entity not in self.links: - self.links[entity] = [] - self.links[entity].append(context) - - def seenRefPage(self, entity): - """Check if a ref-page markup block has been seen for an entity.""" - return entity in self.refpages - - def addRefPage(self, entity): - """Record seeing a ref-page markup block for an entity.""" - self.refpages.add(entity) - - def findMacroAndEntity(self, macro, entity): - """Look up EntityData by macro and entity pair. - - Forwards to the EntityDatabase. - """ - return self.entity_db.findMacroAndEntity(macro, entity) - - def findEntity(self, entity): - """Look up EntityData by entity name (case-sensitive). - - Forwards to the EntityDatabase. - """ - return self.entity_db.findEntity(entity) - - def findEntityCaseInsensitive(self, entity): - """Look up EntityData by entity name (case-insensitive). - - Forwards to the EntityDatabase. - """ - return self.entity_db.findEntityCaseInsensitive(entity) - - def getMemberNames(self, commandOrStruct): - """Given a command or struct name, retrieve the names of each member/param. - - Returns an empty list if the entity is not found or doesn't have members/params. - - Forwards to the EntityDatabase. - """ - return self.entity_db.getMemberNames(commandOrStruct) - - def likelyRecognizedEntity(self, entity_name): - """Guess (based on name prefix alone) if an entity is likely to be recognized. - - Forwards to the EntityDatabase. - """ - return self.entity_db.likelyRecognizedEntity(entity_name) - - def isLinkedMacro(self, macro): - """Identify if a macro is considered a "linked" macro. - - Forwards to the EntityDatabase. - """ - return self.entity_db.isLinkedMacro(macro) - - def processFile(self, filename): - """Parse an .adoc file belonging to the spec and check it for errors.""" - class FileStreamMaker(object): - def __init__(self, filename): - self.filename = filename - - def make_stream(self): - return open(self.filename, 'r', encoding='utf-8') - - f = self.macro_checker_file_type(self, filename, self.enabled_messages, - FileStreamMaker(filename)) - f.process() - self.files.append(f) - - def processString(self, s): - """Process a string as if it were a spec file. - - Used for testing purposes. - """ - if "\n" in s.rstrip(): - # remove leading spaces from each line to allow easier - # block-quoting in tests - s = "\n".join((line.lstrip() for line in s.split("\n"))) - # fabricate a "filename" that will display better. - filename = "string{}\n****START OF STRING****\n{}\n****END OF STRING****\n".format( - len(self.files), s.rstrip()) - - else: - filename = "string{}: {}".format( - len(self.files), s.rstrip()) - - class StringStreamMaker(object): - def __init__(self, string): - self.string = string - - def make_stream(self): - return StringIO(self.string) - - f = self.macro_checker_file_type(self, filename, self.enabled_messages, - StringStreamMaker(s)) - f.process() - self.files.append(f) - return f - - def numDiagnostics(self): - """Return the total number of diagnostics (warnings and errors) over all the files processed.""" - return sum((f.numDiagnostics() for f in self.files)) - - def numErrors(self): - """Return the total number of errors over all the files processed.""" - return sum((f.numErrors() for f in self.files)) - - def getMissingUnreferencedApiIncludes(self): - """Return the unreferenced entity names that we expected to see an API include or link target for, but did not. - - Counterpart to getBrokenLinks(): This method returns the entity names - that were not used in a linking macro (and thus wouldn't create a broken link), - but were nevertheless expected and not seen. - """ - return (entity for entity in self.entity_db.generating_entities - if (not self.haveLinkTarget(entity)) and entity not in self.links) - - def getBrokenLinks(self): - """Return the entity names and usage contexts that we expected to see an API include or link target for, but did not. - - Counterpart to getMissingUnreferencedApiIncludes(): This method returns only the - entity names that were used in a linking macro (and thus create a broken link), - but were not seen. The values of the dictionary are a list of MessageContext objects - for each linking macro usage for this entity name. - """ - return {entity: contexts for entity, contexts in self.links.items() - if self.entity_db.entityGenerates(entity) and not self.haveLinkTarget(entity)} - - def getMissingRefPages(self): - """Return a list of entities that we expected, but did not see, a ref page block for. - - The heuristics here are rather crude: we expect a ref page for every generating entry. - """ - return (entity for entity in sorted(self.entity_db.generating_entities) - if entity not in self.refpages) diff --git a/codegen/vulkan/scripts/spec_tools/macro_checker_file.py b/codegen/vulkan/scripts/spec_tools/macro_checker_file.py deleted file mode 100644 index 3bca17a5..00000000 --- a/codegen/vulkan/scripts/spec_tools/macro_checker_file.py +++ /dev/null @@ -1,1592 +0,0 @@ -"""Provides MacroCheckerFile, a subclassable type that validates a single file in the spec.""" - -# Copyright (c) 2018-2019 Collabora, Ltd. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> - -import logging -import re -from collections import OrderedDict, namedtuple -from enum import Enum -from inspect import currentframe - -from .shared import (AUTO_FIX_STRING, CATEGORIES_WITH_VALIDITY, - EXTENSION_CATEGORY, NON_EXISTENT_MACROS, EntityData, - Message, MessageContext, MessageId, MessageType, - generateInclude, toNameAndLine) - -# Code blocks may start and end with any number of ---- -CODE_BLOCK_DELIM = '----' - -# Mostly for ref page blocks, but also used elsewhere? -REF_PAGE_LIKE_BLOCK_DELIM = '--' - -# For insets/blocks like the implicit valid usage -# TODO think it must start with this - does it have to be exactly this? -BOX_BLOCK_DELIM = '****' - - -INTERNAL_PLACEHOLDER = re.compile( - r'(?P<delim>__+)([a-zA-Z]+)(?P=delim)' -) - -# Matches a generated (api or validity) include line. -INCLUDE = re.compile( - r'include::(?P<directory_traverse>((../){1,4}|\{(INCS-VAR|generated)\}/)(generated/)?)(?P<generated_type>[\w]+)/(?P<category>\w+)/(?P<entity_name>[^./]+).txt[\[][\]]') - -# Matches an [[AnchorLikeThis]] -ANCHOR = re.compile(r'\[\[(?P<entity_name>[^\]]+)\]\]') - -# Looks for flink:foo:: or slink::foo:: at the end of string: -# used to detect explicit pname context. -PRECEDING_MEMBER_REFERENCE = re.compile( - r'\b(?P<macro>[fs](text|link)):(?P<entity_name>[\w*]+)::$') - -# Matches something like slink:foo::pname:bar as well as -# the under-marked-up slink:foo::bar. -MEMBER_REFERENCE = re.compile( - r'\b(?P<first_part>(?P<scope_macro>[fs](text|link)):(?P<scope>[\w*]+))(?P<double_colons>::)(?P<second_part>(?P<member_macro>pname:?)(?P<entity_name>[\w]+))\b' -) - -# Matches if a string ends while a link is still "open". -# (first half of a link being broken across two lines, -# or containing our interested area when matched against the text preceding). -# Used to skip checking in some places. -OPEN_LINK = re.compile( - r'.*(?<!`)<<[^>]*$' -) - -# Matches if a string begins and is followed by a link "close" without a matching open. -# (second half of a link being broken across two lines) -# Used to skip checking in some places. -CLOSE_LINK = re.compile( - r'[^<]*>>.*$' -) - -# Matches if a line should be skipped without further considering. -# Matches lines starting with: -# - `ifdef:` -# - `endif:` -# - `todo` (followed by something matching \b, like : or (. capitalization ignored) -SKIP_LINE = re.compile( - r'^(ifdef:)|(endif:)|([tT][oO][dD][oO]\b).*' -) - -# Matches the whole inside of a refpage tag. -BRACKETS = re.compile(r'\[(?P<tags>.*)\]') - -# Matches a key='value' pair from a ref page tag. -REF_PAGE_ATTRIB = re.compile( - r"(?P<key>[a-z]+)='(?P<value>[^'\\]*(?:\\.[^'\\]*)*)'") - - -class Attrib(Enum): - """Attributes of a ref page.""" - - REFPAGE = 'refpage' - DESC = 'desc' - TYPE = 'type' - ALIAS = 'alias' - XREFS = 'xrefs' - ANCHOR = 'anchor' - - -VALID_REF_PAGE_ATTRIBS = set( - (e.value for e in Attrib)) - -AttribData = namedtuple('AttribData', ['match', 'key', 'value']) - - -def makeAttribFromMatch(match): - """Turn a match of REF_PAGE_ATTRIB into an AttribData value.""" - return AttribData(match=match, key=match.group( - 'key'), value=match.group('value')) - - -def parseRefPageAttribs(line): - """Parse a ref page tag into a dictionary of attribute_name: AttribData.""" - return {m.group('key'): makeAttribFromMatch(m) - for m in REF_PAGE_ATTRIB.finditer(line)} - - -def regenerateIncludeFromMatch(match, generated_type): - """Create an include directive from an INCLUDE match and a (new or replacement) generated_type.""" - return generateInclude( - match.group('directory_traverse'), - generated_type, - match.group('category'), - match.group('entity_name')) - - -BlockEntry = namedtuple( - 'BlockEntry', ['delimiter', 'context', 'block_type', 'refpage']) - - -class BlockType(Enum): - """Enumeration of the various distinct block types known.""" - CODE = 'code' - REF_PAGE_LIKE = 'ref-page-like' # with or without a ref page tag before - BOX = 'box' - - @classmethod - def lineToBlockType(self, line): - """Return a BlockType if the given line is a block delimiter. - - Returns None otherwise. - """ - if line == REF_PAGE_LIKE_BLOCK_DELIM: - return BlockType.REF_PAGE_LIKE - if line.startswith(CODE_BLOCK_DELIM): - return BlockType.CODE - if line.startswith(BOX_BLOCK_DELIM): - return BlockType.BOX - - return None - - -def _pluralize(word, num): - if num == 1: - return word - if word.endswith('y'): - return word[:-1] + 'ies' - return word + 's' - - -def _s_suffix(num): - """Simplify pluralization.""" - if num > 1: - return 's' - return '' - - -def shouldEntityBeText(entity, subscript): - """Determine if an entity name appears to use placeholders, wildcards, etc. and thus merits use of a *text macro. - - Call with the entity and subscript groups from a match of MacroChecker.macro_re. - """ - entity_only = entity - if subscript: - if subscript == '[]' or subscript == '[i]' or subscript.startswith( - '[_') or subscript.endswith('_]'): - return True - entity_only = entity[:-len(subscript)] - - if ('*' in entity) or entity.startswith('_') or entity_only.endswith('_'): - return True - - if INTERNAL_PLACEHOLDER.search(entity): - return True - return False - - -class MacroCheckerFile(object): - """Object performing processing of a single AsciiDoctor file from a specification. - - For testing purposes, may also process a string as if it were a file. - """ - - def __init__(self, checker, filename, enabled_messages, stream_maker): - """Construct a MacroCheckerFile object. - - Typically called by MacroChecker.processFile or MacroChecker.processString(). - - Arguments: - checker -- A MacroChecker object. - filename -- A string to use in messages to refer to this checker, typically the file name. - enabled_messages -- A set() of MessageId values that should be considered "enabled" and thus stored. - stream_maker -- An object with a makeStream() method that returns a stream. - """ - self.checker = checker - self.filename = filename - self.stream_maker = stream_maker - self.enabled_messages = enabled_messages - self.missing_validity_suppressions = set( - self.getMissingValiditySuppressions()) - - self.logger = logging.getLogger(__name__) - self.logger.addHandler(logging.NullHandler()) - - self.fixes = set() - self.messages = [] - - self.pname_data = None - self.pname_mentions = {} - - self.refpage_includes = {} - - self.lines = [] - - # For both of these: - # keys: entity name - # values: MessageContext - self.fs_api_includes = {} - self.validity_includes = {} - - self.in_code_block = False - self.in_ref_page = False - self.prev_line_ref_page_tag = None - self.current_ref_page = None - - # Stack of block-starting delimiters. - self.block_stack = [] - - # Regexes that are members because they depend on the name prefix. - self.suspected_missing_macro_re = self.checker.suspected_missing_macro_re - self.heading_command_re = self.checker.heading_command_re - - ### - # Main process/checking methods, arranged roughly from largest scope to smallest scope. - ### - - def process(self): - """Check the stream (file, string) created by the streammaker supplied to the constructor. - - This is the top-level method for checking a spec file. - """ - self.logger.info("processing file %s", self.filename) - - # File content checks - performed line-by-line - with self.stream_maker.make_stream() as f: - # Iterate through lines, calling processLine on each. - for lineIndex, line in enumerate(f): - trimmedLine = line.rstrip() - self.lines.append(trimmedLine) - self.processLine(lineIndex + 1, trimmedLine) - - # End of file checks follow: - - # Check "state" at end of file: should have blocks closed. - if self.prev_line_ref_page_tag: - self.error(MessageId.REFPAGE_BLOCK, - "Reference page tag seen, but block not opened before end of file.", - context=self.storeMessageContext(match=None)) - - if self.block_stack: - locations = (x.context for x in self.block_stack) - formatted_locations = ['{} opened at {}'.format(x.delimiter, self.getBriefLocation(x.context)) - for x in self.block_stack] - self.logger.warning("Unclosed blocks: %s", - ', '.join(formatted_locations)) - - self.error(MessageId.UNCLOSED_BLOCK, - ["Reached end of page, with these unclosed blocks remaining:"] + - formatted_locations, - context=self.storeMessageContext(match=None), - see_also=locations) - - # Check that every include of an /api/ file in the protos or structs category - # had a matching /validity/ include - for entity, includeContext in self.fs_api_includes.items(): - if not self.checker.entity_db.entityHasValidity(entity): - continue - - if entity in self.missing_validity_suppressions: - continue - - if entity not in self.validity_includes: - self.warning(MessageId.MISSING_VALIDITY_INCLUDE, - ['Saw /api/ include for {}, but no matching /validity/ include'.format(entity), - 'Expected a line with ' + regenerateIncludeFromMatch(includeContext.match, 'validity')], - context=includeContext) - - # Check that we never include a /validity/ file - # without a matching /api/ include - for entity, includeContext in self.validity_includes.items(): - if entity not in self.fs_api_includes: - self.error(MessageId.MISSING_API_INCLUDE, - ['Saw /validity/ include for {}, but no matching /api/ include'.format(entity), - 'Expected a line with ' + regenerateIncludeFromMatch(includeContext.match, 'api')], - context=includeContext) - - if not self.numDiagnostics(): - # no problems, exit quietly - return - - print('\nFor file {}:'.format(self.filename)) - - self.printMessageCounts() - numFixes = len(self.fixes) - if numFixes > 0: - fixes = ', '.join(('{} -> {}'.format(search, replace) - for search, replace in self.fixes)) - - print('{} unique auto-fix {} recorded: {}'.format(numFixes, - _pluralize('pattern', numFixes), fixes)) - - def processLine(self, lineNum, line): - """Check the contents of a single line from a file. - - Eventually populates self.match, self.entity, self.macro, - before calling processMatch. - """ - self.lineNum = lineNum - self.line = line - self.match = None - self.entity = None - self.macro = None - - self.logger.debug("processing line %d", lineNum) - - if self.processPossibleBlockDelimiter(): - # This is a block delimiter - proceed to next line. - # Block-type-specific stuff goes in processBlockOpen and processBlockClosed. - return - - if self.in_code_block: - # We do no processing in a code block. - return - - ### - # Detect if the previous line was [open,...] starting a refpage - # but this line isn't -- - # If the line is some other block delimiter, - # the related code in self.processPossibleBlockDelimiter() - # would have handled it. - # (because execution would never get to here for that line) - if self.prev_line_ref_page_tag: - self.handleExpectedRefpageBlock() - - ### - # Detect headings - if line.startswith('=='): - # Headings cause us to clear our pname_context - self.pname_data = None - - command = self.heading_command_re.match(line) - if command: - data = self.checker.findEntity(command) - if data: - self.pname_data = data - return - - ### - # Detect [open, lines for manpages - if line.startswith('[open,'): - self.checkRefPage() - return - - ### - # Skip comments - if line.lstrip().startswith('//'): - return - - ### - # Skip ifdef/endif - if SKIP_LINE.match(line): - return - - ### - # Detect include:::....[] lines - match = INCLUDE.match(line) - if match: - self.match = match - entity = match.group('entity_name') - - data = self.checker.findEntity(entity) - if not data: - self.error(MessageId.UNKNOWN_INCLUDE, - 'Saw include for {}, but that entity is unknown.'.format(entity)) - self.pname_data = None - return - - self.pname_data = data - - if match.group('generated_type') == 'api': - self.recordInclude(self.checker.apiIncludes) - - # Set mentions to None. The first time we see something like `* pname:paramHere`, - # we will set it to an empty set - self.pname_mentions[entity] = None - - if match.group('category') in CATEGORIES_WITH_VALIDITY: - self.fs_api_includes[entity] = self.storeMessageContext() - - if entity in self.validity_includes: - name_and_line = toNameAndLine( - self.validity_includes[entity], root_path=self.checker.root_path) - self.error(MessageId.API_VALIDITY_ORDER, - ['/api/ include found for {} after a corresponding /validity/ include'.format(entity), - 'Validity include located at {}'.format(name_and_line)]) - - elif match.group('generated_type') == 'validity': - self.recordInclude(self.checker.validityIncludes) - self.validity_includes[entity] = self.storeMessageContext() - - if entity not in self.pname_mentions: - self.error(MessageId.API_VALIDITY_ORDER, - '/validity/ include found for {} without a preceding /api/ include'.format(entity)) - return - - if self.pname_mentions[entity]: - # Got a validity include and we have seen at least one * pname: line - # since we got the API include - # so we can warn if we haven't seen a reference to every - # parameter/member. - members = self.checker.getMemberNames(entity) - missing = [member for member in members - if member not in self.pname_mentions[entity]] - if missing: - self.error(MessageId.UNDOCUMENTED_MEMBER, - ['Validity include found for {}, but not all members/params apparently documented'.format(entity), - 'Members/params not mentioned with pname: {}'.format(', '.join(missing))]) - - # If we found an include line, we're done with this line. - return - - if self.pname_data is not None and '* pname:' in line: - context_entity = self.pname_data.entity - if self.pname_mentions[context_entity] is None: - # First time seeting * pname: after an api include, prepare the set that - # tracks - self.pname_mentions[context_entity] = set() - - ### - # Detect [[Entity]] anchors - for match in ANCHOR.finditer(line): - entity = match.group('entity_name') - if self.checker.findEntity(entity): - # We found an anchor with the same name as an entity: - # treat it (mostly) like an API include - self.match = match - self.recordInclude(self.checker.apiIncludes, - generated_type='api (manual anchor)') - - ### - # Detect :: without pname - for match in MEMBER_REFERENCE.finditer(line): - if not match.group('member_macro'): - self.match = match - # Got :: but not followed by pname - - search = match.group() - replacement = match.group( - 'first_part') + '::pname:' + match.group('second_part') - self.error(MessageId.MEMBER_PNAME_MISSING, - 'Found a function parameter or struct member reference with :: but missing pname:', - group='double_colons', - replacement='::pname:', - fix=(search, replacement)) - - # check pname here because it won't come up in normal iteration below - # because of the missing macro - self.entity = match.group('entity_name') - self.checkPname(match.group('scope')) - - ### - # Look for things that seem like a missing macro. - for match in self.suspected_missing_macro_re.finditer(line): - if OPEN_LINK.match(line, endpos=match.start()): - # this is in a link, skip it. - continue - if CLOSE_LINK.match(line[match.end():]): - # this is in a link, skip it. - continue - - entity = match.group('entity_name') - self.match = match - self.entity = entity - data = self.checker.findEntity(entity) - if data: - - if data.category == EXTENSION_CATEGORY: - # Ah, this is an extension - self.warning(MessageId.EXTENSION, "Seems like this is an extension name that was not linked.", - group='entity_name', replacement=self.makeExtensionLink()) - else: - self.warning(MessageId.MISSING_MACRO, - ['Seems like a "{}" macro was omitted for this reference to a known entity in category "{}".'.format(data.macro, data.category), - 'Wrap in ` ` to silence this if you do not want a verified macro here.'], - group='entity_name', - replacement=self.makeMacroMarkup(data.macro)) - else: - - dataArray = self.checker.findEntityCaseInsensitive(entity) - # We might have found the goof... - - if dataArray: - if len(dataArray) == 1: - # Yep, found the goof: - # incorrect macro and entity capitalization - data = dataArray[0] - if data.category == EXTENSION_CATEGORY: - # Ah, this is an extension - self.warning(MessageId.EXTENSION, - "Seems like this is an extension name that was not linked.", - group='entity_name', replacement=self.makeExtensionLink(data.entity)) - else: - self.warning(MessageId.MISSING_MACRO, - 'Seems like a macro was omitted for this reference to a known entity in category "{}", found by searching case-insensitively.'.format( - data.category), - replacement=self.makeMacroMarkup(data=data)) - - else: - # Ugh, more than one resolution - - self.warning(MessageId.MISSING_MACRO, - ['Seems like a macro was omitted for this reference to a known entity, found by searching case-insensitively.', - 'More than one apparent match.'], - group='entity_name', see_also=dataArray[:]) - - ### - # Main operations: detect markup macros - for match in self.checker.macro_re.finditer(line): - self.match = match - self.macro = match.group('macro') - self.entity = match.group('entity_name') - self.subscript = match.group('subscript') - self.processMatch() - - def processPossibleBlockDelimiter(self): - """Look at the current line, and if it's a delimiter, update the block stack. - - Calls self.processBlockDelimiter() as required. - - Returns True if a delimiter was processed, False otherwise. - """ - line = self.line - new_block_type = BlockType.lineToBlockType(line) - if not new_block_type: - return False - - ### - # Detect if the previous line was [open,...] starting a refpage - # but this line is some block delimiter other than -- - # Must do this here because if we get a different block open instead of the one we want, - # the order of block opening will be wrong. - if new_block_type != BlockType.REF_PAGE_LIKE and self.prev_line_ref_page_tag: - self.handleExpectedRefpageBlock() - - # Delegate to the main process for delimiters. - self.processBlockDelimiter(line, new_block_type) - - return True - - def processBlockDelimiter(self, line, new_block_type, context=None): - """Update the block stack based on the current or supplied line. - - Calls self.processBlockOpen() or self.processBlockClosed() as required. - - Called by self.processPossibleBlockDelimiter() both in normal operation, as well as - when "faking" a ref page block open. - - Returns BlockProcessResult. - """ - if not context: - context = self.storeMessageContext() - - location = self.getBriefLocation(context) - - top = self.getInnermostBlockEntry() - top_delim = self.getInnermostBlockDelimiter() - if top_delim == line: - self.processBlockClosed() - return - - if top and top.block_type == new_block_type: - # Same block type, but not matching - might be an error? - # TODO maybe create a diagnostic here? - self.logger.warning( - "processPossibleBlockDelimiter: %s: Matched delimiter type %s, but did not exactly match current delim %s to top of stack %s, may be a typo?", - location, new_block_type, line, top_delim) - - # Empty stack, or top doesn't match us. - self.processBlockOpen(new_block_type, delimiter=line) - - def processBlockOpen(self, block_type, context=None, delimiter=None): - """Do any block-type-specific processing and push the new block. - - Must call self.pushBlock(). - May be overridden (carefully) or extended. - - Called by self.processBlockDelimiter(). - """ - if block_type == BlockType.REF_PAGE_LIKE: - if self.prev_line_ref_page_tag: - if self.current_ref_page: - refpage = self.current_ref_page - else: - refpage = '?refpage-with-invalid-tag?' - - self.logger.info( - 'processBlockOpen: Opening refpage for %s', refpage) - # Opening of refpage block "consumes" the preceding ref - # page context - self.prev_line_ref_page_tag = None - self.pushBlock(block_type, refpage=refpage, - context=context, delimiter=delimiter) - self.in_ref_page = True - return - - if block_type == BlockType.CODE: - self.in_code_block = True - - self.pushBlock(block_type, context=context, delimiter=delimiter) - - def processBlockClosed(self): - """Do any block-type-specific processing and pop the top block. - - Must call self.popBlock(). - May be overridden (carefully) or extended. - - Called by self.processPossibleBlockDelimiter(). - """ - old_top = self.popBlock() - - if old_top.block_type == BlockType.CODE: - self.in_code_block = False - - elif old_top.block_type == BlockType.REF_PAGE_LIKE and old_top.refpage: - self.logger.info( - 'processBlockClosed: Closing refpage for %s', old_top.refpage) - # leaving a ref page so reset associated state. - self.current_ref_page = None - self.prev_line_ref_page_tag = None - self.in_ref_page = False - - def processMatch(self): - """Process a match of the macro:entity regex for correctness.""" - match = self.match - entity = self.entity - macro = self.macro - - ### - # Track entities that we're actually linking to. - ### - if self.checker.entity_db.isLinkedMacro(macro): - self.checker.addLinkToEntity(entity, self.storeMessageContext()) - - ### - # Link everything that should be, and nothing that shouldn't be - ### - if self.checkRecognizedEntity(): - # if this returns true, - # then there is no need to do the remaining checks on this match - return - - ### - # Non-existent macros - if macro in NON_EXISTENT_MACROS: - self.error(MessageId.BAD_MACRO, '{} is not a macro provided in the specification, despite resembling other macros.'.format( - macro), group='macro') - - ### - # Wildcards (or leading underscore, or square brackets) - # if and only if a 'text' macro - self.checkText() - - # Do some validation of pname references. - if macro == 'pname': - # See if there's an immediately-preceding entity - preceding = self.line[:match.start()] - scope = PRECEDING_MEMBER_REFERENCE.search(preceding) - if scope: - # Yes there is, check it out. - self.checkPname(scope.group('entity_name')) - elif self.current_ref_page is not None: - # No, but there is a current ref page: very reliable - self.checkPnameImpliedContext(self.current_ref_page) - elif self.pname_data is not None: - # No, but there is a pname_context - better than nothing. - self.checkPnameImpliedContext(self.pname_data) - else: - # no, and no existing context we can imply: - # can't check this. - pass - - def checkRecognizedEntity(self): - """Check the current macro:entity match to see if it is recognized. - - Returns True if there is no need to perform further checks on this match. - - Helps avoid duplicate warnings/errors: typically each macro should have at most - one of this class of errors. - """ - entity = self.entity - macro = self.macro - if self.checker.findMacroAndEntity(macro, entity) is not None: - # We know this macro-entity combo - return True - - # We don't know this macro-entity combo. - possibleCats = self.checker.entity_db.getCategoriesForMacro(macro) - if possibleCats is None: - possibleCats = ['???'] - msg = ['Definition of link target {} with macro {} (used for {} {}) does not exist.'.format( - entity, - macro, - _pluralize('category', len(possibleCats)), - ', '.join(possibleCats))] - - data = self.checker.findEntity(entity) - if data: - # We found the goof: incorrect macro - msg.append('Apparently matching entity in category {} found.'.format( - data.category)) - self.handleWrongMacro(msg, data) - return True - - see_also = [] - dataArray = self.checker.findEntityCaseInsensitive(entity) - if dataArray: - # We might have found the goof... - - if len(dataArray) == 1: - # Yep, found the goof: - # incorrect macro and entity capitalization - data = dataArray[0] - msg.append('Apparently matching entity in category {} found by searching case-insensitively.'.format( - data.category)) - self.handleWrongMacro(msg, data) - return True - else: - # Ugh, more than one resolution - msg.append( - 'More than one apparent match found by searching case-insensitively, cannot auto-fix.') - see_also = dataArray[:] - - # OK, so we don't recognize this entity (and couldn't auto-fix it). - - if self.checker.entity_db.shouldBeRecognized(macro, entity): - # We should know the target - it's a link macro, - # or there's some reason the entity DB thinks we should know it. - if self.checker.likelyRecognizedEntity(entity): - # Should be linked and it matches our pattern, - # so probably not wrong macro. - # Human brains required. - if not self.checkText(): - self.error(MessageId.BAD_ENTITY, msg + ['Might be a misspelling, or, less likely, the wrong macro.'], - see_also=see_also) - else: - # Doesn't match our pattern, - # so probably should be name instead of link. - newMacro = macro[0] + 'name' - if self.checker.entity_db.isValidMacro(newMacro): - self.error(MessageId.BAD_ENTITY, msg + - ['Entity name does not fit the pattern for this API, which would mean it should be a "name" macro instead of a "link" macro'], - group='macro', replacement=newMacro, fix=self.makeFix(newMacro=newMacro), see_also=see_also) - else: - self.error(MessageId.BAD_ENTITY, msg + - ['Entity name does not fit the pattern for this API, which would mean it should be a "name" macro instead of a "link" macro', - 'However, {} is not a known macro so cannot auto-fix.'.format(newMacro)], see_also=see_also) - - elif macro == 'ename': - # TODO This might be an ambiguity in the style guide - ename might be a known enumerant value, - # or it might be an enumerant value in an external library, etc. that we don't know about - so - # hard to check this. - if self.checker.likelyRecognizedEntity(entity): - if not self.checkText(): - self.warning(MessageId.BAD_ENUMERANT, msg + - ['Unrecognized ename:{} that we would expect to recognize since it fits the pattern for this API.'.format(entity)], see_also=see_also) - else: - # This is fine: - # it doesn't need to be recognized since it's not linked. - pass - # Don't skip other tests. - return False - - def checkText(self): - """Evaluate the usage (or non-usage) of a *text macro. - - Wildcards (or leading or trailing underscore, or square brackets with - nothing or a placeholder) if and only if a 'text' macro. - - Called by checkRecognizedEntity() when appropriate. - """ - macro = self.macro - entity = self.entity - shouldBeText = shouldEntityBeText(entity, self.subscript) - if shouldBeText and not self.macro.endswith( - 'text') and not self.macro == 'code': - newMacro = macro[0] + 'text' - if self.checker.entity_db.getCategoriesForMacro(newMacro): - self.error(MessageId.MISSING_TEXT, - ['Asterisk/leading or trailing underscore/bracket found - macro should end with "text:", probably {}:'.format(newMacro), - AUTO_FIX_STRING], - group='macro', replacement=newMacro, fix=self.makeFix(newMacro=newMacro)) - else: - self.error(MessageId.MISSING_TEXT, - ['Asterisk/leading or trailing underscore/bracket found, so macro should end with "text:".', - 'However {}: is not a known macro so cannot auto-fix.'.format(newMacro)], - group='macro') - return True - elif macro.endswith('text') and not shouldBeText: - msg = [ - "No asterisk/leading or trailing underscore/bracket in the entity, so this might be a mistaken use of the 'text' macro {}:".format(macro)] - data = self.checker.findEntity(entity) - if data: - # We found the goof: incorrect macro - msg.append('Apparently matching entity in category {} found.'.format( - data.category)) - msg.append(AUTO_FIX_STRING) - replacement = self.makeFix(data=data) - if data.category == EXTENSION_CATEGORY: - self.error(MessageId.EXTENSION, msg, - replacement=replacement, fix=replacement) - else: - self.error(MessageId.WRONG_MACRO, msg, - group='macro', replacement=data.macro, fix=replacement) - else: - if self.checker.likelyRecognizedEntity(entity): - # This is a use of *text: for something that fits the pattern but isn't in the spec. - # This is OK. - return False - msg.append('Entity not found in spec, either.') - if macro[0] != 'e': - # Only suggest a macro if we aren't in elink/ename/etext, - # since ename and elink are not related in an equivalent way - # to the relationship between flink and fname. - newMacro = macro[0] + 'name' - if self.checker.entity_db.getCategoriesForMacro(newMacro): - msg.append( - 'Consider if {}: might be the correct macro to use here.'.format(newMacro)) - else: - msg.append( - 'Cannot suggest a new macro because {}: is not a known macro.'.format(newMacro)) - self.warning(MessageId.MISUSED_TEXT, msg) - return True - return False - - def checkPnameImpliedContext(self, pname_context): - """Handle pname: macros not immediately preceded by something like flink:entity or slink:entity. - - Also records pname: mentions of members/parameters for completeness checking in doc blocks. - - Contains call to self.checkPname(). - Called by self.processMatch() - """ - self.checkPname(pname_context.entity) - if pname_context.entity in self.pname_mentions and \ - self.pname_mentions[pname_context.entity] is not None: - # Record this mention, - # in case we're in the documentation block. - self.pname_mentions[pname_context.entity].add(self.entity) - - def checkPname(self, pname_context): - """Check the current match (as a pname: usage) with the given entity as its 'pname context', if possible. - - e.g. slink:foo::pname:bar, pname_context would be 'foo', while self.entity would be 'bar', etc. - - Called by self.processLine(), self.processMatch(), as well as from self.checkPnameImpliedContext(). - """ - if '*' in pname_context: - # This context has a placeholder, can't verify it. - return - - entity = self.entity - - context_data = self.checker.findEntity(pname_context) - members = self.checker.getMemberNames(pname_context) - - if context_data and not members: - # This is a recognized parent entity that doesn't have detectable member names, - # skip validation - # TODO: Annotate parameters of function pointer types with <name> - # and <param>? - return - if not members: - self.warning(MessageId.UNRECOGNIZED_CONTEXT, - 'pname context entity was un-recognized {}'.format(pname_context)) - return - - if entity not in members: - self.warning(MessageId.UNKNOWN_MEMBER, ["Could not find member/param named '{}' in {}".format(entity, pname_context), - 'Known {} mamber/param names are: {}'.format( - pname_context, ', '.join(members))], group='entity_name') - - def checkIncludeRefPageRelation(self, entity, generated_type): - """Identify if our current ref page (or lack thereof) is appropriate for an include just recorded. - - Called by self.recordInclude(). - """ - if not self.in_ref_page: - # Not in a ref page block: This probably means this entity needs a - # ref-page block added. - self.handleIncludeMissingRefPage(entity, generated_type) - return - - if not isinstance(self.current_ref_page, EntityData): - # This isn't a fully-valid ref page, so can't check the includes any better. - return - - ref_page_entity = self.current_ref_page.entity - if ref_page_entity not in self.refpage_includes: - self.refpage_includes[ref_page_entity] = set() - expected_ref_page_entity = self.computeExpectedRefPageFromInclude( - entity) - self.refpage_includes[ref_page_entity].add((generated_type, entity)) - - if ref_page_entity == expected_ref_page_entity: - # OK, this is a total match. - pass - elif self.checker.entity_db.areAliases(expected_ref_page_entity, ref_page_entity): - # This appears to be a promoted synonym which is OK. - pass - else: - # OK, we are in a ref page block that doesn't match - self.handleIncludeMismatchRefPage(entity, generated_type) - - def checkRefPage(self): - """Check if the current line (a refpage tag) meets requirements. - - Called by self.processLine(). - """ - line = self.line - - # Should always be found - self.match = BRACKETS.match(line) - - data = None - directory = None - if self.in_ref_page: - msg = ["Found reference page markup, but we are already in a refpage block.", - "The block before the first message of this type is most likely not closed.", ] - # Fake-close the previous ref page, if it's trivial to do so. - if self.getInnermostBlockEntry().block_type == BlockType.REF_PAGE_LIKE: - msg.append( - "Pretending that there was a line with `--` immediately above to close that ref page, for more readable messages.") - self.processBlockDelimiter( - REF_PAGE_LIKE_BLOCK_DELIM, BlockType.REF_PAGE_LIKE) - else: - msg.append( - "Ref page wasn't the last block opened, so not pretending to auto-close it for more readable messages.") - - self.error(MessageId.REFPAGE_BLOCK, msg) - - attribs = parseRefPageAttribs(line) - - unknown_attribs = set(attribs.keys()).difference( - VALID_REF_PAGE_ATTRIBS) - if unknown_attribs: - self.error(MessageId.REFPAGE_UNKNOWN_ATTRIB, - "Found unknown attrib(s) in reference page markup: " + ','.join(unknown_attribs)) - - # Required field: refpage='xrValidEntityHere' - if Attrib.REFPAGE.value in attribs: - attrib = attribs[Attrib.REFPAGE.value] - text = attrib.value - self.entity = text - - context = self.storeMessageContext( - group='value', match=attrib.match) - if self.checker.seenRefPage(text): - self.error(MessageId.REFPAGE_DUPLICATE, - ["Found reference page markup when we already saw refpage='{}' elsewhere.".format( - text), - "This (or the other mention) may be a copy-paste error."], - context=context) - self.checker.addRefPage(text) - - # Skip entity check if it's a spir-v built in - type = '' - if Attrib.TYPE.value in attribs: - type = attribs[Attrib.TYPE.value].value - - if type != 'builtins' and type != 'spirv': - data = self.checker.findEntity(text) - self.current_ref_page = data - if data: - # OK, this is a known entity that we're seeing a refpage for. - directory = data.directory - else: - # TODO suggest fixes here if applicable - self.error(MessageId.REFPAGE_NAME, - [ "Found reference page markup, but refpage='{}' type='{}' does not refer to a recognized entity".format( - text, type), - 'If this is intentional, add the entity to EXTRA_DEFINES or EXTRA_REFPAGES in check_spec_links.py.' ], - context=context) - - else: - self.error(MessageId.REFPAGE_TAG, - "Found apparent reference page markup, but missing refpage='...'", - group=None) - - # Required field: desc='preferably non-empty' - if Attrib.DESC.value in attribs: - attrib = attribs[Attrib.DESC.value] - text = attrib.value - if not text: - context = self.storeMessageContext( - group=None, match=attrib.match) - self.warning(MessageId.REFPAGE_MISSING_DESC, - "Found reference page markup, but desc='' is empty", - context=context) - else: - self.error(MessageId.REFPAGE_TAG, - "Found apparent reference page markup, but missing desc='...'", - group=None) - - # Required field: type='protos' for example - # (used by genRef.py to compute the macro to use) - if Attrib.TYPE.value in attribs: - attrib = attribs[Attrib.TYPE.value] - text = attrib.value - if directory and not text == directory: - context = self.storeMessageContext( - group='value', match=attrib.match) - self.error(MessageId.REFPAGE_TYPE, - "Found reference page markup, but type='{}' is not the expected value '{}'".format( - text, directory), - context=context) - else: - self.error(MessageId.REFPAGE_TAG, - "Found apparent reference page markup, but missing type='...'", - group=None) - - # Optional field: alias='spaceDelimited validEntities' - # Currently does nothing. Could modify checkRefPageXrefs to also - # check alias= attribute value - # if Attrib.ALIAS.value in attribs: - # # This field is optional - # self.checkRefPageXrefs(attribs[Attrib.XREFS.value]) - - # Optional field: xrefs='spaceDelimited validEntities' - if Attrib.XREFS.value in attribs: - # This field is optional - self.checkRefPageXrefs(attribs[Attrib.XREFS.value]) - self.prev_line_ref_page_tag = self.storeMessageContext() - - def checkRefPageXrefs(self, xrefs_attrib): - """Check all cross-refs indicated in an xrefs attribute for a ref page. - - Called by self.checkRefPage(). - - Argument: - xrefs_attrib -- A match of REF_PAGE_ATTRIB where the group 'key' is 'xrefs'. - """ - text = xrefs_attrib.value - context = self.storeMessageContext( - group='value', match=xrefs_attrib.match) - - def splitRefs(s): - """Split the string on whitespace, into individual references.""" - return s.split() # [x for x in s.split() if x] - - def remakeRefs(refs): - """Re-create a xrefs string from something list-shaped.""" - return ' '.join(refs) - - refs = splitRefs(text) - - # Pre-checking if messages are enabled, so that we can correctly determine - # the current string following any auto-fixes: - # the fixes for messages directly in this method would interact, - # and thus must be in the order specified here. - - if self.messageEnabled(MessageId.REFPAGE_XREFS_COMMA) and ',' in text: - old_text = text - # Re-split after replacing commas. - refs = splitRefs(text.replace(',', ' ')) - # Re-create the space-delimited text. - text = remakeRefs(refs) - self.error(MessageId.REFPAGE_XREFS_COMMA, - "Found reference page markup, with an unexpected comma in the (space-delimited) xrefs attribute", - context=context, - replacement=text, - fix=(old_text, text)) - - # We could conditionally perform this creation, but the code complexity would increase substantially, - # for presumably minimal runtime improvement. - unique_refs = OrderedDict.fromkeys(refs) - if self.messageEnabled(MessageId.REFPAGE_XREF_DUPE) and len(unique_refs) != len(refs): - # TODO is it safe to auto-fix here? - old_text = text - text = remakeRefs(unique_refs.keys()) - self.warning(MessageId.REFPAGE_XREF_DUPE, - ["Reference page for {} contains at least one duplicate in its cross-references.".format( - self.entity), - "Look carefully to see if this is a copy and paste error and should be changed to a different but related entity:", - "auto-fix simply removes the duplicate."], - context=context, - replacement=text, - fix=(old_text, text)) - - if self.messageEnabled(MessageId.REFPAGE_SELF_XREF) and self.entity and self.entity in unique_refs: - # Not modifying unique_refs here because that would accidentally affect the whitespace auto-fix. - new_text = remakeRefs( - [x for x in unique_refs.keys() if x != self.entity]) - - # DON'T AUTOFIX HERE because these are likely copy-paste between related entities: - # e.g. a Create function and the associated CreateInfo struct. - self.warning(MessageId.REFPAGE_SELF_XREF, - ["Reference page for {} included itself in its cross-references.".format(self.entity), - "This is typically a copy and paste error, and the dupe should likely be changed to a different but related entity.", - "Not auto-fixing for this reason."], - context=context, - replacement=new_text,) - - # We didn't have another reason to replace the whole attribute value, - # so let's make sure it doesn't have any extra spaces - if self.messageEnabled(MessageId.REFPAGE_WHITESPACE) and xrefs_attrib.value == text: - old_text = text - text = remakeRefs(unique_refs.keys()) - if old_text != text: - self.warning(MessageId.REFPAGE_WHITESPACE, - ["Cross-references for reference page for {} had non-minimal whitespace,".format(self.entity), - "and no other enabled message has re-constructed this value already."], - context=context, - replacement=text, - fix=(old_text, text)) - - for entity in unique_refs.keys(): - self.checkRefPageXref(entity, context) - - def checkRefPageXref(self, referenced_entity, line_context): - """Check a single cross-reference entry for a refpage. - - Called by self.checkRefPageXrefs(). - - Arguments: - referenced_entity -- The individual entity under consideration from the xrefs='...' string. - line_context -- A MessageContext referring to the entire line. - """ - data = self.checker.findEntity(referenced_entity) - if data: - # This is OK - return - context = line_context - match = re.search(r'\b{}\b'.format(referenced_entity), self.line) - if match: - context = self.storeMessageContext( - group=None, match=match) - msg = ["Found reference page markup, with an unrecognized entity listed: {}".format( - referenced_entity)] - - see_also = None - dataArray = self.checker.findEntityCaseInsensitive( - referenced_entity) - - if dataArray: - # We might have found the goof... - - if len(dataArray) == 1: - # Yep, found the goof - incorrect entity capitalization - data = dataArray[0] - new_entity = data.entity - self.error(MessageId.REFPAGE_XREFS, msg + [ - 'Apparently matching entity in category {} found by searching case-insensitively.'.format( - data.category), - AUTO_FIX_STRING], - replacement=new_entity, - fix=(referenced_entity, new_entity), - context=context) - return - - # Ugh, more than one resolution - msg.append( - 'More than one apparent match found by searching case-insensitively, cannot auto-fix.') - see_also = dataArray[:] - else: - # Probably not just a typo - msg.append( - 'If this is intentional, add the entity to EXTRA_DEFINES or EXTRA_REFPAGES in check_spec_links.py.') - - # Multiple or no resolutions found - self.error(MessageId.REFPAGE_XREFS, - msg, - see_also=see_also, - context=context) - - ### - # Message-related methods. - ### - - def warning(self, message_id, messageLines, context=None, group=None, - replacement=None, fix=None, see_also=None, frame=None): - """Log a warning for the file, if the message ID is enabled. - - Wrapper around self.diag() that automatically sets severity as well as frame. - - Arguments: - message_id -- A MessageId value. - messageLines -- A string or list of strings containing a human-readable error description. - - Optional, named arguments: - context -- A MessageContext. If None, will be constructed from self.match and group. - group -- The name of the regex group in self.match that contains the problem. Only used if context is None. - If needed and is None, self.group is used instead. - replacement -- The string, if any, that should be suggested as a replacement for the group in question. - Does not create an auto-fix: sometimes we want to show a possible fix but aren't confident enough - (or can't easily phrase a regex) to do it automatically. - fix -- A (old text, new text) pair if this error is auto-fixable safely. - see_also -- An optional array of other MessageContext locations relevant to this message. - frame -- The 'inspect' stack frame corresponding to the location that raised this message. - If None, will assume it is the direct caller of self.warning(). - """ - if not frame: - frame = currentframe().f_back - self.diag(MessageType.WARNING, message_id, messageLines, group=group, - replacement=replacement, context=context, fix=fix, see_also=see_also, frame=frame) - - def error(self, message_id, messageLines, group=None, replacement=None, - context=None, fix=None, see_also=None, frame=None): - """Log an error for the file, if the message ID is enabled. - - Wrapper around self.diag() that automatically sets severity as well as frame. - - Arguments: - message_id -- A MessageId value. - messageLines -- A string or list of strings containing a human-readable error description. - - Optional, named arguments: - context -- A MessageContext. If None, will be constructed from self.match and group. - group -- The name of the regex group in self.match that contains the problem. Only used if context is None. - If needed and is None, self.group is used instead. - replacement -- The string, if any, that should be suggested as a replacement for the group in question. - Does not create an auto-fix: sometimes we want to show a possible fix but aren't confident enough - (or can't easily phrase a regex) to do it automatically. - fix -- A (old text, new text) pair if this error is auto-fixable safely. - see_also -- An optional array of other MessageContext locations relevant to this message. - frame -- The 'inspect' stack frame corresponding to the location that raised this message. - If None, will assume it is the direct caller of self.error(). - """ - if not frame: - frame = currentframe().f_back - self.diag(MessageType.ERROR, message_id, messageLines, group=group, - replacement=replacement, context=context, fix=fix, see_also=see_also, frame=frame) - - def diag(self, severity, message_id, messageLines, context=None, group=None, - replacement=None, fix=None, see_also=None, frame=None): - """Log a diagnostic for the file, if the message ID is enabled. - - Also records the auto-fix, if applicable. - - Arguments: - severity -- A MessageType value. - message_id -- A MessageId value. - messageLines -- A string or list of strings containing a human-readable error description. - - Optional, named arguments: - context -- A MessageContext. If None, will be constructed from self.match and group. - group -- The name of the regex group in self.match that contains the problem. Only used if context is None. - If needed and is None, self.group is used instead. - replacement -- The string, if any, that should be suggested as a replacement for the group in question. - Does not create an auto-fix: sometimes we want to show a possible fix but aren't confident enough - (or can't easily phrase a regex) to do it automatically. - fix -- A (old text, new text) pair if this error is auto-fixable safely. - see_also -- An optional array of other MessageContext locations relevant to this message. - frame -- The 'inspect' stack frame corresponding to the location that raised this message. - If None, will assume it is the direct caller of self.diag(). - """ - if not self.messageEnabled(message_id): - self.logger.debug( - 'Discarding a %s message because it is disabled.', message_id) - return - - if isinstance(messageLines, str): - messageLines = [messageLines] - - self.logger.info('Recording a %s message: %s', - message_id, ' '.join(messageLines)) - - # Ensure all auto-fixes are marked as such. - if fix is not None and AUTO_FIX_STRING not in messageLines: - messageLines.append(AUTO_FIX_STRING) - - if not frame: - frame = currentframe().f_back - if context is None: - message = Message(message_id=message_id, - message_type=severity, - message=messageLines, - context=self.storeMessageContext(group=group), - replacement=replacement, - see_also=see_also, - fix=fix, - frame=frame) - else: - message = Message(message_id=message_id, - message_type=severity, - message=messageLines, - context=context, - replacement=replacement, - see_also=see_also, - fix=fix, - frame=frame) - if fix is not None: - self.fixes.add(fix) - self.messages.append(message) - - def messageEnabled(self, message_id): - """Return true if the given message ID is enabled.""" - return message_id in self.enabled_messages - - ### - # Accessors for externally-interesting information - - def numDiagnostics(self): - """Count the total number of diagnostics (errors or warnings) for this file.""" - return len(self.messages) - - def numErrors(self): - """Count the total number of errors for this file.""" - return self.numMessagesOfType(MessageType.ERROR) - - def numMessagesOfType(self, message_type): - """Count the number of messages of a particular type (severity).""" - return len( - [msg for msg in self.messages if msg.message_type == message_type]) - - def hasFixes(self): - """Return True if any messages included auto-fix patterns.""" - return len(self.fixes) > 0 - - ### - # Assorted internal methods. - def printMessageCounts(self): - """Print a simple count of each MessageType of diagnostics.""" - for message_type in [MessageType.ERROR, MessageType.WARNING]: - count = self.numMessagesOfType(message_type) - if count > 0: - print('{num} {mtype}{s} generated.'.format( - num=count, mtype=message_type, s=_s_suffix(count))) - - def dumpInternals(self): - """Dump internal variables to screen, for debugging.""" - print('self.lineNum: ', self.lineNum) - print('self.line:', self.line) - print('self.prev_line_ref_page_tag: ', self.prev_line_ref_page_tag) - print('self.current_ref_page:', self.current_ref_page) - - def getMissingValiditySuppressions(self): - """Return an enumerable of entity names that we shouldn't warn about missing validity. - - May override. - """ - return [] - - def recordInclude(self, include_dict, generated_type=None): - """Store the current line as being the location of an include directive or equivalent. - - Reports duplicate include errors, as well as include/ref-page mismatch or missing ref-page, - by calling self.checkIncludeRefPageRelation() for "actual" includes (where generated_type is None). - - Arguments: - include_dict -- The include dictionary to update: one of self.apiIncludes or self.validityIncludes. - generated_type -- The type of include (e.g. 'api', 'valid', etc). By default, extracted from self.match. - """ - entity = self.match.group('entity_name') - if generated_type is None: - generated_type = self.match.group('generated_type') - - # Only checking the ref page relation if it's retrieved from regex. - # Otherwise it might be a manual anchor recorded as an include, - # etc. - self.checkIncludeRefPageRelation(entity, generated_type) - - if entity in include_dict: - self.error(MessageId.DUPLICATE_INCLUDE, - "Included {} docs for {} when they were already included.".format(generated_type, - entity), see_also=include_dict[entity]) - include_dict[entity].append(self.storeMessageContext()) - else: - include_dict[entity] = [self.storeMessageContext()] - - def getInnermostBlockEntry(self): - """Get the BlockEntry for the top block delim on our stack.""" - if not self.block_stack: - return None - return self.block_stack[-1] - - def getInnermostBlockDelimiter(self): - """Get the delimiter for the top block on our stack.""" - top = self.getInnermostBlockEntry() - if not top: - return None - return top.delimiter - - def pushBlock(self, block_type, refpage=None, context=None, delimiter=None): - """Push a new entry on the block stack.""" - if not delimiter: - self.logger.info("pushBlock: not given delimiter") - delimiter = self.line - if not context: - context = self.storeMessageContext() - - old_top_delim = self.getInnermostBlockDelimiter() - - self.block_stack.append(BlockEntry( - delimiter=delimiter, - context=context, - refpage=refpage, - block_type=block_type)) - - location = self.getBriefLocation(context) - self.logger.info( - "pushBlock: %s: Pushed %s delimiter %s, previous top was %s, now %d elements on the stack", - location, block_type.value, delimiter, old_top_delim, len(self.block_stack)) - - self.dumpBlockStack() - - def popBlock(self): - """Pop and return the top entry from the block stack.""" - old_top = self.block_stack.pop() - location = self.getBriefLocation(old_top.context) - self.logger.info( - "popBlock: %s: popping %s delimiter %s, now %d elements on the stack", - location, old_top.block_type.value, old_top.delimiter, len(self.block_stack)) - - self.dumpBlockStack() - - return old_top - - def dumpBlockStack(self): - self.logger.debug('Block stack, top first:') - for distFromTop, x in enumerate(reversed(self.block_stack)): - self.logger.debug(' - block_stack[%d]: Line %d: "%s" refpage=%s', - -1 - distFromTop, - x.context.lineNum, x.delimiter, x.refpage) - - def getBriefLocation(self, context): - """Format a context briefly - omitting the filename if it has newlines in it.""" - if '\n' in context.filename: - return 'input string line {}'.format(context.lineNum) - return '{}:{}'.format( - context.filename, context.lineNum) - - ### - # Handlers for a variety of diagnostic-meriting conditions - # - # Split out for clarity and for allowing fine-grained override on a per-project basis. - ### - - def handleIncludeMissingRefPage(self, entity, generated_type): - """Report a message about an include outside of a ref-page block.""" - msg = ["Found {} include for {} outside of a reference page block.".format(generated_type, entity), - "This is probably a missing reference page block."] - refpage = self.computeExpectedRefPageFromInclude(entity) - data = self.checker.findEntity(refpage) - if data: - msg.append('Expected ref page block might start like:') - msg.append(self.makeRefPageTag(refpage, data=data)) - else: - msg.append( - "But, expected ref page entity name {} isn't recognized...".format(refpage)) - self.warning(MessageId.REFPAGE_MISSING, msg) - - def handleIncludeMismatchRefPage(self, entity, generated_type): - """Report a message about an include not matching its containing ref-page block.""" - self.warning(MessageId.REFPAGE_MISMATCH, "Found {} include for {}, inside the reference page block of {}".format( - generated_type, entity, self.current_ref_page.entity)) - - def handleWrongMacro(self, msg, data): - """Report an appropriate message when we found that the macro used is incorrect. - - May be overridden depending on each API's behavior regarding macro misuse: - e.g. in some cases, it may be considered a MessageId.LEGACY warning rather than - a MessageId.WRONG_MACRO or MessageId.EXTENSION. - """ - message_type = MessageType.WARNING - message_id = MessageId.WRONG_MACRO - group = 'macro' - - if data.category == EXTENSION_CATEGORY: - # Ah, this is an extension - msg.append( - 'This is apparently an extension name, which should be marked up as a link.') - message_id = MessageId.EXTENSION - group = None # replace the whole thing - else: - # Non-extension, we found the macro though. - message_type = MessageType.ERROR - msg.append(AUTO_FIX_STRING) - self.diag(message_type, message_id, msg, - group=group, replacement=self.makeMacroMarkup(data=data), fix=self.makeFix(data=data)) - - def handleExpectedRefpageBlock(self): - """Handle expecting to see -- to start a refpage block, but not seeing that at all.""" - self.error(MessageId.REFPAGE_BLOCK, - ["Expected, but did not find, a line containing only -- following a reference page tag,", - "Pretending to insert one, for more readable messages."], - see_also=[self.prev_line_ref_page_tag]) - # Fake "in ref page" regardless, to avoid spurious extra errors. - self.processBlockDelimiter('--', BlockType.REF_PAGE_LIKE, - context=self.prev_line_ref_page_tag) - - ### - # Construct related values (typically named tuples) based on object state and supplied arguments. - # - # Results are typically supplied to another method call. - ### - - def storeMessageContext(self, group=None, match=None): - """Create message context from corresponding instance variables. - - Arguments: - group -- The regex group name, if any, identifying the part of the match to highlight. - match -- The regex match. If None, will use self.match. - """ - if match is None: - match = self.match - return MessageContext(filename=self.filename, - lineNum=self.lineNum, - line=self.line, - match=match, - group=group) - - def makeFix(self, newMacro=None, newEntity=None, data=None): - """Construct a fix pair for replacing the old macro:entity with new. - - Wrapper around self.makeSearch() and self.makeMacroMarkup(). - """ - return (self.makeSearch(), self.makeMacroMarkup( - newMacro, newEntity, data)) - - def makeSearch(self): - """Construct the string self.macro:self.entity, for use in the old text part of a fix pair.""" - return '{}:{}'.format(self.macro, self.entity) - - def makeMacroMarkup(self, newMacro=None, newEntity=None, data=None): - """Construct appropriate markup for referring to an entity. - - Typically constructs macro:entity, but can construct `<<EXTENSION_NAME>>` if the supplied - entity is identified as an extension. - - Arguments: - newMacro -- The macro to use. Defaults to data.macro (if available), otherwise self.macro. - newEntity -- The entity to use. Defaults to data.entity (if available), otherwise self.entity. - data -- An EntityData value corresponding to this entity. If not provided, will be looked up by newEntity. - """ - if not newEntity: - if data: - newEntity = data.entity - else: - newEntity = self.entity - if not newMacro: - if data: - newMacro = data.macro - else: - newMacro = self.macro - if not data: - data = self.checker.findEntity(newEntity) - if data and data.category == EXTENSION_CATEGORY: - return self.makeExtensionLink(newEntity) - return '{}:{}'.format(newMacro, newEntity) - - def makeExtensionLink(self, newEntity=None): - """Create a correctly-formatted link to an extension. - - Result takes the form `<<EXTENSION_NAME>>`. - - Argument: - newEntity -- The extension name to link to. Defaults to self.entity. - """ - if not newEntity: - newEntity = self.entity - return '`<<{}>>`'.format(newEntity) - - def computeExpectedRefPageFromInclude(self, entity): - """Compute the expected ref page entity based on an include entity name.""" - # No-op in general. - return entity - - def makeRefPageTag(self, entity, data=None, - ref_type=None, desc='', xrefs=None): - """Construct a ref page tag string from attribute values.""" - if ref_type is None and data is not None: - ref_type = data.directory - if ref_type is None: - ref_type = "????" - return "[open,refpage='{}',type='{}',desc='{}',xrefs='{}']".format( - entity, ref_type, desc, ' '.join(xrefs or [])) diff --git a/codegen/vulkan/scripts/spec_tools/main.py b/codegen/vulkan/scripts/spec_tools/main.py deleted file mode 100644 index 2cd4f69c..00000000 --- a/codegen/vulkan/scripts/spec_tools/main.py +++ /dev/null @@ -1,244 +0,0 @@ -"""Provides a re-usable command-line interface to a MacroChecker.""" - -# Copyright (c) 2018-2019 Collabora, Ltd. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> - - -import argparse -import logging -import re -from pathlib import Path - -from .shared import MessageId - - -def checkerMain(default_enabled_messages, make_macro_checker, - all_docs, available_messages=None): - """Perform the bulk of the work for a command-line interface to a MacroChecker. - - Arguments: - default_enabled_messages -- The MessageId values that should be enabled by default. - make_macro_checker -- A function that can be called with a set of enabled MessageId to create a - properly-configured MacroChecker. - all_docs -- A list of all spec documentation files. - available_messages -- a list of all MessageId values that can be generated for this project. - Defaults to every value. (e.g. some projects don't have MessageId.LEGACY) - """ - enabled_messages = set(default_enabled_messages) - if not available_messages: - available_messages = list(MessageId) - - disable_args = [] - enable_args = [] - - parser = argparse.ArgumentParser() - parser.add_argument( - "--scriptlocation", - help="Append the script location generated a message to the output.", - action="store_true") - parser.add_argument( - "--verbose", - "-v", - help="Output 'info'-level development logging messages.", - action="store_true") - parser.add_argument( - "--debug", - "-d", - help="Output 'debug'-level development logging messages (more verbose than -v).", - action="store_true") - parser.add_argument( - "-Werror", - "--warning_error", - help="Make warnings act as errors, exiting with non-zero error code", - action="store_true") - parser.add_argument( - "--include_warn", - help="List all expected but unseen include files, not just those that are referenced.", - action='store_true') - parser.add_argument( - "-Wmissing_refpages", - help="List all entities with expected but unseen ref page blocks. NOT included in -Wall!", - action='store_true') - parser.add_argument( - "--include_error", - help="Make expected but unseen include files cause exiting with non-zero error code", - action='store_true') - parser.add_argument( - "--broken_error", - help="Make missing include/anchor for linked-to entities cause exiting with non-zero error code. Weaker version of --include_error.", - action='store_true') - parser.add_argument( - "--dump_entities", - help="Just dump the parsed entity data to entities.json and exit.", - action='store_true') - parser.add_argument( - "--html", - help="Output messages to the named HTML file instead of stdout.") - parser.add_argument( - "file", - help="Only check the indicated file(s). By default, all chapters and extensions are checked.", - nargs="*") - parser.add_argument( - "--ignore_count", - type=int, - help="Ignore up to the given number of errors without exiting with a non-zero error code.") - parser.add_argument("-Wall", - help="Enable all warning categories.", - action='store_true') - - for message_id in MessageId: - enable_arg = message_id.enable_arg() - enable_args.append((message_id, enable_arg)) - - disable_arg = message_id.disable_arg() - disable_args.append((message_id, disable_arg)) - if message_id in enabled_messages: - parser.add_argument('-' + disable_arg, action="store_true", - help="Disable message category {}: {}".format(str(message_id), message_id.desc())) - # Don't show the enable flag in help since it's enabled by default - parser.add_argument('-' + enable_arg, action="store_true", - help=argparse.SUPPRESS) - else: - parser.add_argument('-' + enable_arg, action="store_true", - help="Enable message category {}: {}".format(str(message_id), message_id.desc())) - # Don't show the disable flag in help since it's disabled by - # default - parser.add_argument('-' + disable_arg, action="store_true", - help=argparse.SUPPRESS) - - args = parser.parse_args() - - arg_dict = vars(args) - for message_id, arg in enable_args: - if args.Wall or (arg in arg_dict and arg_dict[arg]): - enabled_messages.add(message_id) - - for message_id, arg in disable_args: - if arg in arg_dict and arg_dict[arg]: - enabled_messages.discard(message_id) - - if args.verbose: - logging.basicConfig(level='INFO') - - if args.debug: - logging.basicConfig(level='DEBUG') - - checker = make_macro_checker(enabled_messages) - - if args.dump_entities: - with open('entities.json', 'w', encoding='utf-8') as f: - f.write(checker.getEntityJson()) - exit(0) - - if args.file: - files = (str(Path(f).resolve()) for f in args.file) - else: - files = all_docs - - for fn in files: - checker.processFile(fn) - - if args.html: - from .html_printer import HTMLPrinter - printer = HTMLPrinter(args.html) - else: - from .console_printer import ConsolePrinter - printer = ConsolePrinter() - - if args.scriptlocation: - printer.show_script_location = True - - if args.file: - printer.output("Only checked specified files.") - for f in args.file: - printer.output(f) - else: - printer.output("Checked all chapters and extensions.") - - if args.warning_error: - numErrors = checker.numDiagnostics() - else: - numErrors = checker.numErrors() - - check_includes = args.include_warn - check_broken = not args.file - - if args.file and check_includes: - print('Note: forcing --include_warn off because only checking supplied files.') - check_includes = False - - printer.outputResults(checker, broken_links=(not args.file), - missing_includes=check_includes) - - if check_broken: - numErrors += len(checker.getBrokenLinks()) - - if args.file and args.include_error: - print('Note: forcing --include_error off because only checking supplied files.') - args.include_error = False - if args.include_error: - numErrors += len(checker.getMissingUnreferencedApiIncludes()) - - check_missing_refpages = args.Wmissing_refpages - if args.file and check_missing_refpages: - print('Note: forcing -Wmissing_refpages off because only checking supplied files.') - check_missing_refpages = False - - if check_missing_refpages: - missing = checker.getMissingRefPages() - if missing: - printer.output("Expected, but did not find, ref page blocks for the following {} entities: {}".format( - len(missing), - ', '.join(missing) - )) - if args.warning_error: - numErrors += len(missing) - - printer.close() - - if args.broken_error and not args.file: - numErrors += len(checker.getBrokenLinks()) - - if checker.hasFixes(): - fixFn = 'applyfixes.sh' - print('Saving shell script to apply fixes as {}'.format(fixFn)) - with open(fixFn, 'w', encoding='utf-8') as f: - f.write('#!/bin/sh -e\n') - for fileChecker in checker.files: - wroteComment = False - for msg in fileChecker.messages: - if msg.fix is not None: - if not wroteComment: - f.write('\n# {}\n'.format(fileChecker.filename)) - wroteComment = True - search, replace = msg.fix - f.write( - r"sed -i -r 's~\b{}\b~{}~g' {}".format( - re.escape(search), - replace, - fileChecker.filename)) - f.write('\n') - - print('Total number of errors with this run: {}'.format(numErrors)) - - if args.ignore_count: - if numErrors > args.ignore_count: - # Exit with non-zero error code so that we "fail" CI, etc. - print('Exceeded specified limit of {}, so exiting with error'.format( - args.ignore_count)) - exit(1) - else: - print('At or below specified limit of {}, so exiting with success'.format( - args.ignore_count)) - exit(0) - - if numErrors: - # Exit with non-zero error code so that we "fail" CI, etc. - print('Exiting with error') - exit(1) - else: - print('Exiting with success') - exit(0) diff --git a/codegen/vulkan/scripts/spec_tools/shared.py b/codegen/vulkan/scripts/spec_tools/shared.py deleted file mode 100644 index bb6f1657..00000000 --- a/codegen/vulkan/scripts/spec_tools/shared.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Types, constants, and utility functions used by multiple sub-modules in spec_tools.""" - -# Copyright (c) 2018-2019 Collabora, Ltd. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> - -import platform -from collections import namedtuple -from enum import Enum -from inspect import getframeinfo -from pathlib import Path -from sys import stdout - -# if we have termcolor and we know our stdout is a TTY, -# pull it in and use it. -if hasattr(stdout, 'isatty') and stdout.isatty(): - try: - from termcolor import colored as colored_impl - HAVE_COLOR = True - except ImportError: - HAVE_COLOR = False -elif platform.system() == 'Windows': - try: - from termcolor import colored as colored_impl - import colorama - colorama.init() - HAVE_COLOR = True - except ImportError: - HAVE_COLOR = False - -else: - HAVE_COLOR = False - - -def colored(s, color=None, attrs=None): - """Call termcolor.colored with same arguments if this is a tty and it is available.""" - if HAVE_COLOR: - return colored_impl(s, color, attrs=attrs) - return s - - -### -# Constants used in multiple places. -AUTO_FIX_STRING = 'Note: Auto-fix available.' -EXTENSION_CATEGORY = 'extension' -CATEGORIES_WITH_VALIDITY = set(('protos', 'structs')) -NON_EXISTENT_MACROS = set(('plink', 'ttext', 'dtext')) - -### -# MessageContext: All the information about where a message relates to. -MessageContext = namedtuple('MessageContext', - ['filename', 'lineNum', 'line', - 'match', 'group']) - - -def getInterestedRange(message_context): - """Return a (start, end) pair of character index for the match in a MessageContext.""" - if not message_context.match: - # whole line - return (0, len(message_context.line)) - return (message_context.match.start(), message_context.match.end()) - - -def getHighlightedRange(message_context): - """Return a (start, end) pair of character index for the highlighted range in a MessageContext.""" - if message_context.group is not None and message_context.match is not None: - return (message_context.match.start(message_context.group), - message_context.match.end(message_context.group)) - # no group (whole match) or no match (whole line) - return getInterestedRange(message_context) - - -def toNameAndLine(context, root_path=None): - """Convert MessageContext into a simple filename:line string.""" - my_fn = Path(context.filename) - if root_path: - my_fn = my_fn.relative_to(root_path) - return '{}:{}'.format(str(my_fn), context.lineNum) - - -def generateInclude(dir_traverse, generated_type, category, entity): - """Create an include:: directive for geneated api or validity from the various pieces.""" - return 'include::{directory_traverse}{generated_type}/{category}/{entity_name}.txt[]'.format( - directory_traverse=dir_traverse, - generated_type=generated_type, - category=category, - entity_name=entity) - - -# Data stored per entity (function, struct, enumerant type, enumerant, extension, etc.) -EntityData = namedtuple( - 'EntityData', ['entity', 'macro', 'elem', 'filename', 'category', 'directory']) - - -class MessageType(Enum): - """Type of a message.""" - - WARNING = 1 - ERROR = 2 - NOTE = 3 - - def __str__(self): - """Format a MessageType as a lowercase string.""" - return str(self.name).lower() - - def formattedWithColon(self): - """Format a MessageType as a colored, lowercase string followed by a colon.""" - if self == MessageType.WARNING: - return colored(str(self) + ':', 'magenta', attrs=['bold']) - if self == MessageType.ERROR: - return colored(str(self) + ':', 'red', attrs=['bold']) - return str(self) + ':' - - -class MessageId(Enum): - """Enumerates the varieties of messages that can be generated. - - Control over enabled messages with -Wbla or -Wno_bla is per-MessageId. - """ - - MISSING_TEXT = 1 - LEGACY = 2 - WRONG_MACRO = 3 - MISSING_MACRO = 4 - BAD_ENTITY = 5 - BAD_ENUMERANT = 6 - BAD_MACRO = 7 - UNRECOGNIZED_CONTEXT = 8 - UNKNOWN_MEMBER = 9 - DUPLICATE_INCLUDE = 10 - UNKNOWN_INCLUDE = 11 - API_VALIDITY_ORDER = 12 - UNDOCUMENTED_MEMBER = 13 - MEMBER_PNAME_MISSING = 14 - MISSING_VALIDITY_INCLUDE = 15 - MISSING_API_INCLUDE = 16 - MISUSED_TEXT = 17 - EXTENSION = 18 - REFPAGE_TAG = 19 - REFPAGE_MISSING_DESC = 20 - REFPAGE_XREFS = 21 - REFPAGE_XREFS_COMMA = 22 - REFPAGE_TYPE = 23 - REFPAGE_NAME = 24 - REFPAGE_BLOCK = 25 - REFPAGE_MISSING = 26 - REFPAGE_MISMATCH = 27 - REFPAGE_UNKNOWN_ATTRIB = 28 - REFPAGE_SELF_XREF = 29 - REFPAGE_XREF_DUPE = 30 - REFPAGE_WHITESPACE = 31 - REFPAGE_DUPLICATE = 32 - UNCLOSED_BLOCK = 33 - - def __str__(self): - """Format as a lowercase string.""" - return self.name.lower() - - def enable_arg(self): - """Return the corresponding Wbla string to make the 'enable this message' argument.""" - return 'W{}'.format(self.name.lower()) - - def disable_arg(self): - """Return the corresponding Wno_bla string to make the 'enable this message' argument.""" - return 'Wno_{}'.format(self.name.lower()) - - def desc(self): - """Return a brief description of the MessageId suitable for use in --help.""" - return MessageId.DESCRIPTIONS[self] - - -MessageId.DESCRIPTIONS = { - MessageId.MISSING_TEXT: "a *text: macro is expected but not found", - MessageId.LEGACY: "legacy usage of *name: macro when *link: is applicable", - MessageId.WRONG_MACRO: "wrong macro used for an entity", - MessageId.MISSING_MACRO: "a macro might be missing", - MessageId.BAD_ENTITY: "entity not recognized, etc.", - MessageId.BAD_ENUMERANT: "unrecognized enumerant value used in ename:", - MessageId.BAD_MACRO: "unrecognized macro used", - MessageId.UNRECOGNIZED_CONTEXT: "pname used with an unrecognized context", - MessageId.UNKNOWN_MEMBER: "pname used but member/argument by that name not found", - MessageId.DUPLICATE_INCLUDE: "duplicated include line", - MessageId.UNKNOWN_INCLUDE: "include line specified file we wouldn't expect to exists", - MessageId.API_VALIDITY_ORDER: "saw API include after validity include", - MessageId.UNDOCUMENTED_MEMBER: "saw an apparent struct/function documentation, but missing a member", - MessageId.MEMBER_PNAME_MISSING: "pname: missing from a 'scope' operator", - MessageId.MISSING_VALIDITY_INCLUDE: "missing validity include", - MessageId.MISSING_API_INCLUDE: "missing API include", - MessageId.MISUSED_TEXT: "a *text: macro is found but not expected", - MessageId.EXTENSION: "an extension name is incorrectly marked", - MessageId.REFPAGE_TAG: "a refpage tag is missing an expected field", - MessageId.REFPAGE_MISSING_DESC: "a refpage tag has an empty description", - MessageId.REFPAGE_XREFS: "an unrecognized entity is mentioned in xrefs of a refpage tag", - MessageId.REFPAGE_XREFS_COMMA: "a comma was founds in xrefs of a refpage tag, which is space-delimited", - MessageId.REFPAGE_TYPE: "a refpage tag has an incorrect type field", - MessageId.REFPAGE_NAME: "a refpage tag has an unrecognized entity name in its refpage field", - MessageId.REFPAGE_BLOCK: "a refpage block is not correctly opened or closed.", - MessageId.REFPAGE_MISSING: "an API include was found outside of a refpage block.", - MessageId.REFPAGE_MISMATCH: "an API or validity include was found in a non-matching refpage block.", - MessageId.REFPAGE_UNKNOWN_ATTRIB: "a refpage tag has an unrecognized attribute", - MessageId.REFPAGE_SELF_XREF: "a refpage tag has itself in the list of cross-references", - MessageId.REFPAGE_XREF_DUPE: "a refpage cross-references list has at least one duplicate", - MessageId.REFPAGE_WHITESPACE: "a refpage cross-references list has non-minimal whitespace", - MessageId.REFPAGE_DUPLICATE: "a refpage tag has been seen for a single entity for a second time", - MessageId.UNCLOSED_BLOCK: "one or more blocks remain unclosed at the end of a file" -} - - -class Message(object): - """An Error, Warning, or Note with a MessageContext, MessageId, and message text. - - May optionally have a replacement, a see_also array, an auto-fix, - and a stack frame where the message was created. - """ - - def __init__(self, message_id, message_type, message, context, - replacement=None, see_also=None, fix=None, frame=None): - """Construct a Message. - - Typically called by MacroCheckerFile.diag(). - """ - self.message_id = message_id - - self.message_type = message_type - - if isinstance(message, str): - self.message = [message] - else: - self.message = message - - self.context = context - if context is not None and context.match is not None and context.group is not None: - if context.group not in context.match.groupdict(): - raise RuntimeError( - 'Group "{}" does not exist in the match'.format(context.group)) - - self.replacement = replacement - - self.fix = fix - - if see_also is None: - self.see_also = None - elif isinstance(see_also, MessageContext): - self.see_also = [see_also] - else: - self.see_also = see_also - - self.script_location = None - if frame: - try: - frameinfo = getframeinfo(frame) - self.script_location = "{}:{}".format( - frameinfo.filename, frameinfo.lineno) - finally: - del frame diff --git a/codegen/vulkan/scripts/spec_tools/util.py b/codegen/vulkan/scripts/spec_tools/util.py deleted file mode 100644 index b8906798..00000000 --- a/codegen/vulkan/scripts/spec_tools/util.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Utility functions not closely tied to other spec_tools types.""" -# Copyright 2018-2019 Collabora, Ltd. -# Copyright 2013-2021 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 - - -def getElemName(elem, default=None): - """Get the name associated with an element, either a name child or name attribute.""" - name_elem = elem.find('name') - if name_elem is not None: - return name_elem.text - # Fallback if there is no child. - return elem.get('name', default) - - -def getElemType(elem, default=None): - """Get the type associated with an element, either a type child or type attribute.""" - type_elem = elem.find('type') - if type_elem is not None: - return type_elem.text - # Fallback if there is no child. - return elem.get('type', default) - - -def findFirstWithPredicate(collection, pred): - """Return the first element that satisfies the predicate, or None if none exist. - - NOTE: Some places where this is used might be better served by changing to a dictionary. - """ - for elt in collection: - if pred(elt): - return elt - return None - - -def findNamedElem(elems, name): - """Traverse a collection of elements with 'name' nodes or attributes, looking for and returning one with the right name. - - NOTE: Many places where this is used might be better served by changing to a dictionary. - """ - return findFirstWithPredicate(elems, lambda elem: getElemName(elem) == name) - - -def findTypedElem(elems, typename): - """Traverse a collection of elements with 'type' nodes or attributes, looking for and returning one with the right typename. - - NOTE: Many places where this is used might be better served by changing to a dictionary. - """ - return findFirstWithPredicate(elems, lambda elem: getElemType(elem) == typename) - - -def findNamedObject(collection, name): - """Traverse a collection of elements with 'name' attributes, looking for and returning one with the right name. - - NOTE: Many places where this is used might be better served by changing to a dictionary. - """ - return findFirstWithPredicate(collection, lambda elt: elt.name == name) diff --git a/codegen/vulkan/scripts/spec_tools/validity.py b/codegen/vulkan/scripts/spec_tools/validity.py deleted file mode 100644 index 745ba013..00000000 --- a/codegen/vulkan/scripts/spec_tools/validity.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright 2013-2021 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -import re - - -_A_VS_AN_RE = re.compile(r' a ([a-z]+:)?([aAeEiIoOxX]\w+\b)(?!:)') - -_STARTS_WITH_MACRO_RE = re.compile(r'^[a-z]+:.*') - - -def _checkAnchorComponents(anchor): - """Raise an exception if any component of a VUID anchor name is illegal.""" - if anchor: - # Any other invalid things in an anchor name should be detected here. - if any((' ' in anchor_part for anchor_part in anchor)): - raise RuntimeError("Illegal component of a VUID anchor name!") - - -def _fix_a_vs_an(s): - """Fix usage (often generated) of the indefinite article 'a' when 'an' is appropriate. - - Explicitly excludes the markup macros.""" - return _A_VS_AN_RE.sub(r' an \1\2', s) - - -class ValidityCollection: - """Combines validity for a single entity.""" - - def __init__(self, entity_name=None, conventions=None, strict=True): - self.entity_name = entity_name - self.conventions = conventions - self.lines = [] - self.strict = strict - - def possiblyAddExtensionRequirement(self, extension_name, entity_preface): - """Add an extension-related validity statement if required. - - entity_preface is a string that goes between "must be enabled prior to " - and the name of the entity, and normally ends in a macro. - For instance, might be "calling flink:" for a function. - """ - if extension_name and not extension_name.startswith(self.conventions.api_version_prefix): - msg = 'The {} extension must: be enabled prior to {}{}'.format( - self.conventions.formatExtension(extension_name), entity_preface, self.entity_name) - self.addValidityEntry(msg, anchor=('extension', 'notenabled')) - - def addValidityEntry(self, msg, anchor=None): - """Add a validity entry, optionally with a VUID anchor. - - If any trailing arguments are supplied, - an anchor is generated by concatenating them with dashes - at the end of the VUID anchor name. - """ - if not msg: - raise RuntimeError("Tried to add a blank validity line!") - parts = ['*'] - _checkAnchorComponents(anchor) - if anchor: - if not self.entity_name: - raise RuntimeError('Cannot add a validity entry with an anchor to a collection that does not know its entity name.') - parts.append('[[{}]]'.format( - '-'.join(['VUID', self.entity_name] + list(anchor)))) - parts.append(msg) - combined = _fix_a_vs_an(' '.join(parts)) - if combined in self.lines: - raise RuntimeError("Duplicate validity added!") - self.lines.append(combined) - - def addText(self, msg): - """Add already formatted validity text.""" - if self.strict: - raise RuntimeError('addText called when collection in strict mode') - if not msg: - return - msg = msg.rstrip() - if not msg: - return - self.lines.append(msg) - - def _extend(self, lines): - lines = list(lines) - dupes = set(lines).intersection(self.lines) - if dupes: - raise RuntimeError("The two sets contain some shared entries! " + str(dupes)) - self.lines.extend(lines) - - def __iadd__(self, other): - """Perform += with a string, iterable, or ValidityCollection.""" - if other is None: - pass - elif isinstance(other, str): - if self.strict: - raise RuntimeError( - 'Collection += a string when collection in strict mode') - if not other: - # empty string - pass - elif other.startswith('*'): - # Handle already-formatted - self.addText(other) - else: - # Do the formatting ourselves. - self.addValidityEntry(other) - elif isinstance(other, ValidityEntry): - if other: - if other.verbose: - print(self.entity_name, 'Appending', str(other)) - self.addValidityEntry(str(other), anchor=other.anchor) - elif isinstance(other, ValidityCollection): - if not self.entity_name == other.entity_name: - raise RuntimeError( - "Trying to combine two ValidityCollections for different entities!") - self._extend(other.lines) - else: - # Deal with other iterables. - self._extend(other) - - return self - - def __bool__(self): - """Is the collection non-empty?""" - empty = not self.lines - return not empty - - @property - def text(self): - """Access validity statements as a single string or None.""" - if not self.lines: - return None - return '\n'.join(self.lines) + '\n' - - def __str__(self): - """Access validity statements as a single string or empty string.""" - if not self: - return '' - return self.text - - def __repr__(self): - return '<ValidityCollection: {}>'.format(self.lines) - - -class ValidityEntry: - """A single validity line in progress.""" - - def __init__(self, text=None, anchor=None): - """Prepare to add a validity entry, optionally with a VUID anchor. - - An anchor is generated by concatenating the elements of the anchor tuple with dashes - at the end of the VUID anchor name. - """ - _checkAnchorComponents(anchor) - if isinstance(anchor, str): - # anchor needs to be a tuple - anchor = (anchor,) - - # VUID does not allow special chars except ":" - if anchor is not None: - anchor = [(anchor_value.replace('->', '::').replace('.', '::')) for anchor_value in anchor] - - self.anchor = anchor - self.parts = [] - self.verbose = False - if text: - self.append(text) - - def append(self, part): - """Append a part of a string. - - If this is the first entry part and the part doesn't start - with a markup macro, the first character will be capitalized.""" - if not self.parts and not _STARTS_WITH_MACRO_RE.match(part): - self.parts.append(part[:1].upper()) - self.parts.append(part[1:]) - else: - self.parts.append(part) - if self.verbose: - print('ValidityEntry', id(self), 'after append:', str(self)) - - def drop_end(self, n): - """Remove up to n trailing characters from the string.""" - temp = str(self) - n = min(len(temp), n) - self.parts = [temp[:-n]] - - def __iadd__(self, other): - """Perform += with a string,""" - self.append(other) - return self - - def __bool__(self): - """Return true if we have something more than just an anchor.""" - empty = not self.parts - return not empty - - def __str__(self): - """Access validity statement as a single string or empty string.""" - if not self: - raise RuntimeError("No parts added?") - return ''.join(self.parts).strip() - - def __repr__(self): - parts = ['<ValidityEntry: '] - if self: - parts.append('"') - parts.append(str(self)) - parts.append('"') - else: - parts.append('EMPTY') - if self.anchor: - parts.append(', anchor={}'.format('-'.join(self.anchor))) - parts.append('>') - return ''.join(parts) |