diff options
Diffstat (limited to 'pw_presubmit/py/pw_presubmit/pigweed_presubmit.py')
| -rwxr-xr-x | pw_presubmit/py/pw_presubmit/pigweed_presubmit.py | 441 |
1 files changed, 356 insertions, 85 deletions
diff --git a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py index 1369fbc5e..65f1d7f0e 100755 --- a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py +++ b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py @@ -26,14 +26,6 @@ import subprocess import sys from typing import Callable, Iterable, List, Sequence, TextIO -try: - import pw_presubmit -except ImportError: - # Append the pw_presubmit package path to the module search path to allow - # running this module without installing the pw_presubmit package. - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - import pw_presubmit - import pw_package.pigweed_packages from pw_presubmit import ( @@ -43,23 +35,31 @@ from pw_presubmit import ( format_code, git_repo, gitmodules, - call, - filter_paths, inclusive_language, + javascript_checks, + json_check, keep_sorted, module_owners, npm_presubmit, owners_checks, - plural, - presubmit, - PresubmitContext, - PresubmitFailure, - Programs, python_checks, shell_checks, source_in_build, todo_check, ) +from pw_presubmit.presubmit import ( + FileFilter, + Programs, + call, + filter_paths, +) +from pw_presubmit.presubmit_context import ( + FormatOptions, + PresubmitContext, + PresubmitFailure, +) +from pw_presubmit.tools import log_run, plural + from pw_presubmit.install_hook import install_git_hook _LOG = logging.getLogger(__name__) @@ -67,7 +67,7 @@ _LOG = logging.getLogger(__name__) pw_package.pigweed_packages.initialize() # Trigger builds if files with these extensions change. -_BUILD_FILE_FILTER = presubmit.FileFilter( +_BUILD_FILE_FILTER = FileFilter( suffix=( *format_code.C_FORMAT.extensions, '.cfg', @@ -90,17 +90,24 @@ def _at_all_optimization_levels(target): # # Build presubmit checks # +gn_all = build.GnGenNinja( + name='gn_all', + path_filter=_BUILD_FILE_FILTER, + gn_args=dict(pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS), + ninja_targets=('all',), +) + + def gn_clang_build(ctx: PresubmitContext): """Checks all compile targets that rely on LLVM tooling.""" build_targets = [ *_at_all_optimization_levels('host_clang'), - 'cpp14_compatibility', 'cpp20_compatibility', 'asan', 'tsan', 'ubsan', 'runtime_sanitizers', - # TODO(b/234876100): msan will not work until the C++ standard library + # TODO: b/234876100 - msan will not work until the C++ standard library # included in the sysroot has a variant built with msan. ] @@ -110,18 +117,19 @@ def gn_clang_build(ctx: PresubmitContext): # QEMU doesn't run on Windows. if sys.platform != 'win32': - # TODO(b/244604080): For the pw::InlineString tests, qemu_clang_debug + # TODO: b/244604080 - For the pw::InlineString tests, qemu_clang_debug # and qemu_clang_speed_optimized produce a binary too large for the # QEMU target's 256KB flash. Restore debug and speed optimized # builds when this is fixed. build_targets.append('qemu_clang_size_optimized') - # TODO(b/240982565): SocketStream currently requires Linux. + # TODO: b/240982565 - SocketStream currently requires Linux. if sys.platform.startswith('linux'): build_targets.append('integration_tests') build.gn_gen(ctx, pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS) build.ninja(ctx, *build_targets) + build.gn_check(ctx) _HOST_COMPILER = 'gcc' if sys.platform == 'win32' else 'clang' @@ -141,6 +149,7 @@ def gn_full_qemu_check(ctx: PresubmitContext): *_at_all_optimization_levels('qemu_gcc'), *_at_all_optimization_levels('qemu_clang'), ) + build.gn_check(ctx) def _gn_combined_build_check_targets() -> Sequence[str]: @@ -151,13 +160,16 @@ def _gn_combined_build_check_targets() -> Sequence[str]: 'python.tests', 'python.lint', 'docs', - 'fuzzers', 'pigweed_pypi_distribution', ] - # TODO(b/234645359): Re-enable on Windows when compatibility tests build. + # C headers seem to be missing when building with pw_minimal_cpp_stdlib, so + # skip it on Windows. + if sys.platform != 'win32': + build_targets.append('build_with_pw_minimal_cpp_stdlib') + + # TODO: b/234645359 - Re-enable on Windows when compatibility tests build. if sys.platform != 'win32': - build_targets.append('cpp14_compatibility') build_targets.append('cpp20_compatibility') # clang-tidy doesn't run on Windows. @@ -166,18 +178,21 @@ def _gn_combined_build_check_targets() -> Sequence[str]: # QEMU doesn't run on Windows. if sys.platform != 'win32': - build_targets.extend(_at_all_optimization_levels('qemu_gcc')) - - # TODO(b/244604080): For the pw::InlineString tests, qemu_clang_debug - # and qemu_clang_speed_optimized produce a binary too large for the + # TODO: b/244604080 - For the pw::InlineString tests, qemu_*_debug + # and qemu_*_speed_optimized produce a binary too large for the # QEMU target's 256KB flash. Restore debug and speed optimized # builds when this is fixed. + build_targets.append('qemu_gcc_size_optimized') build_targets.append('qemu_clang_size_optimized') - # TODO(b/240982565): SocketStream currently requires Linux. + # TODO: b/240982565 - SocketStream currently requires Linux. if sys.platform.startswith('linux'): build_targets.append('integration_tests') + # TODO: b/269354373 - clang is not supported on windows yet + if sys.platform != 'win32': + build_targets.append('host_clang_debug_dynamic_allocation') + return build_targets @@ -185,15 +200,37 @@ gn_combined_build_check = build.GnGenNinja( name='gn_combined_build_check', doc='Run most host and device (QEMU) tests.', path_filter=_BUILD_FILE_FILTER, - gn_args=dict(pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS), + gn_args=dict( + pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS, + pw_BUILD_BROKEN_GROUPS=True, # Enable to fully test the GN build + ), ninja_targets=_gn_combined_build_check_targets(), ) +coverage = build.GnGenNinja( + name='coverage', + doc='Run coverage for the host build.', + path_filter=_BUILD_FILE_FILTER, + ninja_targets=('coverage',), + coverage_options=build.CoverageOptions( + target_bucket_root='gs://ng3-metrics/ng3-pigweed-coverage', + target_bucket_project='pigweed', + project='codesearch', + trace_type='LLVM', + trim_prefix='/b/s/w/ir/x/w/co', + ref='refs/heads/main', + source='infra:main', + owner='pigweed-infra@google.com', + bug_component='503634', + ), +) + @_BUILD_FILE_FILTER.apply_to_check() def gn_arm_build(ctx: PresubmitContext): build.gn_gen(ctx, pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS) build.ninja(ctx, *_at_all_optimization_levels('stm32f429i')) + build.gn_check(ctx) stm32f429i = build.GnGenNinja( @@ -212,7 +249,6 @@ stm32f429i = build.GnGenNinja( ninja_targets=_at_all_optimization_levels('stm32f429i'), ) - gn_emboss_build = build.GnGenNinja( name='gn_emboss_build', packages=('emboss',), @@ -241,48 +277,57 @@ gn_nanopb_build = build.GnGenNinja( ), ) -gn_crypto_mbedtls_build = build.GnGenNinja( - name='gn_crypto_mbedtls_build', +gn_chre_build = build.GnGenNinja( + name='gn_chre_build', path_filter=_BUILD_FILE_FILTER, - packages=('mbedtls',), - gn_args={ - 'dir_pw_third_party_mbedtls': lambda ctx: '"{}"'.format( - ctx.package_root / 'mbedtls' + packages=('chre',), + gn_args=dict( + dir_pw_third_party_chre=lambda ctx: '"{}"'.format( + ctx.package_root / 'chre' ), - 'pw_crypto_SHA256_BACKEND': lambda ctx: '"{}"'.format( - ctx.root / 'pw_crypto:sha256_mbedtls' + pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS, + ), + ninja_targets=(*_at_all_optimization_levels('host_clang'),), +) + +gn_emboss_nanopb_build = build.GnGenNinja( + name='gn_emboss_nanopb_build', + path_filter=_BUILD_FILE_FILTER, + packages=('emboss', 'nanopb'), + gn_args=dict( + dir_pw_third_party_emboss=lambda ctx: '"{}"'.format( + ctx.package_root / 'emboss' ), - 'pw_crypto_ECDSA_BACKEND': lambda ctx: '"{}"'.format( - ctx.root / 'pw_crypto:ecdsa_mbedtls' + dir_pw_third_party_nanopb=lambda ctx: '"{}"'.format( + ctx.package_root / 'nanopb' ), - 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, - }, + pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS, + ), ninja_targets=( - *_at_all_optimization_levels(f'host_{_HOST_COMPILER}'), - # TODO(b/240982565): SocketStream currently requires Linux. - *(('integration_tests',) if sys.platform.startswith('linux') else ()), + *_at_all_optimization_levels('stm32f429i'), + *_at_all_optimization_levels('host_clang'), ), ) -gn_crypto_boringssl_build = build.GnGenNinja( - name='gn_crypto_boringssl_build', +gn_crypto_mbedtls_build = build.GnGenNinja( + name='gn_crypto_mbedtls_build', path_filter=_BUILD_FILE_FILTER, - packages=('boringssl',), + packages=('mbedtls',), gn_args={ - 'dir_pw_third_party_boringssl': lambda ctx: '"{}"'.format( - ctx.package_root / 'boringssl' + 'dir_pw_third_party_mbedtls': lambda ctx: '"{}"'.format( + ctx.package_root / 'mbedtls' ), 'pw_crypto_SHA256_BACKEND': lambda ctx: '"{}"'.format( - ctx.root / 'pw_crypto:sha256_boringssl' + ctx.root / 'pw_crypto:sha256_mbedtls_v3' ), 'pw_crypto_ECDSA_BACKEND': lambda ctx: '"{}"'.format( - ctx.root / 'pw_crypto:ecdsa_boringssl' + ctx.root / 'pw_crypto:ecdsa_mbedtls_v3' ), 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, }, ninja_targets=( *_at_all_optimization_levels(f'host_{_HOST_COMPILER}'), - # TODO(b/240982565): SocketStream currently requires Linux. + # TODO: b/240982565 - SocketStream currently requires Linux. *(('integration_tests',) if sys.platform.startswith('linux') else ()), ), ) @@ -302,7 +347,7 @@ gn_crypto_micro_ecc_build = build.GnGenNinja( }, ninja_targets=( *_at_all_optimization_levels(f'host_{_HOST_COMPILER}'), - # TODO(b/240982565): SocketStream currently requires Linux. + # TODO: b/240982565 - SocketStream currently requires Linux. *(('integration_tests',) if sys.platform.startswith('linux') else ()), ), ) @@ -316,7 +361,7 @@ gn_teensy_build = build.GnGenNinja( str(ctx.package_root) ), 'pw_arduino_build_CORE_NAME': 'teensy', - 'pw_arduino_build_PACKAGE_NAME': 'teensy/avr', + 'pw_arduino_build_PACKAGE_NAME': 'avr/1.58.1', 'pw_arduino_build_BOARD': 'teensy40', 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, }, @@ -336,6 +381,41 @@ gn_pico_build = build.GnGenNinja( ninja_targets=('pi_pico',), ) +gn_mimxrt595_build = build.GnGenNinja( + name='gn_mimxrt595_build', + path_filter=_BUILD_FILE_FILTER, + packages=('mcuxpresso',), + gn_args={ + 'dir_pw_third_party_mcuxpresso': lambda ctx: '"{}"'.format( + str(ctx.package_root / 'mcuxpresso') + ), + 'pw_target_mimxrt595_evk_MANIFEST': '$dir_pw_third_party_mcuxpresso' + + '/EVK-MIMXRT595_manifest_v3_8.xml', + 'pw_third_party_mcuxpresso_SDK': '//targets/mimxrt595_evk:sample_sdk', + 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, + }, + ninja_targets=('mimxrt595'), +) + +gn_mimxrt595_freertos_build = build.GnGenNinja( + name='gn_mimxrt595_freertos_build', + path_filter=_BUILD_FILE_FILTER, + packages=('freertos', 'mcuxpresso'), + gn_args={ + 'dir_pw_third_party_freertos': lambda ctx: '"{}"'.format( + str(ctx.package_root / 'freertos') + ), + 'dir_pw_third_party_mcuxpresso': lambda ctx: '"{}"'.format( + str(ctx.package_root / 'mcuxpresso') + ), + 'pw_target_mimxrt595_evk_freertos_MANIFEST': '{}/{}'.format( + "$dir_pw_third_party_mcuxpresso", "EVK-MIMXRT595_manifest_v3_8.xml" + ), + 'pw_third_party_mcuxpresso_SDK': '//targets/mimxrt595_evk_freertos:sdk', + 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, + }, + ninja_targets=('mimxrt595_freertos'), +) gn_software_update_build = build.GnGenNinja( name='gn_software_update_build', @@ -358,7 +438,7 @@ gn_software_update_build = build.GnGenNinja( ctx.package_root / 'mbedtls' ), 'pw_crypto_SHA256_BACKEND': lambda ctx: '"{}"'.format( - ctx.root / 'pw_crypto:sha256_mbedtls' + ctx.root / 'pw_crypto:sha256_mbedtls_v3' ), 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, }, @@ -386,7 +466,152 @@ gn_pw_system_demo_build = build.GnGenNinja( ninja_targets=('pw_system_demo',), ) -gn_docs_build = build.GnGenNinja(name='gn_docs_build', ninja_targets=('docs',)) +gn_googletest_build = build.GnGenNinja( + name='gn_googletest_build', + path_filter=_BUILD_FILE_FILTER, + packages=('googletest',), + gn_args={ + 'dir_pw_third_party_googletest': lambda ctx: '"{}"'.format( + ctx.package_root / 'googletest' + ), + 'pw_unit_test_MAIN': lambda ctx: '"{}"'.format( + ctx.root / 'third_party/googletest:gmock_main' + ), + 'pw_unit_test_GOOGLETEST_BACKEND': lambda ctx: '"{}"'.format( + ctx.root / 'third_party/googletest' + ), + 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, + }, + ninja_targets=_at_all_optimization_levels(f'host_{_HOST_COMPILER}'), +) + +gn_fuzz_build = build.GnGenNinja( + name='gn_fuzz_build', + path_filter=_BUILD_FILE_FILTER, + packages=('abseil-cpp', 'fuzztest', 'googletest', 're2'), + gn_args={ + 'dir_pw_third_party_abseil_cpp': lambda ctx: '"{}"'.format( + ctx.package_root / 'abseil-cpp' + ), + 'dir_pw_third_party_fuzztest': lambda ctx: '"{}"'.format( + ctx.package_root / 'fuzztest' + ), + 'dir_pw_third_party_googletest': lambda ctx: '"{}"'.format( + ctx.package_root / 'googletest' + ), + 'dir_pw_third_party_re2': lambda ctx: '"{}"'.format( + ctx.package_root / 're2' + ), + 'pw_unit_test_MAIN': lambda ctx: '"{}"'.format( + ctx.root / 'third_party/googletest:gmock_main' + ), + 'pw_unit_test_GOOGLETEST_BACKEND': lambda ctx: '"{}"'.format( + ctx.root / 'third_party/googletest' + ), + }, + ninja_targets=('fuzzers',), + ninja_contexts=( + lambda ctx: build.modified_env( + FUZZTEST_PRNG_SEED=build.fuzztest_prng_seed(ctx), + ), + ), +) + +oss_fuzz_build = build.GnGenNinja( + name='oss_fuzz_build', + path_filter=_BUILD_FILE_FILTER, + packages=('abseil-cpp', 'fuzztest', 'googletest', 're2'), + gn_args={ + 'dir_pw_third_party_abseil_cpp': lambda ctx: '"{}"'.format( + ctx.package_root / 'abseil-cpp' + ), + 'dir_pw_third_party_fuzztest': lambda ctx: '"{}"'.format( + ctx.package_root / 'fuzztest' + ), + 'dir_pw_third_party_googletest': lambda ctx: '"{}"'.format( + ctx.package_root / 'googletest' + ), + 'dir_pw_third_party_re2': lambda ctx: '"{}"'.format( + ctx.package_root / 're2' + ), + 'pw_toolchain_OSS_FUZZ_ENABLED': True, + }, + ninja_targets=('oss_fuzz',), +) + + +def _env_with_zephyr_vars(ctx: PresubmitContext) -> dict: + """Returns the environment variables with ... set for Zephyr.""" + env = os.environ.copy() + # Set some variables here. + env['ZEPHYR_BASE'] = str(ctx.package_root / 'zephyr') + env['ZEPHYR_MODULES'] = str(ctx.root) + return env + + +def zephyr_build(ctx: PresubmitContext) -> None: + """Run Zephyr compatible tests""" + # Install the Zephyr package + build.install_package(ctx, 'zephyr') + # Configure the environment + env = _env_with_zephyr_vars(ctx) + # Get the python twister runner + twister = ctx.package_root / 'zephyr' / 'scripts' / 'twister' + # Run twister + call( + sys.executable, + twister, + '--ninja', + '--integration', + '--clobber-output', + '--inline-logs', + '--verbose', + '--testsuite-root', + ctx.root / 'pw_unit_test_zephyr', + env=env, + ) + # Produces reports at (ctx.root / 'twister_out' / 'twister*.xml') + + +def docs_build(ctx: PresubmitContext) -> None: + """Build Pigweed docs""" + + # Build main docs through GN/Ninja. + build.install_package(ctx, 'nanopb') + build.gn_gen(ctx, pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS) + build.ninja(ctx, 'docs') + build.gn_check(ctx) + + # Build Rust docs through Bazel. + build.bazel( + ctx, + 'build', + '--', + '//pw_rust:docs', + ) + + # Copy rust docs from Bazel's out directory into where the GN build + # put the main docs. + rust_docs_bazel_dir = ctx.output_dir.joinpath( + '.bazel-bin', 'pw_rust', 'docs.rustdoc' + ) + rust_docs_output_dir = ctx.output_dir.joinpath( + 'docs', 'gen', 'docs', 'html', 'rustdoc' + ) + + # Remove the docs tree to avoid including stale files from previous runs. + shutil.rmtree(rust_docs_output_dir, ignore_errors=True) + + # Bazel generates files and directories without write permissions. In + # order to allow this rule to be run multiple times we use shutil.copyfile + # for the actual copies to not copy permissions of files. + shutil.copytree( + rust_docs_bazel_dir, + rust_docs_output_dir, + copy_function=shutil.copyfile, + dirs_exist_ok=True, + ) + gn_host_tools = build.GnGenNinja( name='gn_host_tools', @@ -418,6 +643,7 @@ def _run_cmake(ctx: PresubmitContext, toolchain='host_clang') -> None: def cmake_clang(ctx: PresubmitContext): _run_cmake(ctx, toolchain='host_clang') build.ninja(ctx, 'pw_apps', 'pw_run_tests.modules') + build.gn_check(ctx) @filter_paths( @@ -426,6 +652,7 @@ def cmake_clang(ctx: PresubmitContext): def cmake_gcc(ctx: PresubmitContext): _run_cmake(ctx, toolchain='host_gcc') build.ninja(ctx, 'pw_apps', 'pw_run_tests.modules') + build.gn_check(ctx) @filter_paths( @@ -497,20 +724,25 @@ def bazel_build(ctx: PresubmitContext) -> None: # This is just a minimal presubmit intended to ensure we don't break what # support we have. # - # TODO(b/271465588): Eventually just build the entire repo for this + # TODO: b/271465588 - Eventually just build the entire repo for this # platform. build.bazel( ctx, 'build', - # Designated initializers produce a warning-treated-as-error when - # compiled with -std=c++17. - # - # TODO(b/271299438): Remove this. - '--copt=-Wno-pedantic', '--platforms=//pw_build/platforms:testonly_freertos', '//pw_sync/...', '//pw_thread/...', '//pw_thread_freertos/...', + '//pw_interrupt/...', + '//pw_cpu_exception/...', + ) + + # Build the pw_system example for the Discovery board using STM32Cube. + build.bazel( + ctx, + 'build', + '--config=stm32f429i', + '//pw_system:system_example', ) @@ -556,7 +788,7 @@ def _clang_system_include_paths(lang: str) -> List[str]: f'{os.devnull}', '-fsyntax-only', ] - process = subprocess.run( + process = log_run( command, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) @@ -597,6 +829,13 @@ _EXCLUDE_FROM_COPYRIGHT_NOTICE: Sequence[str] = ( r'\bDoxyfile$', r'\bPW_PLUGINS$', r'\bconstraint.list$', + r'\bconstraint_hashes_darwin.list$', + r'\bconstraint_hashes_linux.list$', + r'\bconstraint_hashes_windows.list$', + r'\bpython_base_requirements.txt$', + r'\bupstream_requirements_darwin_lock.txt$', + r'\bupstream_requirements_linux_lock.txt$', + r'\bupstream_requirements_windows_lock.txt$', r'^(?:.+/)?\..+$', # keep-sorted: end # Metadata @@ -611,6 +850,7 @@ _EXCLUDE_FROM_COPYRIGHT_NOTICE: Sequence[str] = ( r'\brequirements.txt$', r'\byarn.lock$', r'^docker/tag$', + r'^patches.json$', # keep-sorted: end # Data files # keep-sorted: start @@ -623,6 +863,7 @@ _EXCLUDE_FROM_COPYRIGHT_NOTICE: Sequence[str] = ( r'\.json$', r'\.png$', r'\.svg$', + r'\.vsix$', r'\.xml$', # keep-sorted: end # Documentation @@ -636,6 +877,10 @@ _EXCLUDE_FROM_COPYRIGHT_NOTICE: Sequence[str] = ( r'\.pb\.h$', r'\_pb2.pyi?$', # keep-sorted: end + # Generated third-party files + # keep-sorted: start + r'\bthird_party/.*\.bazelrc$', + # keep-sorted: end # Diff/Patch files # keep-sorted: start r'\.diff$', @@ -674,6 +919,8 @@ _COPYRIGHT = re.compile( _SKIP_LINE_PREFIXES = ( '#!', + '#autoload', + '#compdef', '@echo off', ':<<', '/*', @@ -710,9 +957,12 @@ def copyright_notice(ctx: PresubmitContext): if path.stat().st_size == 0: continue # Skip empty files - with path.open() as file: - if not _COPYRIGHT.match(''.join(_read_notice_lines(file))): - errors.append(path) + try: + with path.open() as file: + if not _COPYRIGHT.match(''.join(_read_notice_lines(file))): + errors.append(path) + except UnicodeDecodeError as exc: + raise PresubmitFailure(f'failed to read {path}') from exc if errors: _LOG.warning( @@ -793,7 +1043,7 @@ def commit_message_format(_: PresubmitContext): if ( 'Reland' in lines[0] and 'This is a reland of ' in git_repo.commit_message() - and "Original change's description: " in git_repo.commit_message() + and "Original change's description:" in git_repo.commit_message() ): _LOG.warning('Ignoring apparent Gerrit-generated reland') return @@ -819,7 +1069,7 @@ def commit_message_format(_: PresubmitContext): # Check that the first line matches the expected pattern. match = re.match( - r'^(?:[\w*/]+(?:{[\w* ,]+})?[\w*/]*|SEED-\d+): (?P<desc>.+)$', lines[0] + r'^(?:[.\w*/]+(?:{[\w* ,]+})?[\w*/]*|SEED-\d+): (?P<desc>.+)$', lines[0] ) if not match: _LOG.warning('The first line does not match the expected format') @@ -886,6 +1136,7 @@ def static_analysis(ctx: PresubmitContext): """Runs all available static analysis tools.""" build.gn_gen(ctx) build.ninja(ctx, 'python.lint', 'static_analysis') + build.gn_check(ctx) _EXCLUDE_FROM_TODO_CHECK = ( @@ -895,6 +1146,7 @@ _EXCLUDE_FROM_TODO_CHECK = ( r'.gitignore$', r'.pylintrc$', r'\bdocs/build_system.rst', + r'\bdocs/code_reviews.rst', r'\bpw_assert_basic/basic_handler.cc', r'\bpw_assert_basic/public/pw_assert_basic/handler.h', r'\bpw_blob_store/public/pw_blob_store/flat_file_system_entry.h', @@ -934,7 +1186,7 @@ def owners_lint_checks(ctx: PresubmitContext): owners_checks.presubmit_check(ctx.paths) -SOURCE_FILES_FILTER = presubmit.FileFilter( +SOURCE_FILES_FILTER = FileFilter( endswith=_BUILD_FILE_FILTER.endswith, suffix=('.bazel', '.bzl', '.gn', '.gni', *_BUILD_FILE_FILTER.suffix), exclude=( @@ -945,42 +1197,55 @@ SOURCE_FILES_FILTER = presubmit.FileFilter( ), ) - # # Presubmit check programs # OTHER_CHECKS = ( # keep-sorted: start - # TODO(b/235277910): Enable all Bazel tests when they're fixed. + # TODO: b/235277910 - Enable all Bazel tests when they're fixed. bazel_test, build.gn_gen_check, cmake_clang, cmake_gcc, - gitmodules.create(), + coverage, + # TODO: b/234876100 - Remove once msan is added to all_sanitizers(). + cpp_checks.msan, + docs_build, + gitmodules.create(gitmodules.Config(allow_submodules=False)), + gn_all, gn_clang_build, gn_combined_build_check, module_owners.presubmit_check(), npm_presubmit.npm_test, pw_transfer_integration_test, + python_checks.update_upstream_python_constraints, + python_checks.vendor_python_wheels, # TODO(hepler): Many files are missing from the CMake build. Add this check # to lintformat when the missing files are fixed. source_in_build.cmake(SOURCE_FILES_FILTER, _run_cmake), static_analysis, stm32f429i, todo_check.create(todo_check.BUGS_OR_USERNAMES), + zephyr_build, # keep-sorted: end ) +ARDUINO_PICO = ( + gn_teensy_build, + gn_pico_build, + gn_pw_system_demo_build, +) + +INTERNAL = (gn_mimxrt595_build, gn_mimxrt595_freertos_build) + # The misc program differs from other_checks in that checks in the misc # program block CQ on Linux. MISC = ( # keep-sorted: start - gn_emboss_build, - gn_nanopb_build, - gn_pico_build, - gn_pw_system_demo_build, - gn_teensy_build, + gn_chre_build, + gn_emboss_nanopb_build, + gn_googletest_build, # keep-sorted: end ) @@ -988,15 +1253,16 @@ SANITIZERS = (cpp_checks.all_sanitizers(),) SECURITY = ( # keep-sorted: start - gn_crypto_boringssl_build, gn_crypto_mbedtls_build, gn_crypto_micro_ecc_build, + gn_fuzz_build, gn_software_update_build, + oss_fuzz_build, # keep-sorted: end ) # Avoid running all checks on specific paths. -PATH_EXCLUSIONS = (re.compile(r'\bthird_party/fuchsia/repo/'),) +PATH_EXCLUSIONS = FormatOptions.load().exclude _LINTFORMAT = ( commit_message_format, @@ -1014,6 +1280,8 @@ _LINTFORMAT = ( source_in_build.gn(SOURCE_FILES_FILTER), source_is_in_cmake_build_warn_only, shell_checks.shellcheck if shutil.which('shellcheck') else (), + javascript_checks.eslint if shutil.which('npx') else (), + json_check.presubmit_check, keep_sorted.presubmit_check, todo_check_with_exceptions, ) @@ -1025,14 +1293,14 @@ LINTFORMAT = ( # (https://stackoverflow.com/q/71024130/1224002). These are cached, but # after a roll it can be quite slow. source_in_build.bazel(SOURCE_FILES_FILTER), - pw_presubmit.python_checks.check_python_versions, - pw_presubmit.python_checks.gn_python_lint, + python_checks.check_python_versions, + python_checks.gn_python_lint, ) QUICK = ( _LINTFORMAT, gn_quick_build_check, - # TODO(b/34884583): Re-enable CMake and Bazel for Mac after we have fixed + # TODO: b/34884583 - Re-enable CMake and Bazel for Mac after we have fixed # the clang issues. The problem is that all clang++ invocations need the # two extra flags: "-nostdc++" and "${clang_prefix}/../lib/libc++.a". cmake_clang if sys.platform != 'darwin' else (), @@ -1042,10 +1310,11 @@ FULL = ( _LINTFORMAT, gn_combined_build_check, gn_host_tools, - bazel_test if sys.platform == 'linux' else (), - bazel_build if sys.platform == 'linux' else (), + bazel_test, + bazel_build, python_checks.gn_python_check, python_checks.gn_python_test_coverage, + python_checks.check_upstream_python_constraints, build_env_setup, # Skip gn_teensy_build if running on Windows. The Teensycore installer is # an exe that requires an admin role. @@ -1054,7 +1323,9 @@ FULL = ( PROGRAMS = Programs( # keep-sorted: start + arduino_pico=ARDUINO_PICO, full=FULL, + internal=INTERNAL, lintformat=LINTFORMAT, misc=MISC, other_checks=OTHER_CHECKS, |