diff options
Diffstat (limited to 'pw_watch/py/pw_watch/watch.py')
| -rwxr-xr-x | pw_watch/py/pw_watch/watch.py | 1120 |
1 files changed, 576 insertions, 544 deletions
diff --git a/pw_watch/py/pw_watch/watch.py b/pw_watch/py/pw_watch/watch.py index fe495efba..d956df3ec 100755 --- a/pw_watch/py/pw_watch/watch.py +++ b/pw_watch/py/pw_watch/watch.py @@ -14,15 +14,15 @@ # the License. """Watch files for changes and rebuild. -pw watch runs Ninja in a build directory when source files change. It works with -any Ninja project (GN or CMake). +Run arbitrary commands or invoke build systems (Ninja, Bazel and make) on one or +more build directories whenever source files change. -Usage examples: +Examples: - # Find a build directory and build the default target - pw watch + # Build the default target in out/ using ninja. + pw watch -C out - # Find a build directory and build the stm32f429i target + # Build python.lint and stm32f429i targets in out/ using ninja. pw watch python.lint stm32f429i # Build pw_run_tests.modules in the out/cmake directory @@ -31,41 +31,60 @@ Usage examples: # Build the default target in out/ and pw_apps in out/cmake pw watch -C out -C out/cmake pw_apps - # Find a directory and build python.tests, and build pw_apps in out/cmake + # Build python.tests in out/ and pw_apps in out/cmake/ pw watch python.tests -C out/cmake pw_apps + + # Run 'bazel build' and 'bazel test' on the target '//...' in outbazel/ + pw watch --run-command 'mkdir -p outbazel' + -C outbazel '//...' + --build-system-command outbazel 'bazel build' + --build-system-command outbazel 'bazel test' """ import argparse -from dataclasses import dataclass +import concurrent.futures import errno -from itertools import zip_longest +import http.server import logging import os from pathlib import Path import re -import shlex import subprocess +import socketserver import sys import threading from threading import Thread from typing import ( + Callable, Iterable, List, - NamedTuple, NoReturn, Optional, Sequence, Tuple, ) -import httpwatcher # type: ignore +try: + import httpwatcher # type: ignore[import] +except ImportError: + httpwatcher = None from watchdog.events import FileSystemEventHandler # type: ignore[import] from watchdog.observers import Observer # type: ignore[import] -from prompt_toolkit.formatted_text.base import OneStyleAndTextTuple -from prompt_toolkit.formatted_text import StyleAndTextTuples - +from prompt_toolkit import prompt + +from pw_build.build_recipe import BuildRecipe, create_build_recipes +from pw_build.project_builder import ( + ProjectBuilder, + execute_command_no_logging, + execute_command_with_logging, + log_build_recipe_start, + log_build_recipe_finish, + ASCII_CHARSET, + EMOJI_CHARSET, +) +from pw_build.project_builder_context import get_project_builder_context import pw_cli.branding import pw_cli.color import pw_cli.env @@ -73,12 +92,16 @@ import pw_cli.log import pw_cli.plugins import pw_console.python_logging -from pw_watch.watch_app import WatchApp +from pw_watch.argparser import ( + WATCH_PATTERN_DELIMITER, + WATCH_PATTERNS, + add_parser_arguments, +) from pw_watch.debounce import DebouncedFunction, Debouncer +from pw_watch.watch_app import WatchAppPrefs, WatchApp _COLOR = pw_cli.color.colors() -_LOG = logging.getLogger('pw_watch') -_NINJA_LOG = logging.getLogger('pw_watch_ninja_output') +_LOG = logging.getLogger('pw_build.watch') _ERRNO_INOTIFY_LIMIT_REACHED = 28 # Suppress events under 'fsevents', generated by watchdog on every file @@ -87,59 +110,9 @@ _ERRNO_INOTIFY_LIMIT_REACHED = 28 _FSEVENTS_LOG = logging.getLogger('fsevents') _FSEVENTS_LOG.setLevel(logging.WARNING) -_PASS_MESSAGE = """ - ██████╗ █████╗ ███████╗███████╗██╗ - ██╔══██╗██╔══██╗██╔════╝██╔════╝██║ - ██████╔╝███████║███████╗███████╗██║ - ██╔═══╝ ██╔══██║╚════██║╚════██║╚═╝ - ██║ ██║ ██║███████║███████║██╗ - ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝ -""" - -# Pick a visually-distinct font from "PASS" to ensure that readers can't -# possibly mistake the difference between the two states. -_FAIL_MESSAGE = """ - ▄██████▒░▄▄▄ ██▓ ░██▓ - ▓█▓ ░▒████▄ ▓██▒ ░▓██▒ - ▒████▒ ░▒█▀ ▀█▄ ▒██▒ ▒██░ - ░▓█▒ ░░██▄▄▄▄██ ░██░ ▒██░ - ░▒█░ ▓█ ▓██▒░██░░ ████████▒ - ▒█░ ▒▒ ▓▒█░░▓ ░ ▒░▓ ░ - ░▒ ▒ ▒▒ ░ ▒ ░░ ░ ▒ ░ - ░ ░ ░ ▒ ▒ ░ ░ ░ - ░ ░ ░ ░ ░ -""" - _FULLSCREEN_STATUS_COLUMN_WIDTH = 10 - -# TODO(keir): Figure out a better strategy for exiting. The problem with the -# watcher is that doing a "clean exit" is slow. However, by directly exiting, -# we remove the possibility of the wrapper script doing anything on exit. -def _die(*args) -> NoReturn: - _LOG.critical(*args) - sys.exit(1) - - -class WatchCharset(NamedTuple): - slug_ok: str - slug_fail: str - - -_ASCII_CHARSET = WatchCharset(_COLOR.green('OK '), _COLOR.red('FAIL')) -_EMOJI_CHARSET = WatchCharset('✔️ ', '💥') - - -@dataclass(frozen=True) -class BuildCommand: - build_dir: Path - targets: Tuple[str, ...] = () - - def args(self) -> Tuple[str, ...]: - return (str(self.build_dir), *self.targets) - - def __str__(self) -> str: - return ' '.join(shlex.quote(arg) for arg in self.args()) +BUILDER_CONTEXT = get_project_builder_context() def git_ignored(file: Path) -> bool: @@ -157,7 +130,8 @@ def git_ignored(file: Path) -> bool: ['git', 'check-ignore', '--quiet', '--no-index', file], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - cwd=directory).returncode + cwd=directory, + ).returncode return returncode in (0, 128) except FileNotFoundError: # If the directory no longer exists, try parent directories until @@ -172,77 +146,82 @@ def git_ignored(file: Path) -> bool: class PigweedBuildWatcher(FileSystemEventHandler, DebouncedFunction): """Process filesystem events and launch builds if necessary.""" + # pylint: disable=too-many-instance-attributes NINJA_BUILD_STEP = re.compile( - r'^\[(?P<step>[0-9]+)/(?P<total_steps>[0-9]+)\] (?P<action>.*)$') + r'^\[(?P<step>[0-9]+)/(?P<total_steps>[0-9]+)\] (?P<action>.*)$' + ) - def __init__( + def __init__( # pylint: disable=too-many-arguments self, - build_commands: Sequence[BuildCommand], + project_builder: ProjectBuilder, patterns: Sequence[str] = (), ignore_patterns: Sequence[str] = (), - charset: WatchCharset = _ASCII_CHARSET, restart: bool = True, - jobs: int = None, fullscreen: bool = False, banners: bool = True, + use_logfile: bool = False, + separate_logfiles: bool = False, + parallel_workers: int = 1, ): super().__init__() self.banners = banners - self.status_message: Optional[OneStyleAndTextTuple] = None - self.result_message: Optional[StyleAndTextTuples] = None - self.current_stdout = '' self.current_build_step = '' self.current_build_percent = 0.0 self.current_build_errors = 0 self.patterns = patterns self.ignore_patterns = ignore_patterns - self.build_commands = build_commands - self.charset: WatchCharset = charset + self.project_builder = project_builder + self.parallel_workers = parallel_workers self.restart_on_changes = restart self.fullscreen_enabled = fullscreen self.watch_app: Optional[WatchApp] = None - self._current_build: subprocess.Popen - self._extra_ninja_args = [] if jobs is None else [f'-j{jobs}'] + self.use_logfile = use_logfile + self.separate_logfiles = separate_logfiles + if self.parallel_workers > 1: + self.separate_logfiles = True self.debouncer = Debouncer(self) # Track state of a build. These need to be members instead of locals # due to the split between dispatch(), run(), and on_complete(). self.matching_path: Optional[Path] = None - self.builds_succeeded: List[bool] = [] - if not self.fullscreen_enabled: + if ( + not self.fullscreen_enabled + and not self.project_builder.should_use_progress_bars() + ): self.wait_for_keypress_thread = threading.Thread( - None, self._wait_for_enter) + None, self._wait_for_enter + ) self.wait_for_keypress_thread.start() + if self.fullscreen_enabled: + BUILDER_CONTEXT.using_fullscreen = True + def rebuild(self): - """ Rebuild command triggered from watch app.""" - self._current_build.terminate() - self._current_build.wait() + """Rebuild command triggered from watch app.""" self.debouncer.press('Manual build requested') - def _wait_for_enter(self) -> NoReturn: + def _wait_for_enter(self) -> None: try: while True: - _ = input() - self._current_build.terminate() - self._current_build.wait() - - self.debouncer.press('Manual build requested...') + _ = prompt('') + self.rebuild() # Ctrl-C on Unix generates KeyboardInterrupt # Ctrl-Z on Windows generates EOFError except (KeyboardInterrupt, EOFError): + # Force stop any running ninja builds. _exit_due_to_interrupt() def _path_matches(self, path: Path) -> bool: """Returns true if path matches according to the watcher patterns""" - return (not any(path.match(x) for x in self.ignore_patterns) - and any(path.match(x) for x in self.patterns)) + return not any(path.match(x) for x in self.ignore_patterns) and any( + path.match(x) for x in self.patterns + ) def dispatch(self, event) -> None: # There isn't any point in triggering builds on new directory creation. @@ -273,15 +252,19 @@ class PigweedBuildWatcher(FileSystemEventHandler, DebouncedFunction): log_message = f'File change detected: {os.path.relpath(matching_path)}' if self.restart_on_changes: if self.fullscreen_enabled and self.watch_app: - self.watch_app.rebuild_on_filechange() + self.watch_app.clear_log_panes() self.debouncer.press(f'{log_message} Triggering build...') else: _LOG.info('%s ; not rebuilding', log_message) def _clear_screen(self) -> None: - if not self.fullscreen_enabled: - print('\033c', end='') # TODO(pwbug/38): Not Windows compatible. - sys.stdout.flush() + if self.fullscreen_enabled: + return + if self.project_builder.should_use_progress_bars(): + BUILDER_CONTEXT.clear_progress_scrollback() + return + print('\033c', end='') # TODO(pwbug/38): Not Windows compatible. + sys.stdout.flush() # Implementation of DebouncedFunction.run() # @@ -289,353 +272,211 @@ class PigweedBuildWatcher(FileSystemEventHandler, DebouncedFunction): # than on the main thread that's watching file events. This enables the # watcher to continue receiving file change events during a build. def run(self) -> None: - """Run all the builds in serial and capture pass/fail for each.""" + """Run all the builds and capture pass/fail for each.""" # Clear the screen and show a banner indicating the build is starting. self._clear_screen() + if self.banners: + for line in pw_cli.branding.banner().splitlines(): + _LOG.info(line) if self.fullscreen_enabled: - self.create_result_message() _LOG.info( - _COLOR.green( - 'Watching for changes. Ctrl-d to exit; enter to rebuild')) + self.project_builder.color.green( + 'Watching for changes. Ctrl-d to exit; enter to rebuild' + ) + ) else: - for line in pw_cli.branding.banner().splitlines(): - _LOG.info(line) _LOG.info( - _COLOR.green( - ' Watching for changes. Ctrl-C to exit; enter to rebuild') + self.project_builder.color.green( + 'Watching for changes. Ctrl-C to exit; enter to rebuild' + ) ) - _LOG.info('') - _LOG.info('Change detected: %s', self.matching_path) + if self.matching_path: + _LOG.info('') + _LOG.info('Change detected: %s', self.matching_path) - self._clear_screen() - - self.builds_succeeded = [] - num_builds = len(self.build_commands) + num_builds = len(self.project_builder) _LOG.info('Starting build with %d directories', num_builds) - env = os.environ.copy() - # Force colors in Pigweed subcommands run through the watcher. - env['PW_USE_COLOR'] = '1' - # Force Ninja to output ANSI colors - env['CLICOLOR_FORCE'] = '1' - - for i, cmd in enumerate(self.build_commands, 1): - index = f'[{i}/{num_builds}]' - self.builds_succeeded.append(self._run_build(index, cmd, env)) - if self.builds_succeeded[-1]: - level = logging.INFO - tag = '(OK)' - else: - level = logging.ERROR - tag = '(FAIL)' - - _LOG.log(level, '%s Finished build: %s %s', index, cmd, tag) - self.create_result_message() + if self.project_builder.default_logfile: + _LOG.info( + '%s %s', + self.project_builder.color.blue('Root logfile:'), + self.project_builder.default_logfile.resolve(), + ) - def create_result_message(self): - if not self.fullscreen_enabled: + env = os.environ.copy() + if self.project_builder.colors: + # Force colors in Pigweed subcommands run through the watcher. + env['PW_USE_COLOR'] = '1' + # Force Ninja to output ANSI colors + env['CLICOLOR_FORCE'] = '1' + + # Reset status + BUILDER_CONTEXT.set_project_builder(self.project_builder) + BUILDER_CONTEXT.set_enter_callback(self.rebuild) + BUILDER_CONTEXT.set_building() + + for cfg in self.project_builder: + cfg.reset_status() + + with concurrent.futures.ThreadPoolExecutor( + max_workers=self.parallel_workers + ) as executor: + futures = [] + if ( + not self.fullscreen_enabled + and self.project_builder.should_use_progress_bars() + ): + BUILDER_CONTEXT.add_progress_bars() + + for i, cfg in enumerate(self.project_builder, start=1): + futures.append(executor.submit(self.run_recipe, i, cfg, env)) + + for future in concurrent.futures.as_completed(futures): + future.result() + + BUILDER_CONTEXT.set_idle() + + def run_recipe(self, index: int, cfg: BuildRecipe, env) -> None: + if BUILDER_CONTEXT.interrupted(): + return + if not cfg.enabled: return - self.result_message = [] - first_building_target_found = False - for (succeeded, command) in zip_longest(self.builds_succeeded, - self.build_commands): - if succeeded: - self.result_message.append( - ('class:theme-fg-green', - 'OK'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH))) - elif succeeded is None and not first_building_target_found: - first_building_target_found = True - self.result_message.append( - ('class:theme-fg-yellow', - 'Building'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH))) - elif first_building_target_found: - self.result_message.append( - ('', ''.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH))) - else: - self.result_message.append( - ('class:theme-fg-red', - 'Failed'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH))) - self.result_message.append(('', f' {command}\n')) - - def _run_build(self, index: str, cmd: BuildCommand, env: dict) -> bool: - # Make sure there is a build.ninja file for Ninja to use. - build_ninja = cmd.build_dir / 'build.ninja' - if not build_ninja.exists(): - # If this is a CMake directory, prompt the user to re-run CMake. - if cmd.build_dir.joinpath('CMakeCache.txt').exists(): - _LOG.error('%s %s does not exist; re-run CMake to generate it', - index, build_ninja) - return False + num_builds = len(self.project_builder) + index_message = f'[{index}/{num_builds}]' - _LOG.warning('%s %s does not exist; running gn gen %s', index, - build_ninja, cmd.build_dir) - if not self._execute_command(['gn', 'gen', cmd.build_dir], env): - return False + log_build_recipe_start( + index_message, self.project_builder, cfg, logger=_LOG + ) - command = ['ninja', *self._extra_ninja_args, '-C', *cmd.args()] - _LOG.info('%s Starting build: %s', index, - ' '.join(shlex.quote(arg) for arg in command)) + self.project_builder.run_build( + cfg, + env, + index_message=index_message, + ) - return self._execute_command(command, env) + log_build_recipe_finish( + index_message, + self.project_builder, + cfg, + logger=_LOG, + ) - def _execute_command(self, command: list, env: dict) -> bool: + def execute_command( + self, + command: list, + env: dict, + recipe: BuildRecipe, + # pylint: disable=unused-argument + *args, + **kwargs, + # pylint: enable=unused-argument + ) -> bool: """Runs a command with a blank before/after for visual separation.""" - self.current_build_errors = 0 - self.status_message = ( - 'class:theme-fg-yellow', - 'Building'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH)) if self.fullscreen_enabled: - return self._execute_command_watch_app(command, env) - print() - self._current_build = subprocess.Popen(command, env=env) - returncode = self._current_build.wait() - print() - return returncode == 0 - - def _execute_command_watch_app(self, command: list, env: dict) -> bool: + return self._execute_command_watch_app(command, env, recipe) + + if self.separate_logfiles: + return execute_command_with_logging( + command, env, recipe, logger=recipe.log + ) + + if self.use_logfile: + return execute_command_with_logging( + command, env, recipe, logger=_LOG + ) + + return execute_command_no_logging(command, env, recipe) + + def _execute_command_watch_app( + self, + command: list, + env: dict, + recipe: BuildRecipe, + ) -> bool: """Runs a command with and outputs the logs.""" if not self.watch_app: return False - self.current_stdout = '' - returncode = None - with subprocess.Popen(command, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - errors='replace') as proc: - self._current_build = proc - - # Empty line at the start. - _NINJA_LOG.info('') - while returncode is None: - if not proc.stdout: - continue - - output = proc.stdout.readline() - self.current_stdout += output - - line_match_result = self.NINJA_BUILD_STEP.match(output) - if line_match_result: - matches = line_match_result.groupdict() - self.current_build_step = line_match_result.group(0) - self.current_build_percent = float( - int(matches.get('step', 0)) / - int(matches.get('total_steps', 1))) - - elif output.startswith(WatchApp.NINJA_FAILURE_TEXT): - _NINJA_LOG.critical(output.strip()) - self.current_build_errors += 1 - - else: - # Mypy output mixes character encoding in its colored output - # due to it's use of the curses module retrieving the 'sgr0' - # (or exit_attribute_mode) capability from the host - # machine's terminfo database. - # - # This can result in this sequence ending up in STDOUT as - # b'\x1b(B\x1b[m'. (B tells terminals to interpret text as - # USASCII encoding but will appear in prompt_toolkit as a B - # character. - # - # The following replace calls will strip out those - # instances. - _NINJA_LOG.info( - output.replace('\x1b(B\x1b[m', - '').replace('\x1b[1m', '').strip()) + + self.watch_app.redraw_ui() + + def new_line_callback(recipe: BuildRecipe) -> None: + self.current_build_step = recipe.status.current_step + self.current_build_percent = recipe.status.percent + self.current_build_errors = recipe.status.error_count + + if self.watch_app: self.watch_app.redraw_ui() - returncode = proc.poll() - # Empty line at the end. - _NINJA_LOG.info('') + desired_logger = _LOG + if self.separate_logfiles: + desired_logger = recipe.log + + result = execute_command_with_logging( + command, + env, + recipe, + logger=desired_logger, + line_processed_callback=new_line_callback, + ) + + self.watch_app.redraw_ui() - return returncode == 0 + return result # Implementation of DebouncedFunction.cancel() def cancel(self) -> bool: if self.restart_on_changes: - self._current_build.terminate() - self._current_build.wait() + BUILDER_CONTEXT.restart_flag = True + BUILDER_CONTEXT.terminate_and_wait() return True return False - # Implementation of DebouncedFunction.run() + # Implementation of DebouncedFunction.on_complete() def on_complete(self, cancelled: bool = False) -> None: # First, use the standard logging facilities to report build status. if cancelled: - self.status_message = ( - '', 'Cancelled'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH)) - _LOG.error('Finished; build was interrupted') - elif all(self.builds_succeeded): - self.status_message = ( - 'class:theme-fg-green', - 'Succeeded'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH)) + _LOG.info('Build stopped.') + elif BUILDER_CONTEXT.interrupted(): + pass # Don't print anything. + elif all( + recipe.status.passed() + for recipe in self.project_builder + if recipe.enabled + ): _LOG.info('Finished; all successful') else: - self.status_message = ( - 'class:theme-fg-red', - 'Failed'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH)) _LOG.info('Finished; some builds failed') - # Show individual build results for fullscreen app - if self.fullscreen_enabled: - self.create_result_message() # For non-fullscreen pw watch - else: + if ( + not self.fullscreen_enabled + and not self.project_builder.should_use_progress_bars() + ): # Show a more distinct colored banner. - if not cancelled: - # Write out build summary table so you can tell which builds - # passed and which builds failed. - _LOG.info('') - _LOG.info(' .------------------------------------') - _LOG.info(' |') - for (succeeded, cmd) in zip(self.builds_succeeded, - self.build_commands): - slug = (self.charset.slug_ok - if succeeded else self.charset.slug_fail) - _LOG.info(' | %s %s', slug, cmd) - _LOG.info(' |') - _LOG.info(" '------------------------------------") - else: - # Build was interrupted. - _LOG.info('') - _LOG.info(' .------------------------------------') - _LOG.info(' |') - _LOG.info(' | %s- interrupted', self.charset.slug_fail) - _LOG.info(' |') - _LOG.info(" '------------------------------------") - - # Show a large color banner for the overall result. - if self.banners: - if all(self.builds_succeeded) and not cancelled: - for line in _PASS_MESSAGE.splitlines(): - _LOG.info(_COLOR.green(line)) - else: - for line in _FAIL_MESSAGE.splitlines(): - _LOG.info(_COLOR.red(line)) + self.project_builder.print_build_summary( + cancelled=cancelled, logger=_LOG + ) + self.project_builder.print_pass_fail_banner( + cancelled=cancelled, logger=_LOG + ) if self.watch_app: self.watch_app.redraw_ui() self.matching_path = None # Implementation of DebouncedFunction.on_keyboard_interrupt() - def on_keyboard_interrupt(self) -> NoReturn: + def on_keyboard_interrupt(self) -> None: _exit_due_to_interrupt() -_WATCH_PATTERN_DELIMITER = ',' -_WATCH_PATTERNS = ( - '*.bloaty', - '*.c', - '*.cc', - '*.css', - '*.cpp', - '*.cmake', - 'CMakeLists.txt', - '*.gn', - '*.gni', - '*.go', - '*.h', - '*.hpp', - '*.ld', - '*.md', - '*.options', - '*.proto', - '*.py', - '*.rst', - '*.s', - '*.S', -) - - -def add_parser_arguments(parser: argparse.ArgumentParser) -> None: - """Sets up an argument parser for pw watch.""" - parser.add_argument('--patterns', - help=(_WATCH_PATTERN_DELIMITER + - '-delimited list of globs to ' - 'watch to trigger recompile'), - default=_WATCH_PATTERN_DELIMITER.join(_WATCH_PATTERNS)) - parser.add_argument('--ignore_patterns', - dest='ignore_patterns_string', - help=(_WATCH_PATTERN_DELIMITER + - '-delimited list of globs to ' - 'ignore events from')) - - parser.add_argument('--exclude_list', - nargs='+', - type=Path, - help='directories to ignore during pw watch', - default=[]) - parser.add_argument('--no-restart', - dest='restart', - action='store_false', - help='do not restart ongoing builds if files change') - parser.add_argument( - 'default_build_targets', - nargs='*', - metavar='target', - default=[], - help=('Automatically locate a build directory and build these ' - 'targets. For example, `host docs` searches for a Ninja ' - 'build directory at out/ and builds the `host` and `docs` ' - 'targets. To specify one or more directories, ust the ' - '-C / --build_directory option.')) - parser.add_argument( - '-C', - '--build_directory', - dest='build_directories', - nargs='+', - action='append', - default=[], - metavar=('directory', 'target'), - help=('Specify a build directory and optionally targets to ' - 'build. `pw watch -C out tgt` is equivalent to `ninja ' - '-C out tgt`')) - parser.add_argument( - '--serve-docs', - dest='serve_docs', - action='store_true', - default=False, - help='Start a webserver for docs on localhost. The port for this ' - ' webserver can be set with the --serve-docs-port option. ' - ' Defaults to http://127.0.0.1:8000') - parser.add_argument( - '--serve-docs-port', - dest='serve_docs_port', - type=int, - default=8000, - help='Set the port for the docs webserver. Default to 8000.') - - parser.add_argument( - '--serve-docs-path', - dest='serve_docs_path', - type=Path, - default="docs/gen/docs", - help='Set the path for the docs to serve. Default to docs/gen/docs' - ' in the build directory.') - parser.add_argument( - '-j', - '--jobs', - type=int, - help="Number of cores to use; defaults to Ninja's default") - parser.add_argument('-f', - '--fullscreen', - action='store_true', - default=False, - help='Use a fullscreen interface.') - parser.add_argument('--debug-logging', - action='store_true', - help='Enable debug logging.') - parser.add_argument('--no-banners', - dest='banners', - action='store_false', - help='Hide pass/fail banners.') - - def _exit(code: int) -> NoReturn: + # Flush all log handlers + logging.shutdown() # Note: The "proper" way to exit is via observer.stop(), then # running a join. However it's slower, so just exit immediately. # @@ -645,46 +486,77 @@ def _exit(code: int) -> NoReturn: os._exit(code) # pylint: disable=protected-access -def _exit_due_to_interrupt() -> NoReturn: +def _exit_due_to_interrupt() -> None: # To keep the log lines aligned with each other in the presence of # a '^C' from the keyboard interrupt, add a newline before the log. + print('') _LOG.info('Got Ctrl-C; exiting...') - _exit(0) + BUILDER_CONTEXT.ctrl_c_interrupt() -def _exit_due_to_inotify_watch_limit(): +def _log_inotify_watch_limit_reached(): # Show information and suggested commands in OSError: inotify limit reached. - _LOG.error('Inotify watch limit reached: run this in your terminal if ' - 'you are in Linux to temporarily increase inotify limit. \n') + _LOG.error( + 'Inotify watch limit reached: run this in your terminal if ' + 'you are in Linux to temporarily increase inotify limit.' + ) + _LOG.info('') _LOG.info( - _COLOR.green(' sudo sysctl fs.inotify.max_user_watches=' - '$NEW_LIMIT$\n')) - _LOG.info(' Change $NEW_LIMIT$ with an integer number, ' - 'e.g., 20000 should be enough.') - _exit(0) + _COLOR.green( + ' sudo sysctl fs.inotify.max_user_watches=' '$NEW_LIMIT$' + ) + ) + _LOG.info('') + _LOG.info( + ' Change $NEW_LIMIT$ with an integer number, ' + 'e.g., 20000 should be enough.' + ) -def _exit_due_to_inotify_instance_limit(): +def _exit_due_to_inotify_watch_limit(): + _log_inotify_watch_limit_reached() + _exit(1) + + +def _log_inotify_instance_limit_reached(): # Show information and suggested commands in OSError: inotify limit reached. - _LOG.error('Inotify instance limit reached: run this in your terminal if ' - 'you are in Linux to temporarily increase inotify limit. \n') + _LOG.error( + 'Inotify instance limit reached: run this in your terminal if ' + 'you are in Linux to temporarily increase inotify limit.' + ) + _LOG.info('') + _LOG.info( + _COLOR.green( + ' sudo sysctl fs.inotify.max_user_instances=' '$NEW_LIMIT$' + ) + ) + _LOG.info('') _LOG.info( - _COLOR.green(' sudo sysctl fs.inotify.max_user_instances=' - '$NEW_LIMIT$\n')) - _LOG.info(' Change $NEW_LIMIT$ with an integer number, ' - 'e.g., 20000 should be enough.') - _exit(0) + ' Change $NEW_LIMIT$ with an integer number, ' + 'e.g., 20000 should be enough.' + ) + + +def _exit_due_to_inotify_instance_limit(): + _log_inotify_instance_limit_reached() + _exit(1) def _exit_due_to_pigweed_not_installed(): # Show information and suggested commands when pigweed environment variable # not found. - _LOG.error('Environment variable $PW_ROOT not defined or is defined ' - 'outside the current directory.') - _LOG.error('Did you forget to activate the Pigweed environment? ' - 'Try source ./activate.sh') - _LOG.error('Did you forget to install the Pigweed environment? ' - 'Try source ./bootstrap.sh') + _LOG.error( + 'Environment variable $PW_ROOT not defined or is defined ' + 'outside the current directory.' + ) + _LOG.error( + 'Did you forget to activate the Pigweed environment? ' + 'Try source ./activate.sh' + ) + _LOG.error( + 'Did you forget to install the Pigweed environment? ' + 'Try source ./bootstrap.sh' + ) _exit(1) @@ -712,8 +584,9 @@ def minimal_watch_directories(to_watch: Path, to_exclude: Iterable[Path]): # and generate all parent paths needed to be watched without recursion. exclude_dir_parents = {to_watch} for directory_to_exclude in directories_to_exclude: - parts = list( - Path(directory_to_exclude).relative_to(to_watch).parts)[:-1] + parts = list(Path(directory_to_exclude).relative_to(to_watch).parts)[ + :-1 + ] dir_tmp = to_watch for part in parts: dir_tmp = Path(dir_tmp, part) @@ -726,8 +599,11 @@ def minimal_watch_directories(to_watch: Path, to_exclude: Iterable[Path]): dir_path = Path(directory) yield dir_path, False for item in Path(directory).iterdir(): - if (item.is_dir() and item not in exclude_dir_parents - and item not in directories_to_exclude): + if ( + item.is_dir() + and item not in exclude_dir_parents + and item not in directories_to_exclude + ): yield item, True @@ -747,8 +623,10 @@ def get_common_excludes() -> List[Path]: # Preset exclude list for Pigweed's upstream directories. pw_root_dir = Path(os.environ['PW_ROOT']) - exclude_list.extend(pw_root_dir / ignored_directory - for ignored_directory in typical_ignored_directories) + exclude_list.extend( + pw_root_dir / ignored_directory + for ignored_directory in typical_ignored_directories + ) # Preset exclude for common downstream project structures. # @@ -758,7 +636,8 @@ def get_common_excludes() -> List[Path]: if pw_project_root_dir != pw_root_dir: exclude_list.extend( pw_project_root_dir / ignored_directory - for ignored_directory in typical_ignored_directories) + for ignored_directory in typical_ignored_directories + ) # Check for and warn about legacy directories. legacy_directories = [ @@ -769,33 +648,141 @@ def get_common_excludes() -> List[Path]: for legacy_directory in legacy_directories: full_legacy_directory = pw_root_dir / legacy_directory if full_legacy_directory.is_dir(): - _LOG.warning('Legacy environment directory found: %s', - str(full_legacy_directory)) + _LOG.warning( + 'Legacy environment directory found: %s', + str(full_legacy_directory), + ) exclude_list.append(full_legacy_directory) found_legacy = True if found_legacy: - _LOG.warning('Found legacy environment directory(s); these ' - 'should be deleted') + _LOG.warning( + 'Found legacy environment directory(s); these ' 'should be deleted' + ) return exclude_list -def watch_setup( - default_build_targets: List[str], - build_directories: List[str], - patterns: str, - ignore_patterns_string: str, - exclude_list: List[Path], - restart: bool, - jobs: Optional[int], - serve_docs: bool, - serve_docs_port: int, - serve_docs_path: Path, - fullscreen: bool, - banners: bool, +def _simple_docs_server( + address: str, port: int, path: Path +) -> Callable[[], None]: + class Handler(http.server.SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=path, **kwargs) + + # Disable logs to stdout + def log_message( + self, format: str, *args # pylint: disable=redefined-builtin + ) -> None: + return + + def simple_http_server_thread(): + with socketserver.TCPServer((address, port), Handler) as httpd: + httpd.serve_forever() + + return simple_http_server_thread + + +def _httpwatcher_docs_server( + address: str, port: int, path: Path +) -> Callable[[], None]: + def httpwatcher_thread(): + # Disable logs from httpwatcher and deps + logging.getLogger('httpwatcher').setLevel(logging.CRITICAL) + logging.getLogger('tornado').setLevel(logging.CRITICAL) + + httpwatcher.watch(path, host=address, port=port) + + return httpwatcher_thread + + +def _serve_docs( + build_dir: Path, + docs_path: Path, + address: str = '127.0.0.1', + port: int = 8000, +) -> None: + address = '127.0.0.1' + docs_path = build_dir.joinpath(docs_path.joinpath('html')) + + if httpwatcher is not None: + _LOG.info('Using httpwatcher. Docs will reload when changed.') + server_thread = _httpwatcher_docs_server(address, port, docs_path) + else: + _LOG.info( + 'Using simple HTTP server. Docs will not reload when changed.' + ) + _LOG.info('Install httpwatcher and restart for automatic docs reload.') + server_thread = _simple_docs_server(address, port, docs_path) + + # Spin up server in a new thread since it blocks + threading.Thread(None, server_thread, 'pw_docs_server').start() + + +def watch_logging_init(log_level: int, fullscreen: bool, colors: bool) -> None: + # Logging setup + if not fullscreen: + pw_cli.log.install( + level=log_level, + use_color=colors, + hide_timestamp=False, + ) + return + + watch_logfile = pw_console.python_logging.create_temp_log_file( + prefix=__package__ + ) + + pw_cli.log.install( + level=logging.DEBUG, + use_color=colors, + hide_timestamp=False, + log_file=watch_logfile, + ) + + +def watch_setup( # pylint: disable=too-many-locals + project_builder: ProjectBuilder, + # NOTE: The following args should have defaults matching argparse. This + # allows use of watch_setup by other project build scripts. + patterns: str = WATCH_PATTERN_DELIMITER.join(WATCH_PATTERNS), + ignore_patterns_string: str = '', + exclude_list: Optional[List[Path]] = None, + restart: bool = True, + serve_docs: bool = False, + serve_docs_port: int = 8000, + serve_docs_path: Path = Path('docs/gen/docs'), + fullscreen: bool = False, + banners: bool = True, + logfile: Optional[Path] = None, + separate_logfiles: bool = False, + parallel: bool = False, + parallel_workers: int = 0, + # pylint: disable=unused-argument + default_build_targets: Optional[List[str]] = None, + build_directories: Optional[List[str]] = None, + build_system_commands: Optional[List[str]] = None, + run_command: Optional[List[str]] = None, + jobs: Optional[int] = None, + keep_going: bool = False, + colors: bool = True, + debug_logging: bool = False, + # pylint: enable=unused-argument # pylint: disable=too-many-arguments -) -> Tuple[str, PigweedBuildWatcher, List[Path]]: +) -> Tuple[PigweedBuildWatcher, List[Path]]: """Watches files and runs Ninja commands when they change.""" + watch_logging_init( + log_level=project_builder.default_log_level, + fullscreen=fullscreen, + colors=colors, + ) + + # Update the project_builder log formatters since pw_cli.log.install may + # have changed it. + project_builder.apply_root_log_formatting() + + if project_builder.should_use_progress_bars(): + project_builder.use_stdout_proxy() + _LOG.info('Starting Pigweed build watcher') # Get pigweed directory information from environment variable PW_ROOT. @@ -805,83 +792,81 @@ def watch_setup( if Path.cwd().resolve() not in [pw_root, *pw_root.parents]: _exit_due_to_pigweed_not_installed() + build_recipes = project_builder.build_recipes + # Preset exclude list for pigweed directory. + if not exclude_list: + exclude_list = [] exclude_list += get_common_excludes() # Add build directories to the exclude list. exclude_list.extend( - Path(build_dir[0]).resolve() for build_dir in build_directories) - - build_commands = [ - BuildCommand(Path(build_dir[0]), tuple(build_dir[1:])) - for build_dir in build_directories - ] - - # If no build directory was specified, check for out/build.ninja. - if default_build_targets or not build_directories: - # Make sure we found something; if not, bail. - if not Path('out').exists(): - _die("No build dirs found. Did you forget to run 'gn gen out'?") - - build_commands.append( - BuildCommand(Path('out'), tuple(default_build_targets))) + cfg.build_dir.resolve() + for cfg in build_recipes + if isinstance(cfg.build_dir, Path) + ) - # Verify that the build output directories exist. - for i, build_target in enumerate(build_commands, 1): - if not build_target.build_dir.is_dir(): - _die("Build directory doesn't exist: %s", build_target) - else: - _LOG.info('Will build [%d/%d]: %s', i, len(build_commands), - build_target) + for i, build_recipe in enumerate(build_recipes, start=1): + _LOG.info('Will build [%d/%d]: %s', i, len(build_recipes), build_recipe) _LOG.debug('Patterns: %s', patterns) if serve_docs: - - def _serve_docs(): - # Disable logs from httpwatcher and deps - logging.getLogger('httpwatcher').setLevel(logging.CRITICAL) - logging.getLogger('tornado').setLevel(logging.CRITICAL) - - docs_path = build_commands[0].build_dir.joinpath( - serve_docs_path.joinpath('html')) - httpwatcher.watch(docs_path, - host="127.0.0.1", - port=serve_docs_port) - - # Spin up an httpwatcher in a new thread since it blocks - threading.Thread(None, _serve_docs, "httpwatcher").start() - - # Try to make a short display path for the watched directory that has - # "$HOME" instead of the full home directory. This is nice for users - # who have deeply nested home directories. - path_to_log = str(Path().resolve()).replace(str(Path.home()), '$HOME') + _serve_docs( + build_recipes[0].build_dir, serve_docs_path, port=serve_docs_port + ) # Ignore the user-specified patterns. - ignore_patterns = (ignore_patterns_string.split(_WATCH_PATTERN_DELIMITER) - if ignore_patterns_string else []) + ignore_patterns = ( + ignore_patterns_string.split(WATCH_PATTERN_DELIMITER) + if ignore_patterns_string + else [] + ) - env = pw_cli.env.pigweed_environment() - if env.PW_EMOJI: - charset = _EMOJI_CHARSET - else: - charset = _ASCII_CHARSET + # Add project_builder logfiles to ignore_patterns + if project_builder.default_logfile: + ignore_patterns.append(str(project_builder.default_logfile)) + if project_builder.separate_build_file_logging: + for recipe in project_builder: + if recipe.logfile: + ignore_patterns.append(str(recipe.logfile)) + + workers = 1 + if parallel: + # If parallel is requested and parallel_workers is set to 0 run all + # recipes in parallel. That is, use the number of recipes as the worker + # count. + if parallel_workers == 0: + workers = len(project_builder) + else: + workers = parallel_workers event_handler = PigweedBuildWatcher( - build_commands=build_commands, - patterns=patterns.split(_WATCH_PATTERN_DELIMITER), + project_builder=project_builder, + patterns=patterns.split(WATCH_PATTERN_DELIMITER), ignore_patterns=ignore_patterns, - charset=charset, restart=restart, - jobs=jobs, fullscreen=fullscreen, banners=banners, + use_logfile=bool(logfile), + separate_logfiles=separate_logfiles, + parallel_workers=workers, ) - return path_to_log, event_handler, exclude_list + project_builder.execute_command = event_handler.execute_command + + return event_handler, exclude_list -def watch(path_to_log: Path, event_handler: PigweedBuildWatcher, - exclude_list: List[Path]): + +def watch( + event_handler: PigweedBuildWatcher, + exclude_list: List[Path], +): """Watches files and runs Ninja commands when they change.""" + # Try to make a short display path for the watched directory that has + # "$HOME" instead of the full home directory. This is nice for users + # who have deeply nested home directories. + path_to_log = str(Path().resolve()).replace(str(Path.home()), '$HOME') + try: # It can take awhile to configure the filesystem watcher, so have the # message reflect that with the "...". Run inside the try: to @@ -907,6 +892,7 @@ def watch(path_to_log: Path, event_handler: PigweedBuildWatcher, for observer in observers: while observer.is_alive(): observer.join(1) + _LOG.error('observers joined') # Ctrl-C on Unix generates KeyboardInterrupt # Ctrl-Z on Windows generates EOFError @@ -914,67 +900,113 @@ def watch(path_to_log: Path, event_handler: PigweedBuildWatcher, _exit_due_to_interrupt() except OSError as err: if err.args[0] == _ERRNO_INOTIFY_LIMIT_REACHED: - _exit_due_to_inotify_watch_limit() + if event_handler.watch_app: + event_handler.watch_app.exit( + log_after_shutdown=_log_inotify_watch_limit_reached + ) + elif event_handler.project_builder.should_use_progress_bars(): + BUILDER_CONTEXT.exit( + log_after_shutdown=_log_inotify_watch_limit_reached, + ) + else: + _exit_due_to_inotify_watch_limit() if err.errno == errno.EMFILE: - _exit_due_to_inotify_instance_limit() + if event_handler.watch_app: + event_handler.watch_app.exit( + log_after_shutdown=_log_inotify_instance_limit_reached + ) + elif event_handler.project_builder.should_use_progress_bars(): + BUILDER_CONTEXT.exit( + log_after_shutdown=_log_inotify_instance_limit_reached + ) + else: + _exit_due_to_inotify_instance_limit() raise err - _LOG.critical('Should never get here') - observer.join() +def run_watch( + event_handler: PigweedBuildWatcher, + exclude_list: List[Path], + prefs: Optional[WatchAppPrefs] = None, + fullscreen: bool = False, +) -> None: + """Start pw_watch.""" + if not prefs: + prefs = WatchAppPrefs(load_argparse_arguments=add_parser_arguments) + + if fullscreen: + watch_thread = Thread( + target=watch, + args=(event_handler, exclude_list), + daemon=True, + ) + watch_thread.start() + watch_app = WatchApp( + event_handler=event_handler, + prefs=prefs, + ) -def main() -> None: - """Watch files for changes and rebuild.""" + event_handler.watch_app = watch_app + watch_app.run() + + else: + watch(event_handler, exclude_list) + + +def get_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - add_parser_arguments(parser) + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser = add_parser_arguments(parser) + return parser + + +def main() -> None: + """Watch files for changes and rebuild.""" + parser = get_parser() args = parser.parse_args() - path_to_log, event_handler, exclude_list = watch_setup( - default_build_targets=args.default_build_targets, - build_directories=args.build_directories, - patterns=args.patterns, - ignore_patterns_string=args.ignore_patterns_string, - exclude_list=args.exclude_list, - restart=args.restart, + prefs = WatchAppPrefs(load_argparse_arguments=add_parser_arguments) + prefs.apply_command_line_args(args) + build_recipes = create_build_recipes(prefs) + + env = pw_cli.env.pigweed_environment() + if env.PW_EMOJI: + charset = EMOJI_CHARSET + else: + charset = ASCII_CHARSET + + # Force separate-logfiles for split window panes if running in parallel. + separate_logfiles = args.separate_logfiles + if args.parallel: + separate_logfiles = True + + def _recipe_abort(*args) -> None: + _LOG.critical(*args) + + project_builder = ProjectBuilder( + build_recipes=build_recipes, jobs=args.jobs, - serve_docs=args.serve_docs, - serve_docs_port=args.serve_docs_port, - serve_docs_path=args.serve_docs_path, - fullscreen=args.fullscreen, banners=args.banners, + keep_going=args.keep_going, + colors=args.colors, + charset=charset, + separate_build_file_logging=separate_logfiles, + root_logfile=args.logfile, + root_logger=_LOG, + log_level=logging.DEBUG if args.debug_logging else logging.INFO, + abort_callback=_recipe_abort, ) - if args.fullscreen: - watch_logfile = (pw_console.python_logging.create_temp_log_file( - prefix=__package__)) - pw_cli.log.install( - level=logging.DEBUG, - use_color=True, - hide_timestamp=False, - log_file=watch_logfile, - ) - pw_console.python_logging.setup_python_logging( - last_resort_filename=watch_logfile) - - watch_thread = Thread(target=watch, - args=(path_to_log, event_handler, exclude_list), - daemon=True) - watch_thread.start() - watch_app = WatchApp(event_handler=event_handler, - debug_logging=args.debug_logging, - log_file_name=watch_logfile) + event_handler, exclude_list = watch_setup(project_builder, **vars(args)) - event_handler.watch_app = watch_app - watch_app.run() - else: - pw_cli.log.install( - level=logging.DEBUG if args.debug_logging else logging.INFO, - use_color=True, - hide_timestamp=False, - ) - watch(Path(path_to_log), event_handler, exclude_list) + run_watch( + event_handler, + exclude_list, + prefs=prefs, + fullscreen=args.fullscreen, + ) if __name__ == '__main__': |