diff options
Diffstat (limited to 'llvm_tools/patch_utils.py')
| -rw-r--r-- | llvm_tools/patch_utils.py | 203 |
1 files changed, 133 insertions, 70 deletions
diff --git a/llvm_tools/patch_utils.py b/llvm_tools/patch_utils.py index 0c8ad19c..b21cf314 100644 --- a/llvm_tools/patch_utils.py +++ b/llvm_tools/patch_utils.py @@ -12,11 +12,22 @@ from pathlib import Path import re import subprocess import sys -from typing import Any, Dict, IO, Iterable, List, Optional, Tuple +from typing import ( + Any, + Callable, + Dict, + IO, + Iterable, + List, + Optional, + Tuple, + Union, +) import atomic_write_file +APPLIED_RE = re.compile(r"^Applying: (.+) \(#(\d+)\)$") CHECKED_FILE_RE = re.compile(r"^checking file\s+(.*)$") HUNK_FAILED_RE = re.compile(r"^Hunk #(\d+) FAILED at.*") HUNK_HEADER_RE = re.compile(r"^@@\s+-(\d+),(\d+)\s+\+(\d+),(\d+)\s+@@") @@ -42,11 +53,11 @@ def parse_patch_stream(patch_stream: IO[str]) -> Dict[str, List[Hunk]]: """Parse a patch file-like into Hunks. Args: - patch_stream: A IO stream formatted like a git patch file. + patch_stream: A IO stream formatted like a git patch file. Returns: - A dictionary mapping filenames to lists of Hunks present - in the patch stream. + A dictionary mapping filenames to lists of Hunks present + in the patch stream. """ current_filepath = None @@ -102,6 +113,12 @@ def parse_failed_patch_output(text: str) -> Dict[str, List[int]]: raise ValueError("Input stream was not parsable") hunk_id = int(failed_match.group(1)) failed_hunks[current_file].append(hunk_id) + else: + failed_applied_patches = APPLIED_RE.match(eline) + if failed_applied_patches: + current_file = failed_applied_patches.group(1) + hunk_id = int(failed_applied_patches.group(2)) + failed_hunks[current_file].append(hunk_id) return failed_hunks @@ -124,7 +141,10 @@ class PatchResult: for file, hunks in self.failed_hunks.items(): s += f"{file}:\n" for h in hunks: - s += f"Lines {h.orig_start} to {h.orig_start + h.orig_hunk_len}\n" + s += ( + f"Lines {h.orig_start} to " + f"{h.orig_start + h.orig_hunk_len}\n" + ) s += "--------------------\n" return s @@ -139,10 +159,12 @@ class PatchEntry: platforms: Optional[List[str]] rel_patch_path: str version_range: Optional[Dict[str, Optional[int]]] + verify_workdir: bool = True + """Don't verify the workdir exists. Used for testing.""" _parsed_hunks = None def __post_init__(self): - if not self.workdir.is_dir(): + if self.verify_workdir and not self.workdir.is_dir(): raise ValueError(f"workdir {self.workdir} is not a directory") @classmethod @@ -150,13 +172,12 @@ class PatchEntry: """Instatiate from a dictionary. Dictionary must have at least the following key: - - { + { 'rel_patch_path': '<relative patch path to workdir>', - } + } Returns: - A new PatchEntry. + A new PatchEntry. """ return cls( workdir, @@ -167,8 +188,16 @@ class PatchEntry: ) def to_dict(self) -> Dict[str, Any]: + # We sort the metadata so that it doesn't matter + # how it was passed to patch_utils. + if self.metadata is None: + sorted_metadata = None + else: + sorted_metadata = dict( + sorted(self.metadata.items(), key=lambda x: x[0]) + ) out: Dict[str, Any] = { - "metadata": self.metadata, + "metadata": sorted_metadata, } if self.platforms: # To match patch_sync, only serialized when @@ -214,28 +243,27 @@ class PatchEntry: return svn_version >= until_v def apply( - self, root_dir: Path, extra_args: Optional[List[str]] = None + self, + root_dir: Path, + patch_cmd: Optional[Callable] = None, + extra_args: Optional[List[str]] = None, ) -> PatchResult: """Apply a patch to a given directory.""" - if not extra_args: - extra_args = [] # Cmd to apply a patch in the src unpack path. abs_patch_path = self.patch_path().absolute() if not abs_patch_path.is_file(): raise RuntimeError( f"Cannot apply: patch {abs_patch_path} is not a file" ) - cmd = [ - "patch", - "-d", - root_dir.absolute(), - "-f", - "-E", - "-p1", - "--no-backup-if-mismatch", - "-i", - abs_patch_path, - ] + extra_args + + if not patch_cmd: + patch_cmd = gnu_patch + + if patch_cmd == gnu_patch: + cmd = patch_cmd(root_dir, abs_patch_path) + (extra_args or []) + else: + cmd = patch_cmd(abs_patch_path) + (extra_args or []) + try: subprocess.run( cmd, encoding="utf-8", check=True, stdout=subprocess.PIPE @@ -244,19 +272,27 @@ class PatchEntry: parsed_hunks = self.parsed_hunks() failed_hunks_id_dict = parse_failed_patch_output(e.stdout) failed_hunks = {} - for path, failed_hunk_ids in failed_hunks_id_dict.items(): - hunks_for_file = parsed_hunks[path] - failed_hunks[path] = [ - hunk - for hunk in hunks_for_file - if hunk.hunk_id in failed_hunk_ids - ] + if patch_cmd == gnu_patch: + for path, failed_hunk_ids in failed_hunks_id_dict.items(): + hunks_for_file = parsed_hunks[path] + failed_hunks[path] = [ + hunk + for hunk in hunks_for_file + if hunk.hunk_id in failed_hunk_ids + ] + elif failed_hunks_id_dict: + # use git am + failed_hunks = parsed_hunks + return PatchResult(succeeded=False, failed_hunks=failed_hunks) return PatchResult(succeeded=True) - def test_apply(self, root_dir: Path) -> PatchResult: + def test_apply( + self, root_dir: Path, patch_cmd: Optional[Callable] = None + ) -> PatchResult: """Dry run applying a patch to a given directory.""" - return self.apply(root_dir, ["--dry-run"]) + extra_args = [] if patch_cmd == git_am else ["--dry-run"] + return self.apply(root_dir, patch_cmd, extra_args) def title(self) -> str: if not self.metadata: @@ -273,7 +309,7 @@ class PatchInfo: applied_patches: List[PatchEntry] failed_patches: List[PatchEntry] # Can be deleted once legacy code is removed. - non_applicable_patches: List[str] + non_applicable_patches: List[PatchEntry] # Can be deleted once legacy code is removed. disabled_patches: List[str] # Can be deleted once legacy code is removed. @@ -289,8 +325,8 @@ def json_to_patch_entries(workdir: Path, json_fd: IO[str]) -> List[PatchEntry]: """Convert a json IO object to List[PatchEntry]. Examples: - >>> f = open('PATCHES.json') - >>> patch_entries = json_to_patch_entries(Path(), f) + >>> f = open('PATCHES.json') + >>> patch_entries = json_to_patch_entries(Path(), f) """ return [PatchEntry.from_dict(workdir, d) for d in json.load(json_fd)] @@ -299,8 +335,8 @@ def json_str_to_patch_entries(workdir: Path, json_str: str) -> List[PatchEntry]: """Convert a json IO object to List[PatchEntry]. Examples: - >>> f = open('PATCHES.json').read() - >>> patch_entries = json_str_to_patch_entries(Path(), f) + >>> f = open('PATCHES.json').read() + >>> patch_entries = json_str_to_patch_entries(Path(), f) """ return [PatchEntry.from_dict(workdir, d) for d in json.loads(json_str)] @@ -309,9 +345,9 @@ def _print_failed_patch(pe: PatchEntry, failed_hunks: Dict[str, List[Hunk]]): """Print information about a single failing PatchEntry. Args: - pe: A PatchEntry that failed. - failed_hunks: Hunks for pe which failed as dict: - filepath: [Hunk...] + pe: A PatchEntry that failed. + failed_hunks: Hunks for pe which failed as dict: + filepath: [Hunk...] """ print(f"Could not apply {pe.rel_patch_path}: {pe.title()}", file=sys.stderr) for fp, hunks in failed_hunks.items(): @@ -328,6 +364,7 @@ def apply_all_from_json( svn_version: int, llvm_src_dir: Path, patches_json_fp: Path, + patch_cmd: Optional[Callable] = None, continue_on_failure: bool = False, ) -> PatchInfo: """Attempt to apply some patches to a given LLVM source tree. @@ -336,11 +373,11 @@ def apply_all_from_json( the patches are applied. Args: - svn_version: LLVM Subversion revision to patch. - llvm_src_dir: llvm-project root-level source directory to patch. - patches_json_fp: Filepath to the PATCHES.json file. - continue_on_failure: Skip any patches which failed to apply, - rather than throw an Exception. + svn_version: LLVM Subversion revision to patch. + llvm_src_dir: llvm-project root-level source directory to patch. + patches_json_fp: Filepath to the PATCHES.json file. + continue_on_failure: Skip any patches which failed to apply, + rather than throw an Exception. """ with patches_json_fp.open(encoding="utf-8") as f: patches = json_to_patch_entries(patches_json_fp.parent, f) @@ -349,7 +386,7 @@ def apply_all_from_json( applied_patches = [] for pe in patches: applied, failed_hunks = apply_single_patch_entry( - svn_version, llvm_src_dir, pe + svn_version, llvm_src_dir, pe, patch_cmd ) if applied: applied_patches.append(pe) @@ -379,24 +416,25 @@ def apply_single_patch_entry( svn_version: int, llvm_src_dir: Path, pe: PatchEntry, + patch_cmd: Optional[Callable] = None, ignore_version_range: bool = False, ) -> Tuple[bool, Optional[Dict[str, List[Hunk]]]]: """Try to apply a single PatchEntry object. Returns: - Tuple where the first element indicates whether the patch applied, - and the second element is a faild hunk mapping from file name to lists of - hunks (if the patch didn't apply). + Tuple where the first element indicates whether the patch applied, and + the second element is a faild hunk mapping from file name to lists of + hunks (if the patch didn't apply). """ # Don't apply patches outside of the version range. if not ignore_version_range and not pe.can_patch_version(svn_version): return False, None # Test first to avoid making changes. - test_application = pe.test_apply(llvm_src_dir) + test_application = pe.test_apply(llvm_src_dir, patch_cmd) if not test_application: return False, test_application.failed_hunks # Now actually make changes. - application_result = pe.apply(llvm_src_dir) + application_result = pe.apply(llvm_src_dir, patch_cmd) if not application_result: # This should be very rare/impossible. return False, application_result.failed_hunks @@ -465,7 +503,10 @@ def predict_indent(patches_lines: List[str]) -> int: def update_version_ranges( - svn_version: int, llvm_src_dir: Path, patches_json_fp: Path + svn_version: int, + llvm_src_dir: Path, + patches_json_fp: Path, + patch_cmd: Optional[Callable] = None, ) -> PatchInfo: """Reduce the version ranges of failing patches. @@ -475,12 +516,13 @@ def update_version_ranges( Modifies the contents of patches_json_fp. Args: - svn_version: LLVM revision number. - llvm_src_dir: llvm-project directory path. - patches_json_fp: Filepath to the PATCHES.json file. + svn_version: LLVM revision number. + llvm_src_dir: llvm-project directory path. + patches_json_fp: Filepath to the PATCHES.json file. + patch_cmd: option to apply patch. Returns: - PatchInfo for applied and disabled patches. + PatchInfo for applied and disabled patches. """ with patches_json_fp.open(encoding="utf-8") as f: contents = f.read() @@ -490,7 +532,7 @@ def update_version_ranges( contents, ) modified_entries, applied_patches = update_version_ranges_with_entries( - svn_version, llvm_src_dir, patch_entries + svn_version, llvm_src_dir, patch_entries, patch_cmd ) with atomic_write_file.atomic_write(patches_json_fp, encoding="utf-8") as f: _write_json_changes( @@ -515,19 +557,20 @@ def update_version_ranges_with_entries( svn_version: int, llvm_src_dir: Path, patch_entries: Iterable[PatchEntry], + patch_cmd: Optional[Callable] = None, ) -> Tuple[List[PatchEntry], List[PatchEntry]]: """Test-able helper for UpdateVersionRanges. Args: - svn_version: LLVM revision number. - llvm_src_dir: llvm-project directory path. - patch_entries: PatchEntry objects to modify. + svn_version: LLVM revision number. + llvm_src_dir: llvm-project directory path. + patch_entries: PatchEntry objects to modify. Returns: - Tuple of (modified entries, applied patches) + Tuple of (modified entries, applied patches) Post: - Modifies patch_entries in place. + Modifies patch_entries in place. """ modified_entries: List[PatchEntry] = [] applied_patches: List[PatchEntry] = [] @@ -536,7 +579,7 @@ def update_version_ranges_with_entries( ) with git_clean_context(llvm_src_dir): for pe in active_patches: - test_result = pe.test_apply(llvm_src_dir) + test_result = pe.test_apply(llvm_src_dir, patch_cmd) if not test_result: if pe.version_range is None: pe.version_range = {} @@ -545,7 +588,7 @@ def update_version_ranges_with_entries( else: # We have to actually apply the patch so that future patches # will stack properly. - if not pe.apply(llvm_src_dir).succeeded: + if not pe.apply(llvm_src_dir, patch_cmd).succeeded: raise RuntimeError( "Could not apply patch that dry ran successfully" ) @@ -563,12 +606,12 @@ def remove_old_patches( each patch entry. Args: - svn_version: LLVM SVN version. - llvm_src_dir: LLVM source directory. - patches_json_fp: Location to edit patches on. + svn_version: LLVM SVN version. + llvm_src_dir: LLVM source directory. + patches_json_fp: Location to edit patches on. Returns: - PatchInfo for modified patches. + PatchInfo for modified patches. """ with patches_json_fp.open(encoding="utf-8") as f: contents = f.read() @@ -595,3 +638,23 @@ def remove_old_patches( removed_patches=[p.rel_patch_path for p in removed_entries], modified_metadata=str(patches_json_fp) if removed_entries else None, ) + + +def git_am(patch_path: Path) -> List[Union[str, Path]]: + cmd = ["git", "am", "--3way", str(patch_path)] + return cmd + + +def gnu_patch(root_dir: Path, patch_path: Path) -> List[Union[str, Path]]: + cmd = [ + "patch", + "-d", + str(root_dir.absolute()), + "-f", + "-E", + "-p1", + "--no-backup-if-mismatch", + "-i", + str(patch_path), + ] + return cmd |