diff options
author | Yabin Cui <yabinc@google.com> | 2023-08-21 18:52:18 +0000 |
---|---|---|
committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2023-08-21 18:52:18 +0000 |
commit | 1bcc74be81c5bfccdd60c6af7ab60d064ddee269 (patch) | |
tree | 3e4cabab5a57ae288c245cff9a1a6ff7926a3207 | |
parent | 40214b48188358a80b7478bfff21d4814dd9177c (diff) | |
parent | e0c166ae0cf21cf88f77dd04137acea9366a9922 (diff) | |
download | toolchain-utils-1bcc74be81c5bfccdd60c6af7ab60d064ddee269.tar.gz |
Upgrade toolchain-utils to 2942bd8023e3e0a79ed78fd4fff22c1f961e9f73 am: 1c2bb62c89 am: e0c166ae0c
Original change: https://android-review.googlesource.com/c/platform/external/toolchain-utils/+/2716013
Change-Id: I4e686942d789b1b9d0fbf37b0f1fcf2f6b9dd0b7
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
147 files changed, 5909 insertions, 6292 deletions
diff --git a/DIR_METADATA b/DIR_METADATA new file mode 100644 index 00000000..fe952b8e --- /dev/null +++ b/DIR_METADATA @@ -0,0 +1,21 @@ +# Metadata information for this directory. +# +# For more information on DIR_METADATA files, see: +# https://source.chromium.org/chromium/infra/infra/+/HEAD:go/src/infra/tools/dirmd/README.md +# +# For the schema of this file, see Metadata message: +# https://source.chromium.org/chromium/infra/infra/+/HEAD:go/src/infra/tools/dirmd/proto/dir_metadata.proto + +team_email: "chromeos-toolchain@google.com" + +buganizer { + # https://b.corp.google.com/issues?q=status:open%20componentid:1034879 + # ChromeOS > Infra > Toolchain + component_id: 1034879 +} + +buganizer_public { + # https://b.corp.google.com/issues?q=status:open%20componentid:1038090 + # ChromeOS Public Tracker > Services > Infra > Toolchain + component_id: 1038090 +} @@ -1,6 +1,6 @@ # This project was upgraded with external_updater. # Usage: tools/external_updater/updater.sh update toolchain-utils -# For more info, check https://cs.android.com/android/platform/superproject/+/master:tools/external_updater/README.md +# For more info, check https://cs.android.com/android/platform/superproject/+/main:tools/external_updater/README.md name: "toolchain-utils" description: "Various utilities used by the ChromeOS toolchain team." @@ -9,11 +9,11 @@ third_party { type: GIT value: "https://chromium.googlesource.com/chromiumos/third_party/toolchain-utils/" } - version: "2c474af4f370b143032144aff1ff1985f789e20f" + version: "2942bd8023e3e0a79ed78fd4fff22c1f961e9f73" license_type: NOTICE last_upgrade_date { - year: 2022 - month: 11 - day: 8 + year: 2023 + month: 8 + day: 17 } } diff --git a/OWNERS.toolchain b/OWNERS.toolchain index e8cdbf98..967440e3 100644 --- a/OWNERS.toolchain +++ b/OWNERS.toolchain @@ -1,9 +1,7 @@ -adriandole@google.com ajordanr@google.com cjdb@google.com denik@chromium.org gbiv@chromium.org inglorion@chromium.org manojgupta@chromium.org -mbenfield@google.com ryanbeltran@chromium.org diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index 98dab12e..0d6cecc2 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -1,17 +1,14 @@ { - "chromeos-kernel-4_4": { - "name": "R108-15117.10-1664184941" - }, "chromeos-kernel-4_14": { - "name": "R109-15183.8-1666603998" - }, - "chromeos-kernel-4_19": { - "name": "R109-15183.8-1666604011" + "name": "R117-15563.0-1691400981" }, "chromeos-kernel-5_4": { - "name": "R109-15183.8-1666603918" + "name": "R117-15563.0-1691401051" }, "chromeos-kernel-5_10": { - "name": "R109-15183.8-1666604219" + "name": "R117-15563.0-1691400779" + }, + "chromeos-kernel-5_15": { + "name": "R117-15563.0-1691400946" } } diff --git a/afdo_metadata/kernel_arm_afdo.json b/afdo_metadata/kernel_arm_afdo.json index e73d2eb8..c90edba4 100644 --- a/afdo_metadata/kernel_arm_afdo.json +++ b/afdo_metadata/kernel_arm_afdo.json @@ -1,5 +1,5 @@ { "chromeos-kernel-5_15": { - "name": "R109-15183.8-1666604194" + "name": "R117-15563.0-1691400899" } } diff --git a/afdo_tools/update_kernel_afdo b/afdo_tools/update_kernel_afdo index 9e4d645d..6bfa53fa 100755 --- a/afdo_tools/update_kernel_afdo +++ b/afdo_tools/update_kernel_afdo @@ -9,14 +9,17 @@ # USAGE=" -Usage: $(basename "$0") [--noupload|-upload] [main|beta|stable|all] [--help] +Usage: $(basename "$0") [--help] [--(no)upload] [--nointeractive] + [main|beta|stable|all] Description: The script takes one optional argument which is the channel where we want to update the kernel afdo and creates a commit (or commits with \"all\" channels) in the corresponding branch. No arguments defaults to \"all\". - Follow the prompt to upload the changes. + Follow the prompt to upload the changes with --noupload. Otherwise + the script will automatically create CL and send to the detective + for review. NO CLEAN-UP NEEDED. The script ignores any local changes and keeps the current branch unchanged. @@ -24,43 +27,43 @@ the current branch unchanged. --help Show this help. --upload Upload CLs when the update succeeded (default). --noupload Do not upload CLs. Instead, print the upload commands. + --nointeractive Runs the script without user interaction. main|beta|stable Update metadata only on the specified channel. " set -eu set -o pipefail -AMD_GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel +# Branch independent constants. +# Changes here will affect kernel afdo update in cros branches. +# ------------------- +ARCHS="amd arm" +AMD_GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel/amd64 ARM_GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel/arm -AMD_KVERS="4.14 4.19 5.4 5.10" -ARM_KVERS="5.15" -failed_channels="" +UPDATE_CONFIG_FILE="afdo_tools/update_kernel_afdo.cfg" +# CL reviewers and cc. +REVIEWERS="c-compiler-chrome@google.com" +CC="denik@google.com,gbiv@google.com" # Add skipped chrome branches in ascending order here. SKIPPED_BRANCHES="95" - # NOTE: We enable/disable kernel AFDO starting from a particular branch. # For example if we want to enable kernel AFDO in 5.15, first, we do it # in main. In this case we want to disable it in beta and stable branches. # The second scenario is when we want to disable kernel AFDO (when all devices # move to kernelnext and there are no new profiles from the field). In this # case we disable AFDO in main but still keep it live in beta and stable. -declare -A SKIPPED_KVERS_IN_BRANCHES -# In SKIPPED_KVERS_IN_BRANCHES +declare -A SKIPPED_ARCHKVERS_IN_BRANCHES +# In SKIPPED_ARCHKVERS_IN_BRANCHES # - key is a branch number string; -# - value is the list of kernels separated by space. -# Example: SKIPPED_KVERS_IN_BRANCHES["105"]="4.4 4.14" - -# b/223115767. In M-100 there are no new profiles in 5.10. And AFDO is not -# enabled on any 5.10 board in M-100 either. -SKIPPED_KVERS_IN_BRANCHES["100"]="5.10" +# - value is the list of arch/kver separated by space. +# Example: SKIPPED_ARCHKVERS_IN_BRANCHES["105"]="amd/4.4 arm/5.15" +# ------------------- +# Kernel tracing was disabled on arm in 114, b/275560674. +SKIPPED_ARCHKVERS_IN_BRANCHES["114"]="arm/5.15" +SKIPPED_ARCHKVERS_IN_BRANCHES["115"]="arm/5.15" script_dir=$(dirname "$0") tc_utils_dir="${script_dir}/.." -metadata_dir="${tc_utils_dir}/afdo_metadata" -amd_outfile="$(realpath --relative-to="${tc_utils_dir}" \ - "${metadata_dir}"/kernel_afdo.json)" -arm_outfile="$(realpath --relative-to="${tc_utils_dir}" \ - "${metadata_dir}"/kernel_arm_afdo.json)" # Convert toolchain_utils into the absolute path. abs_tc_utils_dir="$(realpath "${tc_utils_dir}")" @@ -68,16 +71,13 @@ abs_tc_utils_dir="$(realpath "${tc_utils_dir}")" expected_time=$(date +%s -d "week ago") # Upload CLs on success. upload_cl=true +# Interactive mode. +interactive=true +# Without arguments the script updates all branches. +channels="" +failed_channels="" -ARCHS="amd arm" declare -A arch_gsbase arch_kvers arch_outfile -arch_gsbase["amd"]="${AMD_GS_BASE}" -arch_gsbase["arm"]="${ARM_GS_BASE}" -arch_kvers["amd"]="${AMD_KVERS}" -arch_kvers["arm"]="${ARM_KVERS}" -arch_outfile["amd"]="${amd_outfile}" -arch_outfile["arm"]="${arm_outfile}" - declare -A branch branch_number commit remote_repo=$(git -C "${tc_utils_dir}" remote) canary_ref="refs/heads/main" @@ -103,9 +103,9 @@ for skipped_branch in ${SKIPPED_BRANCHES} ; do ((branch_number[canary]++)) fi done +config_file="$(realpath --relative-to="${tc_utils_dir}" \ + "${tc_utils_dir}/${UPDATE_CONFIG_FILE}")" -# Without arguments the script updates all branches. -channels="" for arg in "$@" do case "${arg}" in @@ -124,12 +124,15 @@ do --upload) upload_cl=true ;; + --nointeractive) + interactive=false + ;; --help | help | -h ) echo "${USAGE}" exit 0 ;; -*) - echo "Option \"${arg}\" is not supported." >&2 + echo "ERROR: Option \"${arg}\" is not supported." >&2 echo "${USAGE}" exit 1 ;; @@ -155,7 +158,8 @@ echo "-> Working in ${worktree_dir}" # This way we don't need to clean-up and sync toolchain_utils before the # change. Neither we should care about clean-up after the submit. git -C "${tc_utils_dir}" worktree add --detach "${worktree_dir}" -trap 'git -C "${abs_tc_utils_dir}" worktree remove -f "${worktree_dir}"' EXIT +trap 'git -C "${abs_tc_utils_dir}" worktree remove -f "${worktree_dir}" \ + && git -C "${abs_tc_utils_dir}" branch -D ${channels}' EXIT pushd "${worktree_dir}" for channel in ${channels} @@ -178,8 +182,37 @@ do echo "branch_number=${curr_branch_number} branch=${curr_branch}" git reset --hard HEAD - git checkout "${remote_repo}/${curr_branch}" + git checkout -b "${channel}" "${remote_repo}/${curr_branch}" + # Read branch-dependent constants from $remote_repo. + # shellcheck source=afdo_tools/update_kernel_afdo.cfg + if [[ -e "${config_file}" ]] + then + # Branch dependent constants were moved to config_file. + # IMPORTANT: Starting from M-113 update_kernel_afdo reads branch-dependent + # constants from config_file from remote refs. + source "${config_file}" + else + # DON'T UPDATE THESE CONSTANTS HERE! + # Update ${config_file} instead. + AMD_KVERS="4.14 4.19 5.4 5.10" + ARM_KVERS="5.15" + AMD_METADATA_FILE="afdo_metadata/kernel_afdo.json" + ARM_METADATA_FILE="afdo_metadata/kernel_arm_afdo.json" + fi + + amd_outfile="$(realpath --relative-to="${tc_utils_dir}" \ + "${tc_utils_dir}/${AMD_METADATA_FILE}")" + arm_outfile="$(realpath --relative-to="${tc_utils_dir}" \ + "${tc_utils_dir}/${ARM_METADATA_FILE}")" + arch_gsbase["amd"]="${AMD_GS_BASE}" + arch_gsbase["arm"]="${ARM_GS_BASE}" + arch_kvers["amd"]="${AMD_KVERS}" + arch_kvers["arm"]="${ARM_KVERS}" + arch_outfile["amd"]="${amd_outfile}" + arch_outfile["arm"]="${arm_outfile}" + + new_changes=false for arch in ${ARCHS} do json="{" @@ -188,15 +221,16 @@ do do # Skip kernels disabled in this branch. skipped=false - for skipped_branch in "${!SKIPPED_KVERS_IN_BRANCHES[@]}" + for skipped_branch in "${!SKIPPED_ARCHKVERS_IN_BRANCHES[@]}" do if [[ ${curr_branch_number} == "${skipped_branch}" ]] then - # Current branch is in the keys of SKIPPED_KVERS_IN_BRANCHES. - # Now lets check if $kver is in the list. - for skipped_kver in ${SKIPPED_KVERS_IN_BRANCHES[${skipped_branch}]} + # Current branch is in the keys of SKIPPED_ARCHKVERS_IN_BRANCHES. + # Now lets check if $arch/$kver is in the list. + for skipped_archkver in \ + ${SKIPPED_ARCHKVERS_IN_BRANCHES[${skipped_branch}]} do - if [[ ${kver} == "${skipped_kver}" ]] + if [[ "${arch}/${kver}" == "${skipped_archkver}" ]] then skipped=true break @@ -206,7 +240,7 @@ do done if ${skipped} then - echo "${kver} is skipped in branch ${curr_branch_number}. Skip it." + echo "${arch}/${kver} is skipped in branch ${curr_branch_number}." continue fi # Sort the gs output by timestamp, default ordering is by name. So @@ -214,11 +248,20 @@ do # R86-13310.18-1595237847.gcov.xz. latest=$(gsutil.py ls -l "${arch_gsbase[${arch}]}/${kver}/" | sort -k2 | \ grep "R${curr_branch_number}" | tail -1 || true) + prev_branch=$((curr_branch_number - 1)) if [[ -z "${latest}" && "${channel}" != "stable" ]] then # if no profiles exist for the current branch, try the previous branch latest=$(gsutil.py ls -l "${arch_gsbase[${arch}]}/${kver}/" | \ - sort -k2 | grep "R$((curr_branch_number - 1))" | tail -1) + sort -k2 | grep "R${prev_branch}" | tail -1 || true) + fi + if [[ -z "${latest}" ]] + then + echo "ERROR: No M${curr_branch_number}, M${prev_branch} profiles in\ + ${arch_gsbase[${arch}]}/${kver}/" >&2 + echo "Skipping ${arch}/${kver}" >&2 + errs="${errs} ${kver}" + continue fi # Verify that the file has the expected date. @@ -227,7 +270,8 @@ do if [ "${file_time_unix}" -lt "${expected_time}" ] then expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @"${expected_time}") - echo "Wrong date for ${kver}: ${file_time} is before ${expected}" >&2 + echo "ERROR: Wrong date for ${kver}: ${file_time} is before\ + ${expected}" >&2 errs="${errs} ${kver}" continue fi @@ -256,7 +300,7 @@ EOT # If we did not succeed for any kvers, exit now. if [[ ${successes} -eq 0 ]] then - echo "error: AFDO profiles out of date for all kernel versions" >&2 + echo "ERROR: AFDO profiles out of date for all kernel versions" >&2 failed_channels="${failed_channels} ${channel}" continue fi @@ -276,14 +320,21 @@ EOT # If we had any errors, warn about them. if [[ -n "${errs}" ]] then - echo "warning: failed to update ${errs} in ${channel}" >&2 + echo "WARNING: failed to update ${errs} in ${channel}" >&2 failed_channels="${failed_channels} ${channel}" continue fi git add "${arch_outfile[${arch}]}" + new_changes=true done # ARCHS loop + if ! ${new_changes} + then + echo "Skipping \"${channel}\" - all profiles are up to date" + continue + fi + case "${channel}" in canary ) commit_contents=$'afdo_metadata: Publish the new kernel profiles\n\n' @@ -308,11 +359,16 @@ BUG=None TEST=Verified in kernel-release-afdo-verify-orchestrator" ;; * ) - echo "internal error: unhandled channel \"${channel}\"" >&2 + echo "Internal error: unhandled channel \"${channel}\"" >&2 exit 2 esac - git commit -v -e -m "${commit_contents}" + if ${interactive} + then + git commit -v -e -m "${commit_contents}" + else + git commit -m "${commit_contents}" + fi commit[${channel}]=$(git -C "${worktree_dir}" rev-parse HEAD) done @@ -329,8 +385,15 @@ then then for channel in "${!commit[@]}" do - git -C "${tc_utils_dir}" push "${remote_repo}" \ - "${commit[${channel}]}:refs/for/${branch[${channel}]}" + if ${interactive} + then + (cd "${tc_utils_dir}" && \ + repo upload --br="${channel}" --re="${REVIEWERS}" --cc="${CC}" .) + else + (cd "${tc_utils_dir}" && \ + repo upload --br="${channel}" --no-verify -y --re="${REVIEWERS}" \ + --cc="${CC}" .) + fi done else echo "Run these commands to upload the change:" @@ -346,7 +409,7 @@ then if [[ -n "${failed_channels}" ]] then echo - echo "error: failed to update kernel afdo in ${failed_channels}" >&2 + echo "ERROR: failed to update kernel afdo in ${failed_channels}" >&2 exit 3 fi else @@ -355,7 +418,7 @@ else then echo "No changes are applied. It looks like AFDO versions are up to date." else - echo "error: update in ${failed_channels} failed" >&2 + echo "ERROR: update in ${failed_channels} failed" >&2 exit 3 fi fi diff --git a/afdo_tools/update_kernel_afdo.cfg b/afdo_tools/update_kernel_afdo.cfg new file mode 100644 index 00000000..a196a0af --- /dev/null +++ b/afdo_tools/update_kernel_afdo.cfg @@ -0,0 +1,8 @@ +# Add a new supported kernel version in this file. +# All changes here won't affect kernel afdo update in branches. +# WARNING: Changes must be submitted to have effect. + +AMD_KVERS="4.14 5.4 5.10 5.15" +ARM_KVERS="5.15" +AMD_METADATA_FILE="afdo_metadata/kernel_afdo.json" +ARM_METADATA_FILE="afdo_metadata/kernel_arm_afdo.json" diff --git a/auto_delete_nightly_test_data.py b/auto_delete_nightly_test_data.py index 0dd2dba8..06d02067 100755 --- a/auto_delete_nightly_test_data.py +++ b/auto_delete_nightly_test_data.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be @@ -10,88 +9,17 @@ __author__ = "shenhan@google.com (Han Shen)" import argparse -import datetime import os from pathlib import Path -import re import shutil import stat import sys import time import traceback -from typing import Callable +from typing import Callable, List from cros_utils import command_executer from cros_utils import constants -from cros_utils import misc - - -DIR_BY_WEEKDAY = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun") -NIGHTLY_TESTS_WORKSPACE = os.path.join( - constants.CROSTC_WORKSPACE, "nightly-tests" -) - - -def CleanNumberedDir(s, dry_run=False): - """Deleted directories under each dated_dir.""" - chromeos_dirs = [ - os.path.join(s, x) - for x in os.listdir(s) - if misc.IsChromeOsTree(os.path.join(s, x)) - ] - ce = command_executer.GetCommandExecuter(log_level="none") - all_succeeded = True - for cd in chromeos_dirs: - if misc.DeleteChromeOsTree(cd, dry_run=dry_run): - print(f"Successfully removed chromeos tree {cd!r}.") - else: - all_succeeded = False - print(f"Failed to remove chromeos tree {cd!r}, please check.") - - if not all_succeeded: - print("Failed to delete at least one chromeos tree, please check.") - return False - - ## Now delete the numbered dir Before forcibly removing the directory, just - ## check 's' to make sure it matches the expected pattern. A valid dir to be - ## removed must be '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'. - valid_dir_pattern = ( - "^" + NIGHTLY_TESTS_WORKSPACE + "/(" + "|".join(DIR_BY_WEEKDAY) + ")" - ) - if not re.search(valid_dir_pattern, s): - print( - f"Trying to delete an invalid dir {s!r} (must match " - f"{valid_dir_pattern!r}), please check." - ) - return False - - cmd = f"rm -fr {s}" - if dry_run: - print(cmd) - else: - if ( - ce.RunCommand(cmd, print_to_console=False, terminated_timeout=480) - == 0 - ): - print(f"Successfully removed {s!r}.") - else: - all_succeeded = False - print(f"Failed to remove {s!r}, please check.") - return all_succeeded - - -def CleanDatedDir(dated_dir, dry_run=False): - # List subdirs under dir - subdirs = [ - os.path.join(dated_dir, x) - for x in os.listdir(dated_dir) - if os.path.isdir(os.path.join(dated_dir, x)) - ] - all_succeeded = True - for s in subdirs: - if not CleanNumberedDir(s, dry_run): - all_succeeded = False - return all_succeeded def ProcessArguments(argv): @@ -136,10 +64,10 @@ def RemoveAllSubdirsMatchingPredicate( try: dir_entries = list(base_dir.iterdir()) except FileNotFoundError as e: - # We get this if the directory itself doesn't exist. Since we're cleaning - # tempdirs, that's as good as a success. Further, the prior approach here - # was using the `find` binary, which exits successfully if nothing is - # found. + # We get this if the directory itself doesn't exist. Since we're + # cleaning tempdirs, that's as good as a success. Further, the prior + # approach here was using the `find` binary, which exits successfully + # if nothing is found. print(f"Error enumerating {base_dir}'s contents; skipping removal: {e}") return 0 @@ -149,8 +77,8 @@ def RemoveAllSubdirsMatchingPredicate( continue try: - # Take the stat here and use that later, so we only need to check for a - # nonexistent file once. + # Take the stat here and use that later, so we only need to check + # for a nonexistent file once. st = file.stat() except FileNotFoundError: # This was deleted while were checking; ignore it. @@ -176,30 +104,32 @@ def RemoveAllSubdirsMatchingPredicate( shutil.rmtree(file, onerror=OnError) - # Some errors can be other processes racing with us to delete things. Don't - # count those as an error which we complain loudly about. + # Some errors can be other processes racing with us to delete things. + # Don't count those as an error which we complain loudly about. if this_iteration_had_errors: if file.exists(): had_errors = True else: print( - f"Discarding removal errors for {file}; dir was still removed." + f"Discarding removal errors for {file}; dir was still " + "removed." ) return 1 if had_errors else 0 def IsChromeOsTmpDeletionCandidate(file_name: str): - """Returns whether the given basename can be deleted from a chroot's /tmp.""" + """Returns whether the given basename can be deleted from chroot's /tmp.""" name_prefixes = ( "test_that_", "cros-update", "CrAU_temp_data", + # This might seem a bit broad, but after using custom heuristics for a + # while, `/tmp` was observed to have >75K files that matched all sorts + # of different `tmp.*` name patterns. Just remove them all. + "tmp", ) - if any(file_name.startswith(x) for x in name_prefixes): - return True - # Remove files that look like `tmpABCDEFGHI`. - return len(file_name) == 9 and file_name.startswith("tmp") + return any(file_name.startswith(x) for x in name_prefixes) def CleanChromeOsTmpFiles( @@ -238,47 +168,47 @@ def CleanChromeOsImageFiles( try: shutil.rmtree(subdir_path) print( - "Successfully cleaned chromeos image autotest directories " - f"from {subdir_path!r}." + "Successfully cleaned chromeos image autotest " + f"directories from {subdir_path!r}." ) except OSError: print( - "Some image autotest directories were not removed from " - f'"{subdir_path}".' + "Some image autotest directories were not " + f'"removed from {subdir_path}".' ) errors += 1 return errors -def CleanChromeOsTmpAndImages(days_to_preserve=1, dry_run=False): +def CleanChromeOsTmpAndImages(days_to_preserve=1, dry_run=False) -> int: """Delete temporaries, images under crostc/chromeos.""" chromeos_chroot_tmp = os.path.join( - constants.CROSTC_WORKSPACE, "chromeos", "chroot", "tmp" + constants.CROSTC_WORKSPACE, "chromeos", "out", "tmp" ) # Clean files in tmp directory rv = CleanChromeOsTmpFiles(chromeos_chroot_tmp, days_to_preserve, dry_run) # Clean image files in *-tryjob directories - rv += CleanChromeOsImageFiles( + rv |= CleanChromeOsImageFiles( chromeos_chroot_tmp, "-tryjob", days_to_preserve, dry_run ) # Clean image files in *-release directories - rv += CleanChromeOsImageFiles( + rv |= CleanChromeOsImageFiles( chromeos_chroot_tmp, "-release", days_to_preserve, dry_run ) # Clean image files in *-pfq directories - rv += CleanChromeOsImageFiles( + rv |= CleanChromeOsImageFiles( chromeos_chroot_tmp, "-pfq", days_to_preserve, dry_run ) # Clean image files in *-llvm-next-nightly directories - rv += CleanChromeOsImageFiles( + rv |= CleanChromeOsImageFiles( chromeos_chroot_tmp, "-llvm-next-nightly", days_to_preserve, dry_run ) return rv -def CleanOldCLs(days_to_preserve="1", dry_run=False): +def CleanOldCLs(days_to_preserve: str = "1", dry_run: bool = False) -> int: """Abandon old CLs created by automation tooling.""" ce = command_executer.GetCommandExecuter() chromeos_root = os.path.join(constants.CROSTC_WORKSPACE, "chromeos") @@ -305,15 +235,7 @@ def CleanOldCLs(days_to_preserve="1", dry_run=False): def CleanChromeTelemetryTmpFiles(dry_run: bool) -> int: - tmp_dir = ( - Path(constants.CROSTC_WORKSPACE) - / "chromeos" - / ".cache" - / "distfiles" - / "chrome-src-internal" - / "src" - / "tmp" - ) + tmp_dir = Path(constants.CROSTC_WORKSPACE) / "chrome" / "src" / "tmp" return RemoveAllSubdirsMatchingPredicate( tmp_dir, days_to_preserve=0, @@ -323,47 +245,22 @@ def CleanChromeTelemetryTmpFiles(dry_run: bool) -> int: ) -def Main(argv): +def Main(argv: List[str]) -> int: """Delete nightly test data directories, tmps and test images.""" options = ProcessArguments(argv) - # Function 'isoweekday' returns 1(Monday) - 7 (Sunday). - d = datetime.datetime.today().isoweekday() - # We go back 1 week, delete from that day till we are - # options.days_to_preserve away from today. - s = d - 7 - e = d - int(options.days_to_preserve) - rv = 0 - for i in range(s + 1, e): - if i <= 0: - ## Wrap around if index is negative. 6 is from i + 7 - 1, because - ## DIR_BY_WEEKDAY starts from 0, while isoweekday is from 1-7. - dated_dir = DIR_BY_WEEKDAY[i + 6] - else: - dated_dir = DIR_BY_WEEKDAY[i - 1] - - rv += ( - 0 - if CleanDatedDir( - os.path.join(NIGHTLY_TESTS_WORKSPACE, dated_dir), - options.dry_run, - ) - else 1 - ) - ## Clean temporaries, images under crostc/chromeos - rv2 = CleanChromeOsTmpAndImages( + rv = CleanChromeOsTmpAndImages( int(options.days_to_preserve), options.dry_run ) # Clean CLs that are not updated in last 2 weeks. - rv3 = CleanOldCLs("14", options.dry_run) + rv |= CleanOldCLs("14", options.dry_run) # Clean telemetry temporaries from chrome source tree inside chroot. - rv4 = CleanChromeTelemetryTmpFiles(options.dry_run) + rv |= CleanChromeTelemetryTmpFiles(options.dry_run) - return rv + rv2 + rv3 + rv4 + return 1 if rv else 0 if __name__ == "__main__": - retval = Main(sys.argv[1:]) - sys.exit(retval) + sys.exit(Main(sys.argv[1:])) diff --git a/buildbot_test_llvm.py b/buildbot_test_llvm.py deleted file mode 100755 index 57f029c7..00000000 --- a/buildbot_test_llvm.py +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2017 The ChromiumOS Authors -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Script for running llvm validation tests on ChromeOS. - -This script launches a buildbot to build ChromeOS with the llvm on -a particular board; then it finds and downloads the trybot image and the -corresponding official image, and runs test for correctness. -It then generates a report, emails it to the c-compiler-chrome, as -well as copying the result into a directory. -""" - -# Script to test different toolchains against ChromeOS benchmarks. - - -import argparse -import datetime -import os -import sys -import time - -from cros_utils import buildbot_utils -from cros_utils import command_executer -from cros_utils import logger - - -CROSTC_ROOT = "/usr/local/google/crostc" -ROLE_ACCOUNT = "mobiletc-prebuild" -TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) -MAIL_PROGRAM = "~/var/bin/mail-detective" -VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, "validation_result") -START_DATE = datetime.date(2016, 1, 1) -TEST_PER_DAY = 4 -DATA_DIR = "/google/data/rw/users/mo/mobiletc-prebuild/waterfall-report-data/" - -# Information about Rotating Boards -# Board Arch Reference Platform Kernel -# Board Version -# ------------ ------- ------------ ------------- ------- -# atlas x86_64 poppy kabylake-y 4.4.* -# cave x86_64 glados skylake-y 3.18 -# coral x86_64 reef apollo lake 4.4.* -# cyan x86_64 strago braswell 3.18 -# elm aarch64 oak mediatek-8173 3.18 arm32 userspace -# eve x86_64 poppy kabylake-u/r 4.4.* -# gale armv7 3.18 -# grunt x86_64 grunt stoney ridge 4.14.* -# fizz-moblab x86_64 4.4 -# kevin aarch64 gru rockchip-3399 4.4.* arm32 userspace -# kevin64 aarch64 gru rockchip-3399 4.4.* arm64 userspace -# lakitu x86_64 4.4.* -# nyan_kitty armv7 nyan tegra 3.10.18 -# octopus x86_64 octopus GLK 4.14.* -# sentry x86_64 kunimitsu skylake-u 3.18 -# tidus x86_64 auron broadwell 3.14 -# veyron_mighty armv7 veyron-pinky rockchip-3288 3.14 -# whirlwind armv7 3.14 -# winky x86_64 rambi baytrail 4.4.* - -TEST_BOARD = [ - "atlas", - "cave", - "coral", - "cyan", - "elm", - # 'eve', tested by amd64-llvm-next-toolchain builder. - "gale", - "grunt", - "fizz-moblab", - # 'kevin', tested by arm64-llvm-next-toolchain builder. - "kevin64", - "lakitu", - "nyan_kitty", - "octopus", - "sentry", - "tidus", - # 'veyron_mighty', tested by arm-llvm-next-toolchain builder. - "whirlwind", - "winky", -] - - -class ToolchainVerifier(object): - """Class for the toolchain verifier.""" - - def __init__(self, board, chromeos_root, weekday, patches, compiler): - self._board = board - self._chromeos_root = chromeos_root - self._base_dir = os.getcwd() - self._ce = command_executer.GetCommandExecuter() - self._l = logger.GetLogger() - self._compiler = compiler - self._build = "%s-%s-toolchain-tryjob" % (board, compiler) - self._patches = patches.split(",") if patches else [] - self._patches_string = "_".join(str(p) for p in self._patches) - - if not weekday: - self._weekday = time.strftime("%a") - else: - self._weekday = weekday - self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board) - - def DoAll(self): - """Main function inside ToolchainComparator class. - - Launch trybot, get image names, create crosperf experiment file, run - crosperf, and copy images into seven-day report directories. - """ - buildbucket_id, _ = buildbot_utils.GetTrybotImage( - self._chromeos_root, - self._build, - self._patches, - tryjob_flags=["--hwtest"], - asynchronous=True, - ) - - return buildbucket_id - - -def WriteRotatingReportsData(results_dict, date): - """Write data for waterfall report.""" - fname = "%d-%02d-%02d.builds" % (date.year, date.month, date.day) - filename = os.path.join(DATA_DIR, "rotating-builders", fname) - with open(filename, "w", encoding="utf-8") as out_file: - for board in results_dict.keys(): - buildbucket_id = results_dict[board] - out_file.write("%s,%s\n" % (buildbucket_id, board)) - - -def Main(argv): - """The main function.""" - - # Common initializations - command_executer.InitCommandExecuter() - parser = argparse.ArgumentParser() - parser.add_argument( - "--chromeos_root", - dest="chromeos_root", - help="The chromeos root from which to run tests.", - ) - parser.add_argument( - "--weekday", - default="", - dest="weekday", - help="The day of the week for which to run tests.", - ) - parser.add_argument( - "--board", default="", dest="board", help="The board to test." - ) - parser.add_argument( - "--patch", - dest="patches", - default="", - help="The patches to use for the testing, " - "seprate the patch numbers with ',' " - "for more than one patches.", - ) - parser.add_argument( - "--compiler", - dest="compiler", - help="Which compiler (llvm, llvm-next or gcc) to use for " "testing.", - ) - - options = parser.parse_args(argv[1:]) - if not options.chromeos_root: - print("Please specify the ChromeOS root directory.") - return 1 - if not options.compiler: - print( - "Please specify which compiler to test (gcc, llvm, or llvm-next)." - ) - return 1 - - if options.board: - fv = ToolchainVerifier( - options.board, - options.chromeos_root, - options.weekday, - options.patches, - options.compiler, - ) - return fv.DoAll() - - today = datetime.date.today() - delta = today - START_DATE - days = delta.days - - start_board = (days * TEST_PER_DAY) % len(TEST_BOARD) - results_dict = dict() - for i in range(TEST_PER_DAY): - try: - board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)] - fv = ToolchainVerifier( - board, - options.chromeos_root, - options.weekday, - options.patches, - options.compiler, - ) - buildbucket_id = fv.DoAll() - if buildbucket_id: - results_dict[board] = buildbucket_id - except SystemExit: - logfile = os.path.join( - VALIDATION_RESULT_DIR, options.compiler, board - ) - with open(logfile, "w", encoding="utf-8") as f: - f.write("Verifier got an exception, please check the log.\n") - WriteRotatingReportsData(results_dict, today) - - -if __name__ == "__main__": - retval = Main(sys.argv) - sys.exit(retval) diff --git a/buildbot_test_toolchains.py b/buildbot_test_toolchains.py index 19c31b54..88ab4052 100755 --- a/buildbot_test_toolchains.py +++ b/buildbot_test_toolchains.py @@ -84,10 +84,12 @@ class ToolchainComparator(object): recipe=False, test=False, noschedv2=False, + chrome_src="", ): self._board = board self._remotes = remotes self._chromeos_root = chromeos_root + self._chrome_src = chrome_src self._base_dir = os.getcwd() self._ce = command_executer.GetCommandExecuter() self._l = logger.GetLogger() @@ -178,7 +180,12 @@ class ToolchainComparator(object): test_args: --story-tag-filter=typical iterations: 3 run_local: False - retries: 0 + } + + benchmark: platform.ReportDiskUsage { + suite: tast + iterations: 1 + run_local: False } """ @@ -190,11 +197,13 @@ class ToolchainComparator(object): official_image = """ vanilla_image { chromeos_root: %s + chrome_src: %s build: %s compiler: llvm } """ % ( self._chromeos_root, + self._chrome_src, vanilla_image, ) f.write(official_image) @@ -208,6 +217,7 @@ class ToolchainComparator(object): experiment_image = """ %s { chromeos_root: %s + chrome_src: %s build: %s autotest_path: %s compiler: %s @@ -215,6 +225,7 @@ class ToolchainComparator(object): """ % ( label_string, self._chromeos_root, + self._chrome_src, trybot_image, autotest_files, compiler_string, @@ -338,6 +349,14 @@ def Main(argv): help="The chromeos root from which to run tests.", ) parser.add_argument( + "--chrome_src", + dest="chrome_src", + default="", + help="The path to the source of chrome. " + "This is used to run telemetry benchmarks. " + "The default one is the src inside chroot.", + ) + parser.add_argument( "--weekday", default="", dest="weekday", @@ -398,6 +417,7 @@ def Main(argv): options.recipe, options.test, options.noschedv2, + chrome_src=options.chrome_src, ) return fc.DoAll() diff --git a/compiler_wrapper/README.md b/compiler_wrapper/README.md index bb63798a..e858038e 100644 --- a/compiler_wrapper/README.md +++ b/compiler_wrapper/README.md @@ -18,7 +18,7 @@ To test updates to the wrapper locally: Run `install_compiler_wrapper.sh` to install the new wrapper in the chroot: ``` -(chroot) ~/trunk/src/third_party/toolchain-utils/compiler_wrapper/install_compiler_wrapper.sh +(chroot) ~/chromiumos/src/third_party/toolchain-utils/compiler_wrapper/install_compiler_wrapper.sh ``` Then perform the tests, e.g. build with the new compiler. @@ -39,14 +39,14 @@ Rename chromiumos-overlay/sys-devel/llvm/llvm-${VERSION}.ebuild to the next revision number. For example, if the current version is 11.0_pre394483_p20200618-r2: ``` -(chroot) cd ~/trunk/src/third_party/chromiumos-overlay +(chroot) cd ~/chromiumos/src/third_party/chromiumos-overlay (chroot) git mv llvm-11.0_pre394483_p20200618-r2.ebuild llvm-11.0_pre394483_p20200618-r3.ebuild ``` Rename chromiumos-overlay/sys-devel/gcc/gcc-${VERSION}.ebuild to the next revision number. For example, if the current version is 10.2.0-r3: ``` -(chroot) cd ~/trunk/src/third_party/chromiumos-overlay +(chroot) cd ~/chromiumos/src/third_party/chromiumos-overlay (chroot) git mv sys-devel/gcc/gcc-10.2.0-r3.ebuild sys-devel/gcc/gcc-10.2.0-r4.ebuild ``` @@ -96,8 +96,9 @@ error-prone; compile commands may rely on env vars, they may be done within rely on intermediate state, etc. Because of the usefulness of these crash reports, our wrapper supports crashing -Clang even on files that ordinarily don't cause Clang to crash. For various -reasons (b/236736327), this support currently requires rebuilding and -redeploying the wrapper in order to work. That said, this could be a valuable -tool for devs interested in creating a self-contained reproducer without having -to manually reproduce the environment in which a particular build was performed. +Clang even on files that ordinarily don't cause Clang to crash. This requires +rebuilding and redeploying the wrapper (comments on b/236736327 explain why). +That said, this could be a valuable tool for devs interested in creating a +self-contained reproducer without having to manually reproduce the environment +in which a particular build was performed. See <crash_builds.go> +for instructions for how to use this functionality. diff --git a/compiler_wrapper/build.py b/compiler_wrapper/build.py index 930c2cfb..58822880 100755 --- a/compiler_wrapper/build.py +++ b/compiler_wrapper/build.py @@ -49,7 +49,7 @@ def parse_args(): return args -def calc_go_args(args, version, build_dir): +def calc_go_args(args, version, build_dir, output_file): # These seem unnecessary, and might lead to breakages with Go's ldflag # parsing. Don't allow them. if "'" in version: @@ -89,7 +89,7 @@ def calc_go_args(args, version, build_dir): "go", "build", "-o", - os.path.abspath(args.output_file), + output_file, "-ldflags", " ".join(ldFlags), ] + extra_args @@ -119,10 +119,23 @@ def main(): version += args.version_suffix # Note: Go does not support using absolute package names. # So we run go inside the directory of the the build file. - sys.exit( - subprocess.call(calc_go_args(args, version, build_dir), cwd=build_dir) + output_file = os.path.abspath(args.output_file) + subprocess.check_call( + calc_go_args(args, version, build_dir, output_file), cwd=build_dir ) + # b/203821449: we're occasionally seeing very small (and non-functional) + # compiler-wrapper binaries on SDK builds. To help narrow down why, add a + # size check here. Locally, the wrapper is 1.9MB, so warning on <1MB + # shouldn't flag false-positives. + size = os.path.getsize(output_file) + min_size_bytes = 1024 * 1024 + if size < min_size_bytes: + raise ValueError( + f"Compiler wrapper is {size:,} bytes; expected at " + f"least {min_size_bytes:,}" + ) + if __name__ == "__main__": main() diff --git a/compiler_wrapper/clang_flags.go b/compiler_wrapper/clang_flags.go index 1c45935e..a38e597a 100644 --- a/compiler_wrapper/clang_flags.go +++ b/compiler_wrapper/clang_flags.go @@ -36,18 +36,30 @@ func processClangFlags(builder *commandBuilder) error { clangDir = filepath.Dir(clangDir) } + var languageOverriden = false + for _, arg := range builder.args { + // Reading stdin with "-" requires either -E (which defaults to C) or -x + if arg.value == "-x" || arg.value == "-" { + languageOverriden = true + } + } + clangBasename := "clang" if strings.HasSuffix(builder.target.compiler, "++") { clangBasename = "clang++" + // If a package wants to specify the language then it can set the standard too. + if !languageOverriden { + builder.addPreUserArgs(builder.cfg.cppFlags...) + } } - // GCC flags to remove from the clang command line. - // TODO: Once clang supports GCC compatibility mode, remove - // these checks. - // + // Unsupported flags to remove from the clang command line. // Use of -Qunused-arguments allows this set to be small, just those // that clang still warns about. - unsupported := make(map[string]bool) + unsupported := map[string]bool{ + "-Xcompiler": true, + "-avoid-version": true, + } unsupportedPrefixes := []string{"-Wstrict-aliasing=", "-finline-limit="} diff --git a/compiler_wrapper/clang_flags_test.go b/compiler_wrapper/clang_flags_test.go index 08e8a8dc..8b3f5a24 100644 --- a/compiler_wrapper/clang_flags_test.go +++ b/compiler_wrapper/clang_flags_test.go @@ -33,6 +33,36 @@ func TestClangBasename(t *testing.T) { }) } +func TestAppendCppFlags(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + ctx.cfg.cppFlags = append(ctx.cfg.cppFlags, "cppOnlyFlag") + // C++ only flags are disabled on clang. + clangCmd := ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand("./x86_64-cros-linux-gnu-clang", mainCc))) + if err := verifyArgCount(clangCmd, 0, "cppOnlyFlag"); err != nil { + t.Error(err) + } + // C++ only flags are enabled on clang++. + clangPlusPlusCmd := ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand("./x86_64-cros-linux-gnu-clang++", mainCc))) + if err := verifyArgCount(clangPlusPlusCmd, 1, "cppOnlyFlag"); err != nil { + t.Error(err) + } + // C++ only flags are disabled with -x. + clangPlusPlusWithXCmd := ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand("./x86_64-cros-linux-gnu-clang++", "-x", "c", mainCc))) + if err := verifyArgCount(clangPlusPlusWithXCmd, 0, "cppOnlyFlag"); err != nil { + t.Error(err) + } + // C++ only flags are disabled with " - " . + clangPlusPlusWithStdin := ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand("./x86_64-cros-linux-gnu-clang++", "-", mainCc))) + if err := verifyArgCount(clangPlusPlusWithStdin, 0, "cppOnlyFlag"); err != nil { + t.Error(err) + } + }) +} + func TestClangPathGivenClangEnv(t *testing.T) { withTestContext(t, func(ctx *testContext) { ctx.env = []string{"CLANG=/a/b/clang"} @@ -310,3 +340,18 @@ func TestClangLinkerPathRelativeToRootDir(t *testing.T) { } }) } + +func TestFilterUnsupportedFlags(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand(clangX86_64, "-Xcompiler", mainCc))) + if err := verifyArgCount(cmd, 0, "-Xcompiler"); err != nil { + t.Error(err) + } + cmd = ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand(clangX86_64, "-avoid-version", mainCc))) + if err := verifyArgCount(cmd, 0, "-avoid-version"); err != nil { + t.Error(err) + } + }) +} diff --git a/compiler_wrapper/clang_tidy_flag.go b/compiler_wrapper/clang_tidy_flag.go index b19976d2..9722b39e 100644 --- a/compiler_wrapper/clang_tidy_flag.go +++ b/compiler_wrapper/clang_tidy_flag.go @@ -98,7 +98,8 @@ func calcClangTidyInvocation(env env, clangCmd *command, cSrcFile string, tidyFl }, nil } -func runClangTidyForTricium(env env, clangCmd *command, cSrcFile, fixesDir string, extraTidyFlags []string, crashArtifactsDir string) error { +func runClangTidyForTricium(env env, clangCmd *command, cSrcFile string, extraTidyFlags []string, crashArtifactsDir string) error { + fixesDir := filepath.Join(getCompilerArtifactsDir(env), "linting-output", "clang-tidy") if err := os.MkdirAll(fixesDir, 0777); err != nil { return fmt.Errorf("creating fixes directory at %q: %v", fixesDir, err) } @@ -230,12 +231,3 @@ func runClangTidy(env env, clangCmd *command, cSrcFile string, extraTidyFlags [] } return err } - -func hasAtLeastOneSuffix(s string, suffixes []string) bool { - for _, suffix := range suffixes { - if strings.HasSuffix(s, suffix) { - return true - } - } - return false -} diff --git a/compiler_wrapper/clang_tidy_flag_test.go b/compiler_wrapper/clang_tidy_flag_test.go index 73dec25f..b32d0d63 100644 --- a/compiler_wrapper/clang_tidy_flag_test.go +++ b/compiler_wrapper/clang_tidy_flag_test.go @@ -274,8 +274,9 @@ func TestPartiallyOmitGomaWithClangTidy(t *testing.T) { } func TestTriciumClangTidyIsProperlyDetectedFromEnv(t *testing.T) { - withClangTidyTestContext(t, func(ctx *testContext) { - ctx.env = []string{"WITH_TIDY=tricium"} + withClangTidyTriciumTestContext(t, func(ctx *testContext) { + artifactsBase := strings.Split(ctx.env[0], "=")[1] + outputDir := artifactsBase + "/linting-output/clang-tidy" ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { switch ctx.cmdCount { case 1: @@ -292,8 +293,9 @@ func TestTriciumClangTidyIsProperlyDetectedFromEnv(t *testing.T) { for _, arg := range cmd.Args { if path := strings.TrimPrefix(arg, "--export-fixes="); path != arg { hasFixesFile = true - if !strings.HasPrefix(path, ctx.cfg.triciumNitsDir+"/") { - t.Errorf("fixes file was %q; expected it to be in %q", path, ctx.cfg.triciumNitsDir) + + if !strings.HasPrefix(path, outputDir) { + t.Errorf("fixes file was %q; expected it to be in %q", path, outputDir) } break } @@ -320,8 +322,7 @@ func TestTriciumClangTidyIsProperlyDetectedFromEnv(t *testing.T) { } func TestTriciumClangTidySkipsProtobufFiles(t *testing.T) { - withClangTidyTestContext(t, func(ctx *testContext) { - ctx.env = []string{"WITH_TIDY=tricium"} + withClangTidyTriciumTestContext(t, func(ctx *testContext) { cmd := ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangX86_64, mainCc+".pb.cc"))) if ctx.cmdCount != 1 { @@ -365,8 +366,7 @@ func testClangTidyFiltersClangTidySpecificFlagsWithPresetEnv(t *testing.T, ctx * } func TestClangTidyFiltersClangTidySpecificFlagsForTricium(t *testing.T) { - withClangTidyTestContext(t, func(ctx *testContext) { - ctx.env = []string{"WITH_TIDY=tricium"} + withClangTidyTriciumTestContext(t, func(ctx *testContext) { testClangTidyFiltersClangTidySpecificFlagsWithPresetEnv(t, ctx) }) } @@ -378,7 +378,7 @@ func TestClangTidyFiltersClangTidySpecificFlags(t *testing.T) { } func TestClangTidyFlagsAreFilteredFromGccInvocations(t *testing.T) { - withTestContext(t, func(ctx *testContext) { + withClangTidyTestContext(t, func(ctx *testContext) { cmd := ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(gccX86_64, mainCc, "-clang-tidy-flag=--foo"))) if err := verifyArgCount(cmd, 0, ".*--foo.*"); err != nil { t.Error(err) @@ -387,8 +387,7 @@ func TestClangTidyFlagsAreFilteredFromGccInvocations(t *testing.T) { } func TestTriciumReportsClangTidyCrashesGracefully(t *testing.T) { - withClangTidyTestContext(t, func(ctx *testContext) { - ctx.env = []string{"WITH_TIDY=tricium"} + withClangTidyTriciumTestContext(t, func(ctx *testContext) { ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { switch ctx.cmdCount { case 1: @@ -443,9 +442,24 @@ func TestTriciumReportsClangTidyCrashesGracefully(t *testing.T) { }) } -func withClangTidyTestContext(t *testing.T, work func(ctx *testContext)) { +func withClangTidyTestContextBaseDir(t *testing.T, work func(ctx *testContext)) { withTestContext(t, func(ctx *testContext) { - ctx.env = []string{"WITH_TIDY=1"} + artifactDir := t.TempDir() + ctx.env = []string{"CROS_ARTIFACTS_TMP_DIR=" + artifactDir} + work(ctx) + }) +} + +func withClangTidyTestContext(t *testing.T, work func(ctx *testContext)) { + withClangTidyTestContextBaseDir(t, func(ctx *testContext) { + ctx.env = append(ctx.env, "WITH_TIDY=1") + work(ctx) + }) +} + +func withClangTidyTriciumTestContext(t *testing.T, work func(ctx *testContext)) { + withClangTidyTestContextBaseDir(t, func(ctx *testContext) { + ctx.env = append(ctx.env, "WITH_TIDY=tricium") work(ctx) }) } diff --git a/compiler_wrapper/command.go b/compiler_wrapper/command.go index e2a5176d..d19c903a 100644 --- a/compiler_wrapper/command.go +++ b/compiler_wrapper/command.go @@ -93,7 +93,7 @@ func resolveAgainstPathEnv(env env, cmd string) (string, error) { return resolvedPath, nil } } - return "", fmt.Errorf("Couldn't find cmd %q in path", cmd) + return "", fmt.Errorf("couldn't find cmd %q in path", cmd) } func getAbsCmdPath(env env, cmd *command) string { diff --git a/compiler_wrapper/compiler_wrapper.go b/compiler_wrapper/compiler_wrapper.go index dcaada99..bb144881 100644 --- a/compiler_wrapper/compiler_wrapper.go +++ b/compiler_wrapper/compiler_wrapper.go @@ -94,7 +94,7 @@ func runAndroidClangTidy(env env, cmd *command) error { if !errors.Is(err, context.DeadlineExceeded) { // When used time is over half of TIDY_TIMEOUT, give a warning. // These warnings allow users to fix slow jobs before they get worse. - usedSeconds := int(time.Now().Sub(startTime) / time.Second) + usedSeconds := int(time.Since(startTime) / time.Second) if usedSeconds > seconds/2 { warning := "%s:1:1: warning: clang-tidy used %d seconds.\n" fmt.Fprintf(env.stdout(), warning, getSourceFile(), usedSeconds) @@ -110,6 +110,7 @@ func runAndroidClangTidy(env env, cmd *command) error { } func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int, err error) { + if err := checkUnsupportedFlags(inputCmd); err != nil { return 0, err } @@ -150,7 +151,7 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int return 0, newErrorwithSourceLocf("unsupported compiler: %s", mainBuilder.target.compiler) } } else { - cSrcFile, tidyFlags, tidyMode := processClangTidyFlags(mainBuilder) + _, tidyFlags, tidyMode := processClangTidyFlags(mainBuilder) cSrcFile, iwyuFlags, iwyuMode := processIWYUFlags(mainBuilder) if mainBuilder.target.compilerType == clangType { err := prepareClangCommand(mainBuilder) @@ -159,14 +160,17 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int } if tidyMode != tidyModeNone { allowCCache = false + // Remove and ignore goma flags. + _, err := removeOneUserCmdlineFlagWithValue(mainBuilder, "--gomacc-path") + if err != nil && err != errNoSuchCmdlineArg { + return 0, err + } + clangCmdWithoutRemoteBuildAndCCache := mainBuilder.build() - var err error + switch tidyMode { case tidyModeTricium: - if cfg.triciumNitsDir == "" { - return 0, newErrorwithSourceLocf("tricium linting was requested, but no nits directory is configured") - } - err = runClangTidyForTricium(env, clangCmdWithoutRemoteBuildAndCCache, cSrcFile, cfg.triciumNitsDir, tidyFlags, cfg.crashArtifactsDir) + err = runClangTidyForTricium(env, clangCmdWithoutRemoteBuildAndCCache, cSrcFile, tidyFlags, cfg.crashArtifactsDir) case tidyModeAll: err = runClangTidy(env, clangCmdWithoutRemoteBuildAndCCache, cSrcFile, tidyFlags) default: @@ -180,7 +184,7 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int if iwyuMode != iwyuModeNone { if iwyuMode == iwyuModeError { - panic(fmt.Sprintf("Unknown IWYU mode")) + panic("Unknown IWYU mode") } allowCCache = false @@ -328,10 +332,26 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int } } +// TODO(b/288411201): Add -D_FORTIFY_SOURCE=2 to args if -D_FORITFY_SOURCE=3 is not present. +// This makes migrating to -D_FORTIFY_SOURCE=3 _way_ easier, since the wrapper's implicit +// -D_FORTIFY_SOURCE=2 can be ignored. +func addPreUserFortifyFlag(builder *commandBuilder) { + for _, arg := range builder.args { + if arg.value == "-D_FORTIFY_SOURCE=3" { + return + } + } + + builder.addPreUserArgs("-D_FORTIFY_SOURCE=2") +} + func prepareClangCommand(builder *commandBuilder) (err error) { if !builder.cfg.isHostWrapper { processSysrootFlag(builder) } + if builder.cfg.isHardened { + addPreUserFortifyFlag(builder) + } builder.addPreUserArgs(builder.cfg.clangFlags...) if builder.cfg.crashArtifactsDir != "" { builder.addPreUserArgs("-fcrash-diagnostics-dir=" + builder.cfg.crashArtifactsDir) @@ -357,6 +377,9 @@ func calcGccCommand(enableRusage bool, builder *commandBuilder) (bool, *command, if !builder.cfg.isHostWrapper { processSysrootFlag(builder) } + if builder.cfg.isHardened { + addPreUserFortifyFlag(builder) + } builder.addPreUserArgs(builder.cfg.gccFlags...) calcCommonPreUserArgs(builder) processGccFlags(builder) diff --git a/compiler_wrapper/config.go b/compiler_wrapper/config.go index 6c3fcf52..bcf73d79 100644 --- a/compiler_wrapper/config.go +++ b/compiler_wrapper/config.go @@ -25,17 +25,18 @@ type config struct { // Flags to add to clang only, AFTER user flags (cannot be overridden // by the user). clangPostFlags []string + // Flags to be used only for C++ (not used to compile C code) + cppFlags []string // Toolchain root path relative to the wrapper binary. clangRootRelPath string gccRootRelPath string // Directory to store errors that were prevented with -Wno-error. newWarningsDir string - // Directory to store nits in when using `WITH_TIDY=tricium`. - triciumNitsDir string // Directory to store crash artifacts in. crashArtifactsDir string // Version. Only exposed via -print-config. - version string + version string + isHardened bool } // Version can be set via a linker flag. @@ -113,32 +114,47 @@ func crosCommonClangFlags() []string { "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", + } +} + +func crosCommonCppFlags() []string { + return []string{ + "-std=gnu++14", } } func crosCommonClangPostFlags() []string { // Temporarily disable Wdeprecated-copy. b/191479033 + // Temporarily disabled Wno-array-parameter. b/262076232 return []string{ + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", } } // Full hardening. // Temporarily disable function splitting because of chromium:434751. var crosHardenedConfig = config{ + isHardened: true, clangRootRelPath: "../..", gccRootRelPath: "../../../../..", // Pass "-fcommon" till the packages are fixed to work with new clang/gcc @@ -146,7 +162,6 @@ var crosHardenedConfig = config{ commonFlags: []string{ "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", }, gccFlags: []string{ @@ -165,11 +180,10 @@ var crosHardenedConfig = config{ "-fno-addrsig", "-fuse-ld=lld", "-ftrivial-auto-var-init=zero", - "-enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang", ), clangPostFlags: crosCommonClangPostFlags(), - newWarningsDir: "/tmp/fatal_clang_warnings", - triciumNitsDir: "/tmp/linting_output/clang-tidy", + cppFlags: crosCommonCppFlags(), + newWarningsDir: "fatal_clang_warnings", crashArtifactsDir: "/tmp/clang_crash_diagnostics", } @@ -190,8 +204,8 @@ var crosNonHardenedConfig = config{ "-Wno-section", ), clangPostFlags: crosCommonClangPostFlags(), - newWarningsDir: "/tmp/fatal_clang_warnings", - triciumNitsDir: "/tmp/linting_output/clang-tidy", + cppFlags: crosCommonCppFlags(), + newWarningsDir: "fatal_clang_warnings", crashArtifactsDir: "/tmp/clang_crash_diagnostics", } @@ -220,8 +234,8 @@ var crosHostConfig = config{ ), // Temporarily disable Wdeprecated-copy. b/191479033 clangPostFlags: crosCommonClangPostFlags(), - newWarningsDir: "/tmp/fatal_clang_warnings", - triciumNitsDir: "/tmp/linting_output/clang-tidy", + cppFlags: crosCommonCppFlags(), + newWarningsDir: "fatal_clang_warnings", crashArtifactsDir: "/tmp/clang_crash_diagnostics", } @@ -234,7 +248,7 @@ var androidConfig = config{ gccFlags: []string{}, clangFlags: []string{}, clangPostFlags: []string{}, + cppFlags: []string{}, newWarningsDir: "", - triciumNitsDir: "", crashArtifactsDir: "", } diff --git a/compiler_wrapper/config_test.go b/compiler_wrapper/config_test.go index 0e6b44c5..47432856 100644 --- a/compiler_wrapper/config_test.go +++ b/compiler_wrapper/config_test.go @@ -118,12 +118,7 @@ func TestRealConfigWithConfigNameFlag(t *testing.T) { } func isSysrootHardened(cfg *config) bool { - for _, arg := range cfg.commonFlags { - if arg == "-D_FORTIFY_SOURCE=2" { - return true - } - } - return false + return cfg.isHardened } func resetGlobals() { diff --git a/compiler_wrapper/disable_werror_flag.go b/compiler_wrapper/disable_werror_flag.go index 17077671..91bc5845 100644 --- a/compiler_wrapper/disable_werror_flag.go +++ b/compiler_wrapper/disable_werror_flag.go @@ -18,6 +18,10 @@ import ( const numWErrorEstimate = 30 +func getForceDisableWerrorDir(env env, cfg *config) string { + return path.Join(getCompilerArtifactsDir(env), cfg.newWarningsDir) +} + func shouldForceDisableWerror(env env, cfg *config, ty compilerType) bool { if cfg.isAndroidWrapper { return cfg.useLlvmNext @@ -178,20 +182,21 @@ func doubleBuildWithWNoError(env env, cfg *config, originalCmd *command) (exitCo oldMask := env.umask(0) defer env.umask(oldMask) + warningsDir := getForceDisableWerrorDir(env, cfg) // Allow root and regular users to write to this without issue. - if err := os.MkdirAll(cfg.newWarningsDir, 0777); err != nil { - return 0, wrapErrorwithSourceLocf(err, "error creating warnings directory %s", cfg.newWarningsDir) + if err := os.MkdirAll(warningsDir, 0777); err != nil { + return 0, wrapErrorwithSourceLocf(err, "error creating warnings directory %s", warningsDir) } // Have some tag to show that files aren't fully written. It would be sad if // an interrupted build (or out of disk space, or similar) caused tools to // have to be overly-defensive. - incompleteSuffix := ".incomplete" + const incompleteSuffix = ".incomplete" // Coming up with a consistent name for this is difficult (compiler command's // SHA can clash in the case of identically named files in different // directories, or similar); let's use a random one. - tmpFile, err := ioutil.TempFile(cfg.newWarningsDir, "warnings_report*.json"+incompleteSuffix) + tmpFile, err := ioutil.TempFile(warningsDir, "warnings_report*.json"+incompleteSuffix) if err != nil { return 0, wrapErrorwithSourceLocf(err, "error creating warnings file") } diff --git a/compiler_wrapper/disable_werror_flag_test.go b/compiler_wrapper/disable_werror_flag_test.go index d3be921d..9d3b4aba 100644 --- a/compiler_wrapper/disable_werror_flag_test.go +++ b/compiler_wrapper/disable_werror_flag_test.go @@ -11,6 +11,7 @@ import ( "io" "io/ioutil" "os" + "path" "path/filepath" "strings" "testing" @@ -320,13 +321,18 @@ func withForceDisableWErrorTestContext(t *testing.T, work func(ctx *testContext) withTestContext(t, func(ctx *testContext) { ctx.NoteTestWritesToUmask() - ctx.env = []string{"FORCE_DISABLE_WERROR=1"} + ctx.cfg.newWarningsDir = "new_warnings" + ctx.env = []string{ + "FORCE_DISABLE_WERROR=1", + artifactsTmpDirEnvName + "=" + path.Join(ctx.tempDir, "artifacts"), + } work(ctx) }) } func readLoggedWarnings(ctx *testContext) *warningsJSONData { - files, err := ioutil.ReadDir(ctx.cfg.newWarningsDir) + warningsDir := getForceDisableWerrorDir(ctx, ctx.cfg) + files, err := ioutil.ReadDir(warningsDir) if err != nil { if _, ok := err.(*os.PathError); ok { return nil @@ -336,7 +342,7 @@ func readLoggedWarnings(ctx *testContext) *warningsJSONData { if len(files) != 1 { ctx.t.Fatalf("expected 1 warning log file. Got: %s", files) } - data, err := ioutil.ReadFile(filepath.Join(ctx.cfg.newWarningsDir, files[0].Name())) + data, err := ioutil.ReadFile(filepath.Join(warningsDir, files[0].Name())) if err != nil { ctx.t.Fatal(err) } @@ -373,8 +379,9 @@ func TestDoubleBuildWerrorChmodsThingsAppropriately(t *testing.T) { t.Fatalf("expected 2 calls. Got: %d", ctx.cmdCount) } - t.Logf("Warnings dir is at %q", ctx.cfg.newWarningsDir) - warningsDir, err := os.Open(ctx.cfg.newWarningsDir) + warningsDirPath := getForceDisableWerrorDir(ctx, ctx.cfg) + t.Logf("Warnings dir is at %q", warningsDirPath) + warningsDir, err := os.Open(warningsDirPath) if err != nil { t.Fatalf("failed to open the new warnings dir: %v", err) } diff --git a/compiler_wrapper/env.go b/compiler_wrapper/env.go index 6b25d960..5ec8ea03 100644 --- a/compiler_wrapper/env.go +++ b/compiler_wrapper/env.go @@ -14,6 +14,8 @@ import ( "time" ) +const artifactsTmpDirEnvName = "CROS_ARTIFACTS_TMP_DIR" + type env interface { umask(int) int getenv(key string) (string, bool) @@ -121,7 +123,7 @@ func (env *commandRecordingEnv) exec(cmd *command) error { } func (env *commandRecordingEnv) runWithTimeout(cmd *command, duration time.Duration) error { - return env.runWithTimeout(cmd, duration) + return runCmdWithTimeout(env, cmd, duration) } func (env *commandRecordingEnv) run(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { @@ -143,7 +145,7 @@ type printingEnv struct { env } -var _env = (*printingEnv)(nil) +var _ env = (*printingEnv)(nil) func (env *printingEnv) exec(cmd *command) error { printCmd(env, cmd) @@ -171,3 +173,14 @@ func printCmd(env env, cmd *command) { } io.WriteString(env.stderr(), "\n") } + +func getCompilerArtifactsDir(env env) string { + const defaultArtifactDir = "/tmp" + value, _ := env.getenv(artifactsTmpDirEnvName) + if value == "" { + fmt.Fprintf(env.stdout(), "$%s is not set, artifacts will be written to %s", artifactsTmpDirEnvName, defaultArtifactDir) + return defaultArtifactDir + } + return value + +} diff --git a/compiler_wrapper/env_test.go b/compiler_wrapper/env_test.go index 6b00a8b5..c10942de 100644 --- a/compiler_wrapper/env_test.go +++ b/compiler_wrapper/env_test.go @@ -226,6 +226,9 @@ func TestNewProcessEnvResolvesPwdAwayProperly(t *testing.T) { os.Unsetenv(envPwd) initialWd, err := os.Getwd() + if err != nil { + t.Fatalf("Failed getting working directory: %v", err) + } if initialWd == "/proc/self/cwd" { t.Fatalf("Working directory should never be %q when env is unset", initialWd) } diff --git a/compiler_wrapper/iwyu_flag.go b/compiler_wrapper/iwyu_flag.go index d13d114d..5788d8c7 100644 --- a/compiler_wrapper/iwyu_flag.go +++ b/compiler_wrapper/iwyu_flag.go @@ -8,14 +8,13 @@ import ( "bufio" "bytes" "fmt" + "os" "path/filepath" "strings" ) type useIWYUMode int -const iwyuCrashSubstring = "PLEASE submit a bug report" - const ( iwyuModeNone useIWYUMode = iota iwyuModeAll @@ -135,21 +134,24 @@ func runIWYU(env env, clangCmd *command, cSrcFile string, extraIWYUFlags []strin fmt.Fprintln(env.stderr(), "include-what-you-use failed") } - var path strings.Builder - path.WriteString(strings.TrimSuffix(iwyuCmd.Path, "include-what-you-use")) - path.WriteString("fix_includes.py") - fixIncludesCmd := &command{ - Path: path.String(), - Args: []string{"--nocomment"}, - EnvUpdates: clangCmd.EnvUpdates, + iwyuDir := filepath.Join(getCompilerArtifactsDir(env), "linting-output", "iwyu") + if err := os.MkdirAll(iwyuDir, 0777); err != nil { + return fmt.Errorf("creating fixes directory at %q: %v", iwyuDir, err) } - exitCode, err = wrapSubprocessErrorWithSourceLoc(fixIncludesCmd, - env.run(fixIncludesCmd, strings.NewReader(stderrMessage), env.stdout(), env.stderr())) - if err == nil && exitCode != 0 { - // Note: We continue on purpose when include-what-you-use fails - // to maintain compatibility with the previous wrapper. - fmt.Fprint(env.stderr(), "include-what-you-use failed") + f, err := os.CreateTemp(iwyuDir, "*.out") + if err != nil { + return fmt.Errorf("making output file for iwyu: %v", err) + } + writer := bufio.NewWriter(f) + if _, err := writer.WriteString(stderrMessage); err != nil { + return fmt.Errorf("writing output file for iwyu: %v", err) + } + if err := writer.Flush(); err != nil { + return fmt.Errorf("flushing output file buffer for iwyu: %v", err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("finalizing output file for iwyu: %v", err) } return err diff --git a/compiler_wrapper/iwyu_flag_test.go b/compiler_wrapper/iwyu_flag_test.go index 76135944..e2db3b48 100644 --- a/compiler_wrapper/iwyu_flag_test.go +++ b/compiler_wrapper/iwyu_flag_test.go @@ -84,7 +84,7 @@ func TestUseIWYUBasedOnFileExtension(t *testing.T) { ctx.cmdCount = 0 ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangX86_64, tt.args...))) - if ctx.cmdCount > 1 && !tt.iwyu { + if ctx.cmdCount == 2 && !tt.iwyu { t.Errorf("expected a call to iwyu but got none for args %s", tt.args) } if ctx.cmdCount == 1 && tt.iwyu { @@ -129,7 +129,8 @@ func TestIWYUFiltersIWYUFlags(t *testing.T) { func withIWYUTestContext(t *testing.T, work func(ctx *testContext)) { withTestContext(t, func(ctx *testContext) { - ctx.env = []string{"WITH_IWYU=1"} + artifactDir := t.TempDir() + ctx.env = []string{"WITH_IWYU=1", "CROS_ARTIFACTS_TMP_DIR=" + artifactDir} work(ctx) }) } diff --git a/compiler_wrapper/main.go b/compiler_wrapper/main.go index 8cfa4364..7d1e3f49 100644 --- a/compiler_wrapper/main.go +++ b/compiler_wrapper/main.go @@ -3,22 +3,22 @@ // found in the LICENSE file. // This binary requires the following linker variables: -// - main.UseCCache: Whether to use ccache. -// - main.ConfigName: Name of the configuration to use. -// See config.go for the supported values. +// - main.UseCCache: Whether to use ccache. +// - main.ConfigName: Name of the configuration to use. +// See config.go for the supported values. // // The script ./build simplifies the call to `go build`. // E.g. ./build --use_ccache=true --config=cros.hardened will build a // binary that uses the ccache for ChromeOS with hardened flags. // // Test arguments: -// - updategolden: To update the golden results for the wrapper. Without it, -// the tests will verify that the wrapper output matches the goldens. -// - rungolden: To filter the golden tests by a regex for the wrapper env, path and args. +// - updategolden: To update the golden results for the wrapper. Without it, +// the tests will verify that the wrapper output matches the goldens. +// - rungolden: To filter the golden tests by a regex for the wrapper env, path and args. // // Examples: -// - run all tests in isolation: -// go test third_party/toolchain-utils/compiler_wrapper/ -v +// - run all tests in isolation: +// go test third_party/toolchain-utils/compiler_wrapper/ -v package main import ( diff --git a/compiler_wrapper/rusage_flag.go b/compiler_wrapper/rusage_flag.go index ed59b11e..899e4da8 100644 --- a/compiler_wrapper/rusage_flag.go +++ b/compiler_wrapper/rusage_flag.go @@ -62,7 +62,7 @@ func removeRusageFromCommand(compilerCmd *command) *command { } // maybeCaptureRusage captures rusage for execution of action() -// unless action returns an error or logFileName is "" +// unless action returns an error or logFileName is "" // a function is returned which saves the rusage log data at logFileName unless logFileName is "" // an error is returned if action returns an error, or rusage commands in syscall fails func maybeCaptureRusage(env env, compilerCmd *command, action func(willLogRusage bool) error) (maybeCommitToFile func(exitCode int) error, err error) { diff --git a/compiler_wrapper/sanitizer_flags.go b/compiler_wrapper/sanitizer_flags.go index 58312cc4..dd3529ed 100644 --- a/compiler_wrapper/sanitizer_flags.go +++ b/compiler_wrapper/sanitizer_flags.go @@ -47,6 +47,15 @@ func isInvasiveSanitizerFlag(flag string) bool { return false } +// Returns whether the flag given enables FORTIFY. Notably, this should return false if a flag +// disables FORTIFY. +func isFortifyEnableFlag(flag string) bool { + prefix := "-D_FORTIFY_SOURCE=" + // At the time of writing, -D_FORTIFY_SOURCE has the valid values 0, 1, 2, and 3. Seems + // unlikely to go past 9, so don't handle past 9. + return strings.HasPrefix(flag, prefix) && len(flag) == len(prefix)+1 && flag[len(prefix)] != '0' +} + func processSanitizerFlags(builder *commandBuilder) { hasSanitizeFlags := false // TODO: This doesn't take -fno-sanitize flags into account. This doesn't seem to be an @@ -64,18 +73,21 @@ func processSanitizerFlags(builder *commandBuilder) { // Flags not supported by sanitizers (ASan etc.) unsupportedSanitizerFlags := map[string]bool{ - "-D_FORTIFY_SOURCE=1": true, - "-D_FORTIFY_SOURCE=2": true, - "-Wl,--no-undefined": true, - "-Wl,-z,defs": true, + "-Wl,--no-undefined": true, + "-Wl,-z,defs": true, } builder.transformArgs(func(arg builderArg) string { // TODO: This is a bug in the old wrapper to not filter // non user args for gcc. Fix this once we don't compare to the old wrapper anymore. - if (builder.target.compilerType != gccType || arg.fromUser) && - unsupportedSanitizerFlags[arg.value] { - return "" + linkerDefinedFlag := ",-z,defs" + if builder.target.compilerType != gccType || arg.fromUser { + if unsupportedSanitizerFlags[arg.value] || isFortifyEnableFlag(arg.value) { + return "" + } + if strings.Contains(arg.value, linkerDefinedFlag) { + return strings.ReplaceAll(arg.value, linkerDefinedFlag, "") + } } return arg.value }) diff --git a/compiler_wrapper/sanitizer_flags_test.go b/compiler_wrapper/sanitizer_flags_test.go index b4b1fd83..c81d95c0 100644 --- a/compiler_wrapper/sanitizer_flags_test.go +++ b/compiler_wrapper/sanitizer_flags_test.go @@ -45,6 +45,12 @@ func TestFilterUnsupportedSanitizerFlagsIfSanitizeGiven(t *testing.T) { } cmd = ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-fsanitize=kernel-address", "-Wl,-z,defs,relro", mainCc))) + if err := verifyArgCount(cmd, 1, "-Wl,relro"); err != nil { + t.Error(err) + } + + cmd = ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(gccX86_64, "-fsanitize=kernel-address", "-Wl,-z -Wl,defs", mainCc))) if err := verifyArgCount(cmd, 0, "-Wl,-z"); err != nil { t.Error(err) @@ -67,6 +73,42 @@ func TestFilterUnsupportedSanitizerFlagsIfSanitizeGiven(t *testing.T) { }) } +func TestAllFortifyEnableFlagsAreRemoved(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + flagsToRemove := []string{ + "-D_FORTIFY_SOURCE=1", + "-D_FORTIFY_SOURCE=2", + "-D_FORTIFY_SOURCE=3", + } + for _, fortifyFlag := range flagsToRemove { + ctx.cfg.commonFlags = []string{fortifyFlag} + cmd := ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand(clangX86_64, "-fsanitize=kernel-address", mainCc))) + if err := verifyArgCount(cmd, 0, fortifyFlag); err != nil { + t.Errorf("Verifying FORTIFY flag %q is removed: %v", fortifyFlag, err) + } + } + }) +} + +func TestFortifyDisableFlagsAreKept(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + flagsToKeep := []string{ + "-D_FORTIFY_SOURCE", + "-D_FORTIFY_SOURCE=", + "-D_FORTIFY_SOURCE=0", + } + for _, fortifyFlag := range flagsToKeep { + ctx.cfg.commonFlags = []string{fortifyFlag} + cmd := ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand(clangX86_64, "-fsanitize=kernel-address", mainCc))) + if err := verifyArgCount(cmd, 1, fortifyFlag); err != nil { + t.Errorf("Verifying FORTIFY flag %q is kept: %v", fortifyFlag, err) + } + } + }) +} + func TestFilterUnsupportedDefaultSanitizerFlagsIfSanitizeGivenForClang(t *testing.T) { withTestContext(t, func(ctx *testContext) { ctx.cfg.commonFlags = []string{"-D_FORTIFY_SOURCE=1"} diff --git a/compiler_wrapper/sysroot_flag.go b/compiler_wrapper/sysroot_flag.go index 597153a9..0256abcd 100644 --- a/compiler_wrapper/sysroot_flag.go +++ b/compiler_wrapper/sysroot_flag.go @@ -9,19 +9,37 @@ import ( "strings" ) +const skipSysrootAutodetectionFlag = "--cros-skip-wrapper-sysroot-autodetection" + func processSysrootFlag(builder *commandBuilder) { + hadSkipSysrootMagicFlag := false fromUser := false userSysroot := "" - for _, arg := range builder.args { - if arg.fromUser && strings.HasPrefix(arg.value, "--sysroot=") { + builder.transformArgs(func(arg builderArg) string { + switch { + // In rare cases (e.g., glibc), we want all sysroot autodetection logic to be + // disabled. This flag can be passed to disable that. + case arg.value == skipSysrootAutodetectionFlag: + hadSkipSysrootMagicFlag = true + return "" + + case arg.fromUser && strings.HasPrefix(arg.value, "--sysroot="): fromUser = true sysrootArg := strings.Split(arg.value, "=") if len(sysrootArg) == 2 { userSysroot = sysrootArg[1] } - break + return arg.value + + default: + return arg.value } + }) + + if hadSkipSysrootMagicFlag { + return } + sysroot, syrootPresent := builder.env.getenv("SYSROOT") if syrootPresent { builder.updateEnv("SYSROOT=") diff --git a/compiler_wrapper/sysroot_flag_test.go b/compiler_wrapper/sysroot_flag_test.go index 9fea6848..bca8d1b0 100644 --- a/compiler_wrapper/sysroot_flag_test.go +++ b/compiler_wrapper/sysroot_flag_test.go @@ -6,9 +6,35 @@ package main import ( "path" + "regexp" "testing" ) +func TestOmitSysrootGivenSysrootSuppressionFlag(t *testing.T) { + escapedAutodetectionFlag := regexp.QuoteMeta(skipSysrootAutodetectionFlag) + withTestContext(t, func(ctx *testContext) { + runWithCompiler := func(compiler string) { + cmd := ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand(compiler, skipSysrootAutodetectionFlag, mainCc))) + if err := verifyArgOrder(cmd, mainCc); err != nil { + t.Error(err) + } + if err := verifyArgCount(cmd, 0, "--sysroot.*"); err != nil { + t.Error(err) + } + if err := verifyArgCount(cmd, 0, "-L.*"); err != nil { + t.Error(err) + } + if err := verifyArgCount(cmd, 0, escapedAutodetectionFlag); err != nil { + t.Error(err) + } + } + + runWithCompiler(gccX86_64) + runWithCompiler(clangX86_64) + }) +} + func TestOmitSysrootGivenUserDefinedSysroot(t *testing.T) { withTestContext(t, func(ctx *testContext) { runWithCompiler := func(compiler string) { diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json index 6993499c..9c9dac03 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json @@ -27,26 +27,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ], "env_updates": [ "PYTHONPATH=/somepath/test_binary" @@ -84,26 +91,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ], "env_updates": [ "PYTHONPATH=/somepath/test_binary" @@ -144,26 +158,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ], "env_updates": [ "PYTHONPATH=/somepath/test_binary" diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json index 26654033..281239a5 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json @@ -18,26 +18,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -62,26 +69,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -106,26 +120,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -150,26 +171,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -194,26 +222,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -238,26 +273,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -282,26 +324,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -326,26 +375,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -370,26 +426,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json index b151051c..045b6438 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json @@ -17,26 +17,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json index 396bb95c..7f4b1c9e 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json @@ -17,26 +17,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -60,26 +67,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -103,26 +117,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -146,26 +167,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -189,26 +217,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -232,26 +267,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -275,26 +317,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -318,26 +367,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -361,26 +417,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json index f32a7045..a965ca7f 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json @@ -17,26 +17,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -63,26 +70,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] }, "stdout": "somemessage", @@ -109,26 +123,34 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", + "-std=gnu++14", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -155,26 +177,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -208,15 +237,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -225,11 +259,13 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -266,15 +302,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -283,11 +324,13 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -324,15 +367,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -341,11 +389,13 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] }, "stdout": "somemessage", @@ -372,26 +422,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -415,26 +472,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -458,26 +522,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -501,26 +572,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -547,26 +625,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json index 8df51096..76f3f14b 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json @@ -19,15 +19,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -35,11 +40,13 @@ "-fcommon", "-fsanitize=kernel-address", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -65,15 +72,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -81,11 +93,13 @@ "-fcommon", "-fsanitize=kernel-address", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -111,15 +125,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -127,11 +146,13 @@ "-fcommon", "-fsanitize=kernel-address", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -157,15 +178,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -173,11 +199,13 @@ "-fcommon", "-fsanitize=kernel-address", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -202,15 +230,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -218,11 +251,13 @@ "-fcommon", "-fsanitize=fuzzer", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -248,15 +283,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -265,11 +305,13 @@ "-fsanitize=address", "-fprofile-instr-generate", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -294,15 +336,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -310,11 +357,13 @@ "-fcommon", "-fsanitize=address", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -339,15 +388,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -355,11 +409,13 @@ "-fcommon", "-fprofile-instr-generate", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json index 7c296af9..493a3dc2 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json @@ -27,15 +27,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -50,11 +55,13 @@ "-Woverride-init", "-Wunsafe-loop-optimizations", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -79,15 +86,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -95,11 +107,13 @@ "-fcommon", "-Wno-#warnings", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -124,15 +138,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -140,11 +159,13 @@ "-fcommon", "-Wno-error=uninitialized", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -169,15 +190,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", @@ -185,11 +211,13 @@ "-fcommon", "-someflag", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json b/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json index c1cf0507..e425e074 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json @@ -33,26 +33,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } }, @@ -63,26 +70,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -123,26 +137,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } }, @@ -153,26 +174,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -215,26 +243,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] }, "stdout": "somemessage", @@ -248,26 +283,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -311,26 +353,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } }, @@ -341,26 +390,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json index ad290b45..0fb8e1f2 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json @@ -20,26 +20,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] } } @@ -56,7 +63,8 @@ "args": [ "main.cc" ] - } + }, + "stdout": "$CROS_ARTIFACTS_TMP_DIR is not set, artifacts will be written to /tmp" }, "cmds": [ { @@ -66,26 +74,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] }, "stderr": "-Werror originalerror", @@ -98,26 +113,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-Wno-error", "-Wno-error=poison-system-directories" ] @@ -148,26 +170,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", - "-Wno-string-concatenation" + "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions" ] }, "stderr": "-Werror originalerror", @@ -180,26 +209,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-unused-local-typedefs", "-fno-addrsig", "-fuse-ld=lld", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "main.cc", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-Wno-error", "-Wno-error=poison-system-directories" ] diff --git a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json index 8bd823ed..b4cdd8d2 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json @@ -26,36 +26,44 @@ "/usr/bin/ccache", "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -99,36 +107,44 @@ "/usr/bin/ccache", "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -175,36 +191,44 @@ "/usr/bin/ccache", "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json index 56f78efa..83d7a8df 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json @@ -17,37 +17,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-ftrapv", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -80,37 +88,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-eabi", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-eabi-", "-ftrapv", "main.cc", "-L/usr/x86_64-cros-eabi/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -143,37 +159,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-win-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-win-gnu-", "-ftrapv", "main.cc", "-L/usr/x86_64-cros-win-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -206,37 +230,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/armv7m-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "--prefix=../../bin/armv7m-cros-linux-gnu-", "-ftrapv", "main.cc", "-L/usr/armv7m-cros-linux-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7m-cros-linux-gnu" @@ -268,37 +300,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/armv7m-cros-eabi", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/armv7m-cros-eabi-", "-ftrapv", "main.cc", "-L/usr/armv7m-cros-eabi/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7m-cros-eabi" @@ -330,37 +370,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/armv7m-cros-win-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "--prefix=../../bin/armv7m-cros-win-gnu-", "-ftrapv", "main.cc", "-L/usr/armv7m-cros-win-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7m-cros-win-gnu" @@ -392,37 +440,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/armv8m-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "--prefix=../../bin/armv8m-cros-linux-gnu-", "-ftrapv", "main.cc", "-L/usr/armv8m-cros-linux-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv8m-cros-linux-gnu" @@ -454,37 +510,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/armv8m-cros-eabi", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/armv8m-cros-eabi-", "-ftrapv", "main.cc", "-L/usr/armv8m-cros-eabi/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv8m-cros-eabi" @@ -516,37 +580,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/armv8m-cros-win-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "--prefix=../../bin/armv8m-cros-win-gnu-", "-ftrapv", "main.cc", "-L/usr/armv8m-cros-win-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv8m-cros-win-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json index db2f59aa..d072960a 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json @@ -16,36 +16,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -77,36 +85,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-eabi", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-eabi-", "main.cc", "-L/usr/x86_64-cros-eabi/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -138,36 +154,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-win-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-win-gnu-", "main.cc", "-L/usr/x86_64-cros-win-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -199,36 +223,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/armv7m-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "--prefix=../../bin/armv7m-cros-linux-gnu-", "main.cc", "-L/usr/armv7m-cros-linux-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7m-cros-linux-gnu" @@ -259,36 +291,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/armv7m-cros-eabi", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/armv7m-cros-eabi-", "main.cc", "-L/usr/armv7m-cros-eabi/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7m-cros-eabi" @@ -319,36 +359,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/armv7m-cros-win-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "--prefix=../../bin/armv7m-cros-win-gnu-", "main.cc", "-L/usr/armv7m-cros-win-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7m-cros-win-gnu" @@ -379,36 +427,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/armv8m-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "--prefix=../../bin/armv8m-cros-linux-gnu-", "main.cc", "-L/usr/armv8m-cros-linux-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv8m-cros-linux-gnu" @@ -439,36 +495,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/armv8m-cros-eabi", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/armv8m-cros-eabi-", "main.cc", "-L/usr/armv8m-cros-eabi/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv8m-cros-eabi" @@ -499,36 +563,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/armv8m-cros-win-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "--prefix=../../bin/armv8m-cros-win-gnu-", "main.cc", "-L/usr/armv8m-cros-win-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv8m-cros-win-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json index 186a16ed..fc120977 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json @@ -16,36 +16,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -80,36 +88,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -144,36 +160,45 @@ "args": [ "../../usr/bin/clang++", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", + "-std=gnu++14", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -208,36 +233,44 @@ "args": [ "somepath/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -279,26 +312,32 @@ "args": [ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", @@ -306,11 +345,13 @@ "--gcc-toolchain=/usr", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -355,26 +396,32 @@ "args": [ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", @@ -382,11 +429,13 @@ "--gcc-toolchain=/usr", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -426,26 +475,32 @@ "args": [ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", @@ -453,11 +508,13 @@ "--gcc-toolchain=/usr", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -492,36 +549,44 @@ "args": [ "/usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -553,36 +618,44 @@ "args": [ "a/b/c/d/e/usr/bin/clang", "--sysroot=/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -614,36 +687,44 @@ "args": [ "a/b/c/d/e/usr/bin/clang", "--sysroot=/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -675,36 +756,44 @@ "args": [ "../usr/bin/clang", "--sysroot=/tmp/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../bin", "-target", @@ -739,36 +828,44 @@ "args": [ "/tmp/usr/bin/clang", "--sysroot=/tmp/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json index 6a38e845..0c9376cd 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json @@ -21,19 +21,25 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -43,11 +49,13 @@ "-fsanitize=kernel-address", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -84,19 +92,25 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -106,11 +120,13 @@ "-fsanitize=kernel-address", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -147,19 +163,25 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -169,11 +191,13 @@ "-fsanitize=kernel-address", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -210,19 +234,25 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -232,11 +262,13 @@ "-fsanitize=kernel-address", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -272,19 +304,25 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -294,11 +332,13 @@ "-fsanitize=fuzzer", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -335,19 +375,25 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -358,11 +404,13 @@ "-fprofile-instr-generate", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -398,19 +446,25 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -420,11 +474,13 @@ "-fsanitize=address", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -457,37 +513,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fprofile-instr-generate", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json index ec91216d..d7f73998 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json @@ -26,26 +26,32 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", @@ -59,11 +65,13 @@ "-Wunsafe-loop-optimizations", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -96,37 +104,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-Wno-#warnings", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -159,37 +175,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-Wno-error=uninitialized", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -222,37 +246,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-someflag", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json index 0ad97b7a..537fc89c 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json @@ -16,12 +16,12 @@ "path": "./x86_64-cros-linux-gnu-gcc.real", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -52,36 +52,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -116,36 +124,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -173,37 +189,45 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-nopie", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -236,25 +260,31 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-fno-stack-protector", @@ -262,11 +292,13 @@ "-D__KERNEL__", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -299,25 +331,31 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/armv7a-cros-linux-gnueabihf", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "-fno-stack-protector", @@ -325,11 +363,13 @@ "-D__KERNEL__", "main.cc", "-L/usr/armv7a-cros-linux-gnueabihf/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7a-cros-linux-gnueabihf" @@ -360,37 +400,45 @@ "path": "/usr/bin/ccache", "args": [ "../../usr/bin/clang", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "--sysroot=xyz", "main.cc", "-Lxyz/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json index f7438940..5d88fec8 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json @@ -31,36 +31,44 @@ "--", "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -73,36 +81,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -145,36 +161,44 @@ "--", "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -188,36 +212,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -262,36 +294,44 @@ "--", "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -308,36 +348,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -383,36 +431,44 @@ "--", "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -426,36 +482,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json index 9cd72611..3d1e305e 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json @@ -19,36 +19,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -74,7 +82,8 @@ "args": [ "main.cc" ] - } + }, + "stdout": "$CROS_ARTIFACTS_TMP_DIR is not set, artifacts will be written to /tmp" }, "cmds": [ { @@ -83,36 +92,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -133,36 +150,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -201,36 +226,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -251,36 +284,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json index d9b532f0..9d17797a 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json @@ -16,36 +16,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -61,12 +69,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -102,36 +110,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -147,12 +163,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -183,36 +199,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -247,36 +271,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -292,12 +324,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_maincc_target_specific.json index 63b7da18..86745dc6 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_maincc_target_specific.json @@ -16,12 +16,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -53,12 +53,12 @@ "args": [ "./x86_64-cros-eabi-gcc.real", "--sysroot=/usr/x86_64-cros-eabi", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -90,12 +90,12 @@ "args": [ "./x86_64-cros-win-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-win-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -127,12 +127,12 @@ "args": [ "./armv7m-cros-linux-gnu-gcc.real", "--sysroot=/usr/armv7m-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "main.cc", @@ -163,12 +163,12 @@ "args": [ "./armv7m-cros-eabi-gcc.real", "--sysroot=/usr/armv7m-cros-eabi", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -199,12 +199,12 @@ "args": [ "./armv7m-cros-win-gnu-gcc.real", "--sysroot=/usr/armv7m-cros-win-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "main.cc", @@ -235,12 +235,12 @@ "args": [ "./armv8m-cros-linux-gnu-gcc.real", "--sysroot=/usr/armv8m-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "main.cc", @@ -271,12 +271,12 @@ "args": [ "./armv8m-cros-eabi-gcc.real", "--sysroot=/usr/armv8m-cros-eabi", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -307,12 +307,12 @@ "args": [ "./armv8m-cros-win-gnu-gcc.real", "--sysroot=/usr/armv8m-cros-win-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "main.cc", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_path.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_path.json index 6e06ce18..b6f3d1d9 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_path.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_path.json @@ -16,12 +16,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -56,12 +56,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -96,12 +96,12 @@ "args": [ "/tmp/stable/x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -133,12 +133,12 @@ "args": [ "./a/b/c/d/e/f/g/x86_64-cros-linux-gnu-gcc.real", "--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -170,12 +170,12 @@ "args": [ "./symlinked/x86_64-cros-linux-gnu-gcc.real", "--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -210,12 +210,12 @@ "args": [ "/tmp/stable/pathenv/x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_sanitizer_args.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_sanitizer_args.json index 9aaf9c25..0552bf7c 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_sanitizer_args.json @@ -18,12 +18,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-fsanitize=kernel-address", @@ -58,12 +58,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-fsanitize=kernel-address", @@ -98,12 +98,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-fsanitize=kernel-address", @@ -138,12 +138,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-fsanitize=kernel-address", @@ -177,12 +177,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-fsanitize=fuzzer", @@ -217,12 +217,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-fsanitize=address", @@ -257,12 +257,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-fsanitize=address", @@ -296,12 +296,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-fprofile-instr-generate", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_specific_args.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_specific_args.json index 4ee90cbc..52c9f2cc 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_specific_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_specific_args.json @@ -17,12 +17,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-march=goldmont", @@ -56,12 +56,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-march=goldmont-plus", @@ -95,12 +95,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-march=skylake", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_sysroot_wrapper_common.json index 9397333d..8b28e6e2 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_sysroot_wrapper_common.json @@ -16,12 +16,12 @@ "path": "./x86_64-cros-linux-gnu-gcc.real", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -52,12 +52,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -92,12 +92,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -126,12 +126,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-nopie", @@ -165,11 +165,11 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "-fno-stack-protector", @@ -204,11 +204,11 @@ "args": [ "./armv7a-cros-linux-gnueabihf-gcc.real", "--sysroot=/usr/armv7a-cros-linux-gnueabihf", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", - "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", "-fno-stack-protector", @@ -241,12 +241,12 @@ "path": "/usr/bin/ccache", "args": [ "./x86_64-cros-linux-gnu-gcc.real", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--sysroot=xyz", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json index 8bd823ed..b4cdd8d2 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json @@ -26,36 +26,44 @@ "/usr/bin/ccache", "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -99,36 +107,44 @@ "/usr/bin/ccache", "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -175,36 +191,44 @@ "/usr/bin/ccache", "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json index 186a16ed..fc120977 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json @@ -16,36 +16,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -80,36 +88,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -144,36 +160,45 @@ "args": [ "../../usr/bin/clang++", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", + "-std=gnu++14", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -208,36 +233,44 @@ "args": [ "somepath/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -279,26 +312,32 @@ "args": [ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", @@ -306,11 +345,13 @@ "--gcc-toolchain=/usr", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -355,26 +396,32 @@ "args": [ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", @@ -382,11 +429,13 @@ "--gcc-toolchain=/usr", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -426,26 +475,32 @@ "args": [ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", @@ -453,11 +508,13 @@ "--gcc-toolchain=/usr", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -492,36 +549,44 @@ "args": [ "/usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -553,36 +618,44 @@ "args": [ "a/b/c/d/e/usr/bin/clang", "--sysroot=/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -614,36 +687,44 @@ "args": [ "a/b/c/d/e/usr/bin/clang", "--sysroot=/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -675,36 +756,44 @@ "args": [ "../usr/bin/clang", "--sysroot=/tmp/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../bin", "-target", @@ -739,36 +828,44 @@ "args": [ "/tmp/usr/bin/clang", "--sysroot=/tmp/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json index f7438940..5d88fec8 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json @@ -31,36 +31,44 @@ "--", "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -73,36 +81,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -145,36 +161,44 @@ "--", "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -188,36 +212,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -262,36 +294,44 @@ "--", "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -308,36 +348,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -383,36 +431,44 @@ "--", "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -426,36 +482,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json index 9cd72611..3d1e305e 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json @@ -19,36 +19,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -74,7 +82,8 @@ "args": [ "main.cc" ] - } + }, + "stdout": "$CROS_ARTIFACTS_TMP_DIR is not set, artifacts will be written to /tmp" }, "cmds": [ { @@ -83,36 +92,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -133,36 +150,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -201,36 +226,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -251,36 +284,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json index d9b532f0..9d17797a 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json @@ -16,36 +16,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -61,12 +69,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -102,36 +110,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -147,12 +163,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -183,36 +199,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -247,36 +271,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -292,12 +324,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_path.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_path.json index 6e06ce18..b6f3d1d9 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_path.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_path.json @@ -16,12 +16,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -56,12 +56,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -96,12 +96,12 @@ "args": [ "/tmp/stable/x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -133,12 +133,12 @@ "args": [ "./a/b/c/d/e/f/g/x86_64-cros-linux-gnu-gcc.real", "--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -170,12 +170,12 @@ "args": [ "./symlinked/x86_64-cros-linux-gnu-gcc.real", "--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -210,12 +210,12 @@ "args": [ "/tmp/stable/pathenv/x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json index da522b99..b476031b 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json @@ -25,36 +25,44 @@ "/tmp/sysroot_bisect", "/usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -94,36 +102,44 @@ "someBisectDir", "/usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -166,36 +182,44 @@ "someBisectDir", "/usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json index 287833cf..3849ecf8 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json @@ -15,36 +15,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -73,36 +81,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -131,36 +147,45 @@ "path": "../../usr/bin/clang++", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", + "-std=gnu++14", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -189,36 +214,44 @@ "path": "somepath/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -254,26 +287,32 @@ "path": "/somedir/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", @@ -281,11 +320,13 @@ "--gcc-toolchain=/usr", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -325,26 +366,32 @@ "args": [ "/somedir/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", @@ -352,11 +399,13 @@ "--gcc-toolchain=/usr", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -395,26 +444,32 @@ "path": "/somedir/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", @@ -422,11 +477,13 @@ "--gcc-toolchain=/usr", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -455,36 +512,44 @@ "path": "/usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -510,36 +575,44 @@ "path": "a/b/c/d/e/usr/bin/clang", "args": [ "--sysroot=/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -565,36 +638,44 @@ "path": "a/b/c/d/e/usr/bin/clang", "args": [ "--sysroot=/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -620,36 +701,44 @@ "path": "../usr/bin/clang", "args": [ "--sysroot=/tmp/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../bin", "-target", @@ -678,36 +767,44 @@ "path": "/tmp/usr/bin/clang", "args": [ "--sysroot=/tmp/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json index f7438940..5d88fec8 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json @@ -31,36 +31,44 @@ "--", "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -73,36 +81,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -145,36 +161,44 @@ "--", "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -188,36 +212,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -262,36 +294,44 @@ "--", "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -308,36 +348,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -383,36 +431,44 @@ "--", "-resource-dir=someResourceDir", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -426,36 +482,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json index 54943fb3..a4b2d2b4 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json @@ -18,36 +18,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -68,7 +76,8 @@ "args": [ "main.cc" ] - } + }, + "stdout": "$CROS_ARTIFACTS_TMP_DIR is not set, artifacts will be written to /tmp" }, "cmds": [ { @@ -76,36 +85,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -120,36 +137,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -182,36 +207,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -226,36 +259,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json index 1704cd17..091e00f5 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json @@ -16,36 +16,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -60,12 +68,12 @@ "path": "./x86_64-cros-linux-gnu-gcc.real", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -97,36 +105,44 @@ "args": [ "../../usr/bin/clang", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -142,12 +158,12 @@ "args": [ "./x86_64-cros-linux-gnu-gcc.real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -178,36 +194,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -242,36 +266,44 @@ "path": "../../usr/bin/clang", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "--unwindlib=libunwind", "-Wno-section", "-fno-addrsig", "-fuse-ld=lld", + "-ftrivial-auto-var-init=zero", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -286,12 +318,12 @@ "path": "./x86_64-cros-linux-gnu-gcc.real", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_path.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_path.json index fb877370..ed60e6d6 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_path.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_path.json @@ -15,12 +15,12 @@ "path": "./x86_64-cros-linux-gnu-gcc.real", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -50,12 +50,12 @@ "path": "./x86_64-cros-linux-gnu-gcc.real", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -85,12 +85,12 @@ "path": "/tmp/stable/x86_64-cros-linux-gnu-gcc.real", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -117,12 +117,12 @@ "path": "./a/b/c/d/e/f/g/x86_64-cros-linux-gnu-gcc.real", "args": [ "--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -149,12 +149,12 @@ "path": "./symlinked/x86_64-cros-linux-gnu-gcc.real", "args": [ "--sysroot=/tmp/stable/a/b/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", @@ -184,12 +184,12 @@ "path": "/tmp/stable/pathenv/x86_64-cros-linux-gnu-gcc.real", "args": [ "--sysroot=/usr/x86_64-cros-linux-gnu", + "-D_FORTIFY_SOURCE=2", "-fno-reorder-blocks-and-partition", "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-fcommon", "-fstack-protector-strong", - "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json index 96fd88cf..9a71937a 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json @@ -29,26 +29,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -95,26 +102,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -164,26 +178,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json index 35f90b1e..aa9dfa80 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json @@ -20,15 +20,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -36,11 +41,13 @@ "-ftrapv", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -76,15 +83,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -92,11 +104,13 @@ "-ftrapv", "main.cc", "-L/usr/x86_64-cros-eabi/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -132,15 +146,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -148,11 +167,13 @@ "-ftrapv", "main.cc", "-L/usr/x86_64-cros-win-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -188,15 +209,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -205,11 +231,13 @@ "-ftrapv", "main.cc", "-L/usr/armv7m-cros-linux-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7m-cros-linux-gnu" @@ -244,15 +272,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -260,11 +293,13 @@ "-ftrapv", "main.cc", "-L/usr/armv7m-cros-eabi/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7m-cros-eabi" @@ -299,15 +334,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -316,11 +356,13 @@ "-ftrapv", "main.cc", "-L/usr/armv7m-cros-win-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7m-cros-win-gnu" @@ -355,15 +397,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -372,11 +419,13 @@ "-ftrapv", "main.cc", "-L/usr/armv8m-cros-linux-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv8m-cros-linux-gnu" @@ -411,15 +460,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -427,11 +481,13 @@ "-ftrapv", "main.cc", "-L/usr/armv8m-cros-eabi/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv8m-cros-eabi" @@ -466,15 +522,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -483,11 +544,13 @@ "-ftrapv", "main.cc", "-L/usr/armv8m-cros-win-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv8m-cros-win-gnu" diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json index e2479e9b..778e7a0b 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json @@ -19,26 +19,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -73,26 +80,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-eabi-", "main.cc", "-L/usr/x86_64-cros-eabi/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -127,26 +141,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-win-gnu-", "main.cc", "-L/usr/x86_64-cros-win-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -181,15 +202,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -197,11 +223,13 @@ "--prefix=../../bin/armv7m-cros-linux-gnu-", "main.cc", "-L/usr/armv7m-cros-linux-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7m-cros-linux-gnu" @@ -235,26 +263,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/armv7m-cros-eabi-", "main.cc", "-L/usr/armv7m-cros-eabi/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7m-cros-eabi" @@ -288,15 +323,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -304,11 +344,13 @@ "--prefix=../../bin/armv7m-cros-win-gnu-", "main.cc", "-L/usr/armv7m-cros-win-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7m-cros-win-gnu" @@ -342,15 +384,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -358,11 +405,13 @@ "--prefix=../../bin/armv8m-cros-linux-gnu-", "main.cc", "-L/usr/armv8m-cros-linux-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv8m-cros-linux-gnu" @@ -396,26 +445,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/armv8m-cros-eabi-", "main.cc", "-L/usr/armv8m-cros-eabi/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv8m-cros-eabi" @@ -449,15 +505,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -465,11 +526,13 @@ "--prefix=../../bin/armv8m-cros-win-gnu-", "main.cc", "-L/usr/armv8m-cros-win-gnu/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv8m-cros-win-gnu" diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json index 11c566b6..b9ff8405 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json @@ -19,26 +19,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -76,26 +83,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -133,26 +147,34 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", + "-std=gnu++14", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -190,26 +212,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -254,15 +283,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -271,11 +305,13 @@ "--gcc-toolchain=/usr", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -323,15 +359,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -340,11 +381,13 @@ "--gcc-toolchain=/usr", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -387,15 +430,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -404,11 +452,13 @@ "--gcc-toolchain=/usr", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -446,26 +496,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -500,26 +557,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -554,26 +618,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -608,26 +679,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../bin", "-target", @@ -665,26 +743,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json index fe0df74d..27404a12 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json @@ -21,15 +21,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -37,11 +42,13 @@ "-fsanitize=kernel-address", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -78,15 +85,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -94,11 +106,13 @@ "-fsanitize=kernel-address", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -135,15 +149,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -151,11 +170,13 @@ "-fsanitize=kernel-address", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -192,15 +213,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -208,11 +234,13 @@ "-fsanitize=kernel-address", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -248,15 +276,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -264,11 +297,13 @@ "-fsanitize=fuzzer", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -305,15 +340,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -322,11 +362,13 @@ "-fprofile-instr-generate", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -362,15 +404,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -378,11 +425,13 @@ "-fsanitize=address", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -418,15 +467,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -434,11 +488,13 @@ "-fprofile-instr-generate", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json index b744e8c6..03d90589 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json @@ -29,15 +29,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -52,11 +57,13 @@ "-Wunsafe-loop-optimizations", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -92,15 +99,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -108,11 +120,13 @@ "-Wno-#warnings", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -148,15 +162,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -164,11 +183,13 @@ "-Wno-error=uninitialized", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -204,15 +225,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -220,11 +246,13 @@ "-someflag", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json index ed3a6ef1..3ceaa3f7 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json @@ -52,26 +52,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -109,26 +116,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -159,15 +173,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -175,11 +194,13 @@ "-nopie", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -215,15 +236,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -232,11 +258,13 @@ "-D__KERNEL__", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -272,15 +300,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -290,11 +323,13 @@ "-D__KERNEL__", "main.cc", "-L/usr/armv7a-cros-linux-gnueabihf/usr/lib", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-B../../bin", "-target", "armv7a-cros-linux-gnueabihf" @@ -328,15 +363,20 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", @@ -344,11 +384,13 @@ "--sysroot=xyz", "main.cc", "-Lxyz/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json index 830abee6..b8575a09 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json @@ -34,26 +34,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -69,26 +76,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -134,26 +148,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -170,26 +191,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -237,26 +265,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -276,26 +311,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -344,26 +386,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -380,26 +429,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json index 07c20900..ff37898f 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json @@ -22,26 +22,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -67,7 +74,8 @@ "args": [ "main.cc" ] - } + }, + "stdout": "$CROS_ARTIFACTS_TMP_DIR is not set, artifacts will be written to /tmp" }, "cmds": [ { @@ -79,26 +87,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -122,26 +137,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -183,26 +205,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -226,26 +255,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json index 9dd5687b..771b224a 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json @@ -19,26 +19,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -95,26 +102,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -166,26 +180,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", @@ -223,26 +244,33 @@ "-Qunused-arguments", "-Werror=poison-system-directories", "-Wno-compound-token-split-by-macro", + "-Wno-deprecated-builtins", "-Wno-deprecated-declarations", + "-Wno-enum-constexpr-conversion", "-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int", "-Wno-final-dtor-non-final-class", + "-Wno-single-bit-bitfield-constant-conversion", "-Wno-tautological-constant-compare", "-Wno-tautological-unsigned-enum-zero-compare", "-Wno-unknown-warning-option", "-fdebug-default-version=5", - "-fexperimental-new-pass-manager", + "-Wno-int-conversion", + "-Wno-incompatible-function-pointer-types", + "-D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES", "-Wno-section", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", "-L/usr/x86_64-cros-linux-gnu/usr/lib64", + "-Wno-array-parameter", "-Wno-compound-token-split-by-space", "-Wno-deprecated-copy", "-Wno-unused-but-set-variable", "-Wno-implicit-int-float-conversion", "-Wno-string-concatenation", + "-Wno-gnu-offsetof-extensions", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testutil_test.go b/compiler_wrapper/testutil_test.go index 463e34ac..8bac479b 100644 --- a/compiler_wrapper/testutil_test.go +++ b/compiler_wrapper/testutil_test.go @@ -39,7 +39,6 @@ type testContext struct { tempDir string env []string cfg *config - inputCmd *command lastCmd *command cmdCount int cmdMock func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error @@ -192,8 +191,11 @@ func (ctx *testContext) mustFail(exitCode int) string { func (ctx *testContext) updateConfig(cfg *config) { *ctx.cfg = *cfg ctx.cfg.newWarningsDir = filepath.Join(ctx.tempDir, "fatal_clang_warnings") - ctx.cfg.triciumNitsDir = filepath.Join(ctx.tempDir, "tricium_nits") ctx.cfg.crashArtifactsDir = filepath.Join(ctx.tempDir, "clang_crash_diagnostics") + + // Ensure this is always empty, so any test that depends on it will see no output unless + // it's properly set up. + ctx.cfg.newWarningsDir = "" } func (ctx *testContext) newCommand(path string, args ...string) *command { diff --git a/cros_utils/bugs.py b/cros_utils/bugs.py index 43e0e553..ac1202ae 100755 --- a/cros_utils/bugs.py +++ b/cros_utils/bugs.py @@ -2,46 +2,97 @@ # Copyright 2021 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Utilities to file bugs.""" -import base64 import datetime import enum import json import os +import threading from typing import Any, Dict, List, Optional X20_PATH = "/google/data/rw/teams/c-compiler-chrome/prod_bugs" +# These constants are sourced from +# //google3/googleclient/chrome/chromeos_toolchain/bug_manager/bugs.go class WellKnownComponents(enum.IntEnum): """A listing of "well-known" components recognized by our infra.""" CrOSToolchainPublic = -1 CrOSToolchainPrivate = -2 + AndroidRustToolchain = -3 + + +class _FileNameGenerator: + """Generates unique file names. This container is thread-safe. + + The names generated have the following properties: + - successive, sequenced calls to `get_json_file_name()` will produce + names that sort later in lists over time (e.g., + [generator.generate_json_file_name() for _ in range(10)] will be in + sorted order). + - file names cannot collide with file names generated on the same + machine (ignoring machines with unreasonable PID reuse). + - file names are incredibly unlikely to collide when generated on + multiple machines, as they have 8 bytes of entropy in them. + """ + + _RANDOM_BYTES = 8 + _MAX_OS_ENTROPY_VALUE = 1 << _RANDOM_BYTES * 8 + # The intent of this is "the maximum possible size of our entropy string, + # so we can zfill properly below." Double the value the OS hands us, since + # we add to it in `generate_json_file_name`. + _ENTROPY_STR_SIZE = len(str(2 * _MAX_OS_ENTROPY_VALUE)) + def __init__(self): + self._lock = threading.Lock() + self._entropy = int.from_bytes( + os.getrandom(self._RANDOM_BYTES), byteorder="little", signed=False + ) -def _WriteBugJSONFile(object_type: str, json_object: Dict[str, Any]): - """Writes a JSON file to X20_PATH with the given bug-ish object.""" + def generate_json_file_name(self, now: datetime.datetime): + with self._lock: + my_entropy = self._entropy + self._entropy += 1 + + now = now.isoformat("T", "seconds") + "Z" + entropy_str = str(my_entropy).zfill(self._ENTROPY_STR_SIZE) + pid = os.getpid() + return f"{now}_{entropy_str}_{pid}.json" + + +_GLOBAL_NAME_GENERATOR = _FileNameGenerator() + + +def _WriteBugJSONFile( + object_type: str, + json_object: Dict[str, Any], + directory: Optional[os.PathLike], +): + """Writes a JSON file to `directory` with the given bug-ish object. + + Args: + object_type: name of the object we're writing. + json_object: object to write. + directory: the directory to write to. Uses X20_PATH if None. + """ final_object = { "type": object_type, "value": json_object, } - # The name of this has two parts: - # - An easily sortable time, to provide uniqueness and let our service send - # things in the order they were put into the outbox. - # - 64 bits of entropy, so two racing bug writes don't clobber the same file. - now = datetime.datetime.utcnow().isoformat("T", "seconds") + "Z" - entropy = base64.urlsafe_b64encode(os.getrandom(8)) - entropy_str = entropy.rstrip(b"=").decode("utf-8") - file_path = os.path.join(X20_PATH, f"{now}_{entropy_str}.json") + if directory is None: + directory = X20_PATH + now = datetime.datetime.now(tz=datetime.timezone.utc) + file_path = os.path.join( + directory, _GLOBAL_NAME_GENERATOR.generate_json_file_name(now) + ) temp_path = file_path + ".in_progress" try: - with open(temp_path, "w") as f: + with open(temp_path, "w", encoding="utf-8") as f: json.dump(final_object, f) os.rename(temp_path, file_path) except: @@ -50,7 +101,9 @@ def _WriteBugJSONFile(object_type: str, json_object: Dict[str, Any]): return file_path -def AppendToExistingBug(bug_id: int, body: str): +def AppendToExistingBug( + bug_id: int, body: str, directory: Optional[os.PathLike] = None +): """Sends a reply to an existing bug.""" _WriteBugJSONFile( "AppendToExistingBugRequest", @@ -58,6 +111,7 @@ def AppendToExistingBug(bug_id: int, body: str): "body": body, "bug_id": bug_id, }, + directory, ) @@ -67,6 +121,7 @@ def CreateNewBug( body: str, assignee: Optional[str] = None, cc: Optional[List[str]] = None, + directory: Optional[os.PathLike] = None, ): """Sends a request to create a new bug. @@ -79,6 +134,8 @@ def CreateNewBug( "well-known" assignee (detective, mage). cc: A list of emails to add to the CC list. Must either be an email address, or a "well-known" individual (detective, mage). + directory: The directory to write the report to. Defaults to our x20 bugs + directory. """ obj = { "component_id": component_id, @@ -92,10 +149,16 @@ def CreateNewBug( if cc: obj["cc"] = cc - _WriteBugJSONFile("FileNewBugRequest", obj) + _WriteBugJSONFile("FileNewBugRequest", obj, directory) -def SendCronjobLog(cronjob_name: str, failed: bool, message: str): +def SendCronjobLog( + cronjob_name: str, + failed: bool, + message: str, + turndown_time_hours: int = 0, + directory: Optional[os.PathLike] = None, +): """Sends the record of a cronjob to our bug infra. cronjob_name: The name of the cronjob. Expected to remain consistent over @@ -103,12 +166,18 @@ def SendCronjobLog(cronjob_name: str, failed: bool, message: str): failed: Whether the job failed or not. message: Any seemingly relevant context. This is pasted verbatim in a bug, if the cronjob infra deems it worthy. + turndown_time_hours: If nonzero, this cronjob will be considered + turned down if more than `turndown_time_hours` pass without a report of + success or failure. If zero, this job will not automatically be turned + down. + directory: The directory to write the report to. Defaults to our x20 bugs + directory. """ - _WriteBugJSONFile( - "ChrotomationCronjobUpdate", - { - "name": cronjob_name, - "message": message, - "failed": failed, - }, - ) + json_object = { + "name": cronjob_name, + "message": message, + "failed": failed, + } + if turndown_time_hours: + json_object["cronjob_turndown_time_hours"] = turndown_time_hours + _WriteBugJSONFile("CronjobUpdate", json_object, directory) diff --git a/cros_utils/bugs_test.py b/cros_utils/bugs_test.py index 5a07dbd8..1ee6bfe4 100755 --- a/cros_utils/bugs_test.py +++ b/cros_utils/bugs_test.py @@ -8,12 +8,19 @@ """Tests bug filing bits.""" +import datetime import json +import os +from pathlib import Path import tempfile import unittest +from unittest import mock from unittest.mock import patch -import bugs +from cros_utils import bugs + + +_ARBITRARY_DATETIME = datetime.datetime(2020, 1, 1, 23, 0, 0, 0) class Tests(unittest.TestCase): @@ -36,6 +43,7 @@ class Tests(unittest.TestCase): "foo": "bar", "baz": bugs.WellKnownComponents.CrOSToolchainPublic, }, + None, ) self.assertTrue( @@ -43,7 +51,7 @@ class Tests(unittest.TestCase): f"Expected {file_path} to start with {tempdir}", ) - with open(file_path) as f: + with open(file_path, encoding="utf-8") as f: self.assertEqual( json.load(f), { @@ -57,7 +65,7 @@ class Tests(unittest.TestCase): }, ) - @patch("bugs._WriteBugJSONFile") + @patch.object(bugs, "_WriteBugJSONFile") def testAppendingToBugsSeemsToWork(self, mock_write_json_file): """Tests AppendToExistingBug.""" bugs.AppendToExistingBug(1234, "hello, world!") @@ -67,9 +75,10 @@ class Tests(unittest.TestCase): "body": "hello, world!", "bug_id": 1234, }, + None, ) - @patch("bugs._WriteBugJSONFile") + @patch.object(bugs, "_WriteBugJSONFile") def testBugCreationSeemsToWork(self, mock_write_json_file): """Tests CreateNewBug.""" test_case_additions = ( @@ -110,21 +119,106 @@ class Tests(unittest.TestCase): mock_write_json_file.assert_called_once_with( "FileNewBugRequest", expected_output, + None, ) mock_write_json_file.reset_mock() - @patch("bugs._WriteBugJSONFile") + @patch.object(bugs, "_WriteBugJSONFile") def testCronjobLogSendingSeemsToWork(self, mock_write_json_file): """Tests SendCronjobLog.""" bugs.SendCronjobLog("my_name", False, "hello, world!") mock_write_json_file.assert_called_once_with( - "ChrotomationCronjobUpdate", + "CronjobUpdate", + { + "name": "my_name", + "message": "hello, world!", + "failed": False, + }, + None, + ) + + @patch.object(bugs, "_WriteBugJSONFile") + def testCronjobLogSendingSeemsToWorkWithTurndown( + self, mock_write_json_file + ): + """Tests SendCronjobLog.""" + bugs.SendCronjobLog( + "my_name", False, "hello, world!", turndown_time_hours=42 + ) + mock_write_json_file.assert_called_once_with( + "CronjobUpdate", { "name": "my_name", "message": "hello, world!", "failed": False, + "cronjob_turndown_time_hours": 42, }, + None, + ) + + def testFileNameGenerationProducesFileNamesInSortedOrder(self): + """Tests that _FileNameGenerator gives us sorted file names.""" + gen = bugs._FileNameGenerator() + first = gen.generate_json_file_name(_ARBITRARY_DATETIME) + second = gen.generate_json_file_name(_ARBITRARY_DATETIME) + self.assertLess(first, second) + + def testFileNameGenerationProtectsAgainstRipplingAdds(self): + """Tests that _FileNameGenerator gives us sorted file names.""" + gen = bugs._FileNameGenerator() + gen._entropy = 9 + first = gen.generate_json_file_name(_ARBITRARY_DATETIME) + second = gen.generate_json_file_name(_ARBITRARY_DATETIME) + self.assertLess(first, second) + + gen = bugs._FileNameGenerator() + all_9s = "9" * (gen._ENTROPY_STR_SIZE - 1) + gen._entropy = int(all_9s) + third = gen.generate_json_file_name(_ARBITRARY_DATETIME) + self.assertLess(second, third) + + fourth = gen.generate_json_file_name(_ARBITRARY_DATETIME) + self.assertLess(third, fourth) + + @patch.object(os, "getpid") + def testForkingProducesADifferentReport(self, mock_getpid): + """Tests that _FileNameGenerator gives us sorted file names.""" + gen = bugs._FileNameGenerator() + + mock_getpid.return_value = 1 + gen._entropy = 0 + parent_file = gen.generate_json_file_name(_ARBITRARY_DATETIME) + + mock_getpid.return_value = 2 + gen._entropy = 0 + child_file = gen.generate_json_file_name(_ARBITRARY_DATETIME) + self.assertNotEqual(parent_file, child_file) + + @patch.object(bugs, "_WriteBugJSONFile") + def testCustomDirectoriesArePassedThrough(self, mock_write_json_file): + directory = "/path/to/somewhere/interesting" + bugs.AppendToExistingBug(1, "foo", directory=directory) + mock_write_json_file.assert_called_once_with( + mock.ANY, mock.ANY, directory ) + mock_write_json_file.reset_mock() + + bugs.CreateNewBug(1, "title", "body", directory=directory) + mock_write_json_file.assert_called_once_with( + mock.ANY, mock.ANY, directory + ) + mock_write_json_file.reset_mock() + + bugs.SendCronjobLog("cronjob", False, "message", directory=directory) + mock_write_json_file.assert_called_once_with( + mock.ANY, mock.ANY, directory + ) + + def testWriteBugJSONFileWritesToGivenDirectory(self): + with tempfile.TemporaryDirectory() as tmpdir: + bugs.AppendToExistingBug(1, "body", directory=tmpdir) + json_files = list(Path(tmpdir).glob("*.json")) + self.assertEqual(len(json_files), 1, json_files) if __name__ == "__main__": diff --git a/cros_utils/buildbot_utils.py b/cros_utils/buildbot_utils.py index 8f0ce5e0..36c68c13 100644 --- a/cros_utils/buildbot_utils.py +++ b/cros_utils/buildbot_utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2017 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -41,13 +40,13 @@ def PeekTrybotImage(chromeos_root, buildbucket_id): """Get the artifact URL of a given tryjob. Args: - buildbucket_id: buildbucket-id - chromeos_root: root dir of chrome os checkout + buildbucket_id: buildbucket-id + chromeos_root: root dir of chrome os checkout Returns: - (status, url) where status can be 'pass', 'fail', 'running', + (status, url) where status can be 'pass', 'fail', 'running', and url looks like: - gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789 + gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789 """ command = ( "cros buildresult --report json --buildbucket-id %s" % buildbucket_id @@ -72,10 +71,10 @@ def ParseTryjobBuildbucketId(msg): """Find the buildbucket-id in the messages from `cros tryjob`. Args: - msg: messages from `cros tryjob` + msg: messages from `cros tryjob` Returns: - buildbucket-id, which will be passed to `cros buildresult` + buildbucket-id, which will be passed to `cros buildresult` """ output_list = ast.literal_eval(msg) output_dict = output_list[0] @@ -94,17 +93,17 @@ def SubmitTryjob( """Calls `cros tryjob ...` Args: - chromeos_root: the path to the ChromeOS root, needed for finding chromite - and launching the buildbot. - buildbot_name: the name of the buildbot queue, such as lumpy-release or - daisy-paladin. - patch_list: a python list of the patches, if any, for the buildbot to use. - tryjob_flags: See cros tryjob --help for available options. - build_toolchain: builds and uses the latest toolchain, rather than the - prebuilt one in SDK. + chromeos_root: the path to the ChromeOS root, needed for finding chromite + and launching the buildbot. + buildbot_name: the name of the buildbot queue, such as lumpy-release or + daisy-paladin. + patch_list: a python list of the patches, if any, for the buildbot to use. + tryjob_flags: See cros tryjob --help for available options. + build_toolchain: builds and uses the latest toolchain, rather than the + prebuilt one in SDK. Returns: - buildbucket id + buildbucket id """ patch_arg = "" if patch_list: @@ -151,19 +150,19 @@ def GetTrybotImage( artifact (if one was created), and returns that artifact name. Args: - chromeos_root: the path to the ChromeOS root, needed for finding chromite - and launching the buildbot. - buildbot_name: the name of the buildbot queue, such as lumpy-release or - daisy-paladin. - patch_list: a python list of the patches, if any, for the buildbot to use. - tryjob_flags: See cros tryjob --help for available options. - build_toolchain: builds and uses the latest toolchain, rather than the + chromeos_root: the path to the ChromeOS root, needed for finding chromite + and launching the buildbot. + buildbot_name: the name of the buildbot queue, such as lumpy-release or + daisy-paladin. + patch_list: a python list of the patches, if any, for the buildbot to use. + tryjob_flags: See cros tryjob --help for available options. + build_toolchain: builds and uses the latest toolchain, rather than the prebuilt one in SDK. - asynchronous: don't wait for artifacts; just return the buildbucket id + asynchronous: don't wait for artifacts; just return the buildbucket id Returns: - (buildbucket id, partial image url) e.g. - (8952271933586980528, trybot-elm-release-tryjob/R67-10480.0.0-b2373596) + (buildbucket id, partial image url) e.g. + (8952271933586980528, trybot-elm-release-tryjob/R67-10480.0.0-b2373596) """ buildbucket_id = SubmitTryjob( chromeos_root, buildbot_name, patch_list, tryjob_flags, build_toolchain diff --git a/cros_utils/buildbot_utils_unittest.py b/cros_utils/buildbot_utils_unittest.py index 2c9585b5..9e006f20 100755 --- a/cros_utils/buildbot_utils_unittest.py +++ b/cros_utils/buildbot_utils_unittest.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# # Copyright 2018 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -154,7 +152,7 @@ class TrybotTest(unittest.TestCase): ) mocked_imageexist.return_value = True image = buildbot_utils.GetLatestImage("", IMAGE_DIR) - self.assertEqual(image, "{0}/R78-12423.0.0".format(IMAGE_DIR)) + self.assertEqual(image, f"{IMAGE_DIR}/R78-12423.0.0") def testGetLatestImageInvalid(self): with patch.object( @@ -190,9 +188,7 @@ class TrybotTest(unittest.TestCase): image = buildbot_utils.GetLatestRecipeImage("", IMAGE_DIR) self.assertEqual( image, - "{0}/R83-13003.0.0-30218-8884712858556419".format( - IMAGE_DIR - ), + f"{IMAGE_DIR}/R83-13003.0.0-30218-8884712858556419", ) def testGetLatestRecipeImageInvalid(self): @@ -232,9 +228,7 @@ class TrybotTest(unittest.TestCase): image = buildbot_utils.GetLatestRecipeImage("", IMAGE_DIR) self.assertEqual( image, - "{0}/R83-13003.0.0-30196-8884755532184725".format( - IMAGE_DIR - ), + f"{IMAGE_DIR}/R83-13003.0.0-30196-8884755532184725", ) diff --git a/cros_utils/device_setup_utils.py b/cros_utils/device_setup_utils.py deleted file mode 100644 index 443c6474..00000000 --- a/cros_utils/device_setup_utils.py +++ /dev/null @@ -1,615 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2019 The ChromiumOS Authors -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Utils for setting devices - -This script provides utils to set device specs. -""" - - -__author__ = "zhizhouy@google.com (Zhizhou Yang)" - -from contextlib import contextmanager -import re -import time - -from cros_utils import command_executer - - -class DutWrapper(object): - """Wrap DUT parameters inside.""" - - def __init__( - self, - chromeos_root, - remote, - log_level="verbose", - logger=None, - ce=None, - dut_config=None, - ): - self.chromeos_root = chromeos_root - self.remote = remote - self.log_level = log_level - self.logger = logger - self.ce = ce or command_executer.GetCommandExecuter(log_level=log_level) - self.dut_config = dut_config - - def RunCommandOnDut(self, command, ignore_status=False): - """Helper function to run command on DUT.""" - ret, msg, err_msg = self.ce.CrosRunCommandWOutput( - command, machine=self.remote, chromeos_root=self.chromeos_root - ) - - if ret: - err_msg = ( - "Command execution on DUT %s failed.\n" - "Failing command: %s\n" - "returned %d\n" - "Error message: %s" % (self.remote, command, ret, err_msg) - ) - if ignore_status: - self.logger.LogError( - err_msg + "\n(Failure is considered non-fatal. Continue.)" - ) - else: - self.logger.LogFatal(err_msg) - - return ret, msg, err_msg - - def DisableASLR(self): - """Disable ASLR on DUT.""" - disable_aslr = ( - "set -e; " - "if [[ -e /proc/sys/kernel/randomize_va_space ]]; then " - " echo 0 > /proc/sys/kernel/randomize_va_space; " - "fi" - ) - if self.log_level == "average": - self.logger.LogOutput("Disable ASLR.") - self.RunCommandOnDut(disable_aslr) - - def SetCpuGovernor(self, governor, ignore_status=False): - """Setup CPU Governor on DUT.""" - set_gov_cmd = ( - "for f in `ls -d /sys/devices/system/cpu/cpu*/cpufreq 2>/dev/null`; do " - # Skip writing scaling_governor if cpu is offline. - " [[ -e ${f/cpufreq/online} ]] && grep -q 0 ${f/cpufreq/online} " - " && continue; " - " cd $f; " - " if [[ -e scaling_governor ]]; then " - " echo %s > scaling_governor; fi; " - "done; " - ) - if self.log_level == "average": - self.logger.LogOutput("Setup CPU Governor: %s." % governor) - ret, _, _ = self.RunCommandOnDut( - set_gov_cmd % governor, ignore_status=ignore_status - ) - return ret - - def DisableTurbo(self): - """Disable Turbo on DUT.""" - dis_turbo_cmd = ( - "if [[ -e /sys/devices/system/cpu/intel_pstate/no_turbo ]]; then " - " if grep -q 0 /sys/devices/system/cpu/intel_pstate/no_turbo; then " - " echo -n 1 > /sys/devices/system/cpu/intel_pstate/no_turbo; " - " fi; " - "fi; " - ) - if self.log_level == "average": - self.logger.LogOutput("Disable Turbo.") - self.RunCommandOnDut(dis_turbo_cmd) - - def SetupCpuUsage(self): - """Setup CPU usage. - - Based on self.dut_config['cpu_usage'] configure CPU cores - utilization. - """ - - if ( - self.dut_config["cpu_usage"] == "big_only" - or self.dut_config["cpu_usage"] == "little_only" - ): - _, arch, _ = self.RunCommandOnDut("uname -m") - - if arch.lower().startswith("arm") or arch.lower().startswith( - "aarch64" - ): - self.SetupArmCores() - - def SetupArmCores(self): - """Setup ARM big/little cores.""" - - # CPU implemeters/part numbers of big/LITTLE CPU. - # Format: dict(CPU implementer: set(CPU part numbers)) - LITTLE_CORES = { - "0x41": { - "0xd01", # Cortex A32 - "0xd03", # Cortex A53 - "0xd04", # Cortex A35 - "0xd05", # Cortex A55 - }, - } - BIG_CORES = { - "0x41": { - "0xd07", # Cortex A57 - "0xd08", # Cortex A72 - "0xd09", # Cortex A73 - "0xd0a", # Cortex A75 - "0xd0b", # Cortex A76 - }, - } - - # Values of CPU Implementer and CPU part number are exposed by cpuinfo. - # Format: - # ================= - # processor : 0 - # model name : ARMv8 Processor rev 4 (v8l) - # BogoMIPS : 48.00 - # Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 - # CPU implementer : 0x41 - # CPU architecture: 8 - # CPU variant : 0x0 - # CPU part : 0xd03 - # CPU revision : 4 - - _, cpuinfo, _ = self.RunCommandOnDut("cat /proc/cpuinfo") - - # List of all CPU cores: 0, 1, .. - proc_matches = re.findall( - r"^processor\s*: (\d+)$", cpuinfo, re.MULTILINE - ) - # List of all corresponding CPU implementers - impl_matches = re.findall( - r"^CPU implementer\s*: (0x[\da-f]+)$", cpuinfo, re.MULTILINE - ) - # List of all corresponding CPU part numbers - part_matches = re.findall( - r"^CPU part\s*: (0x[\da-f]+)$", cpuinfo, re.MULTILINE - ) - assert len(proc_matches) == len(impl_matches) - assert len(part_matches) == len(impl_matches) - - all_cores = set(proc_matches) - dut_big_cores = { - core - for core, impl, part in zip( - proc_matches, impl_matches, part_matches - ) - if impl in BIG_CORES and part in BIG_CORES[impl] - } - dut_lit_cores = { - core - for core, impl, part in zip( - proc_matches, impl_matches, part_matches - ) - if impl in LITTLE_CORES and part in LITTLE_CORES[impl] - } - - if self.dut_config["cpu_usage"] == "big_only": - cores_to_enable = dut_big_cores - cores_to_disable = all_cores - dut_big_cores - elif self.dut_config["cpu_usage"] == "little_only": - cores_to_enable = dut_lit_cores - cores_to_disable = all_cores - dut_lit_cores - else: - self.logger.LogError( - "cpu_usage=%s is not supported on ARM.\n" - "Ignore ARM CPU setup and continue." - % self.dut_config["cpu_usage"] - ) - return - - if cores_to_enable: - cmd_enable_cores = ( - "echo 1 | tee /sys/devices/system/cpu/cpu{%s}/online" - % ",".join(sorted(cores_to_enable)) - ) - - cmd_disable_cores = "" - if cores_to_disable: - cmd_disable_cores = ( - "echo 0 | tee /sys/devices/system/cpu/cpu{%s}/online" - % ",".join(sorted(cores_to_disable)) - ) - - self.RunCommandOnDut( - "; ".join([cmd_enable_cores, cmd_disable_cores]) - ) - else: - # If there are no cores enabled by dut_config then configuration - # is invalid for current platform and should be ignored. - self.logger.LogError( - '"cpu_usage" is invalid for targeted platform.\n' - "dut_config[cpu_usage]=%s\n" - "dut big cores: %s\n" - "dut little cores: %s\n" - "Ignore ARM CPU setup and continue." - % (self.dut_config["cpu_usage"], dut_big_cores, dut_lit_cores) - ) - - def GetCpuOnline(self): - """Get online status of CPU cores. - - Return dict of {int(cpu_num): <0|1>}. - """ - get_cpu_online_cmd = ( - 'paste -d" "' - " <(ls /sys/devices/system/cpu/cpu*/online)" - " <(cat /sys/devices/system/cpu/cpu*/online)" - ) - _, online_output_str, _ = self.RunCommandOnDut(get_cpu_online_cmd) - - # Here is the output we expect to see: - # ----------------- - # /sys/devices/system/cpu/cpu0/online 0 - # /sys/devices/system/cpu/cpu1/online 1 - - cpu_online = {} - cpu_online_match = re.compile(r"^[/\S]+/cpu(\d+)/[/\S]+\s+(\d+)$") - for line in online_output_str.splitlines(): - match = cpu_online_match.match(line) - if match: - cpu = int(match.group(1)) - status = int(match.group(2)) - cpu_online[cpu] = status - # At least one CPU has to be online. - assert cpu_online - - return cpu_online - - def SetupCpuFreq(self, online_cores): - """Setup CPU frequency. - - Based on self.dut_config['cpu_freq_pct'] setup frequency of online CPU cores - to a supported value which is less or equal to (freq_pct * max_freq / 100) - limited by min_freq. - - NOTE: scaling_available_frequencies support is required. - Otherwise the function has no effect. - """ - freq_percent = self.dut_config["cpu_freq_pct"] - list_all_avail_freq_cmd = ( - "ls /sys/devices/system/cpu/cpu{%s}/cpufreq/" - "scaling_available_frequencies" - ) - # Ignore error to support general usage of frequency setup. - # Not all platforms support scaling_available_frequencies. - ret, all_avail_freq_str, _ = self.RunCommandOnDut( - list_all_avail_freq_cmd - % ",".join(str(core) for core in online_cores), - ignore_status=True, - ) - if ret or not all_avail_freq_str: - # No scalable frequencies available for the core. - return ret - for avail_freq_path in all_avail_freq_str.split(): - # Get available freq from every scaling_available_frequency path. - # Error is considered fatal in self.RunCommandOnDut(). - _, avail_freq_str, _ = self.RunCommandOnDut( - "cat " + avail_freq_path - ) - assert avail_freq_str - - all_avail_freq = sorted( - int(freq_str) for freq_str in avail_freq_str.split() - ) - min_freq = all_avail_freq[0] - max_freq = all_avail_freq[-1] - # Calculate the frequency we are targeting. - target_freq = round(max_freq * freq_percent / 100) - # More likely it's not in the list of supported frequencies - # and our goal is to find the one which is less or equal. - # Default is min and we will try to maximize it. - avail_ngt_target = min_freq - # Find the largest not greater than the target. - for next_largest in reversed(all_avail_freq): - if next_largest <= target_freq: - avail_ngt_target = next_largest - break - - max_freq_path = avail_freq_path.replace( - "scaling_available_frequencies", "scaling_max_freq" - ) - min_freq_path = avail_freq_path.replace( - "scaling_available_frequencies", "scaling_min_freq" - ) - # With default ignore_status=False we expect 0 status or Fatal error. - self.RunCommandOnDut( - "echo %s | tee %s %s" - % (avail_ngt_target, max_freq_path, min_freq_path) - ) - - def WaitCooldown(self): - """Wait for DUT to cool down to certain temperature.""" - waittime = 0 - timeout_in_sec = int(self.dut_config["cooldown_time"]) * 60 - # Temperature from sensors come in uCelsius units. - temp_in_ucels = int(self.dut_config["cooldown_temp"]) * 1000 - sleep_interval = 30 - - # Wait until any of two events occurs: - # 1. CPU cools down to a specified temperature. - # 2. Timeout cooldown_time expires. - # For the case when targeted temperature is not reached within specified - # timeout the benchmark is going to start with higher initial CPU temp. - # In the worst case it may affect test results but at the same time we - # guarantee the upper bound of waiting time. - # TODO(denik): Report (or highlight) "high" CPU temperature in test results. - # "high" should be calculated based on empirical data per platform. - # Based on such reports we can adjust CPU configuration or - # cooldown limits accordingly. - while waittime < timeout_in_sec: - _, temp_output, _ = self.RunCommandOnDut( - "cat /sys/class/thermal/thermal_zone*/temp", ignore_status=True - ) - if any(int(temp) > temp_in_ucels for temp in temp_output.split()): - time.sleep(sleep_interval) - waittime += sleep_interval - else: - # Exit the loop when: - # 1. Reported temp numbers from all thermal sensors do not exceed - # 'cooldown_temp' or - # 2. No data from the sensors. - break - - self.logger.LogOutput("Cooldown wait time: %.1f min" % (waittime / 60)) - return waittime - - def DecreaseWaitTime(self): - """Change the ten seconds wait time for pagecycler to two seconds.""" - FILE = ( - "/usr/local/telemetry/src/tools/perf/page_sets/page_cycler_story.py" - ) - ret = self.RunCommandOnDut("ls " + FILE) - - if not ret: - sed_command = 'sed -i "s/_TTI_WAIT_TIME = 10/_TTI_WAIT_TIME = 2/g" ' - self.RunCommandOnDut(sed_command + FILE) - - def StopUI(self): - """Stop UI on DUT.""" - # Added "ignore_status" for the case when crosperf stops ui service which - # was already stopped. Command is going to fail with 1. - self.RunCommandOnDut("stop ui", ignore_status=True) - - def StartUI(self): - """Start UI on DUT.""" - # Similar to StopUI, `start ui` fails if the service is already started. - self.RunCommandOnDut("start ui", ignore_status=True) - - def KerncmdUpdateNeeded(self, intel_pstate): - """Check whether kernel cmdline update is needed. - - Args: - intel_pstate: kernel command line argument (active, passive, no_hwp) - - Returns: - True if update is needed. - """ - - good = 0 - - # Check that dut platform supports hwp - cmd = "grep -q '^flags.*hwp' /proc/cpuinfo" - ret_code, _, _ = self.RunCommandOnDut(cmd, ignore_status=True) - if ret_code != good: - # Intel hwp is not supported, update is not needed. - return False - - kern_cmdline_cmd = ( - 'grep -q "intel_pstate=%s" /proc/cmdline' % intel_pstate - ) - ret_code, _, _ = self.RunCommandOnDut( - kern_cmdline_cmd, ignore_status=True - ) - self.logger.LogOutput("grep /proc/cmdline returned %d" % ret_code) - if ( - intel_pstate - and ret_code == good - or not intel_pstate - and ret_code != good - ): - # No need to updated cmdline if: - # 1. We are setting intel_pstate and we found it is already set. - # 2. Not using intel_pstate and it is not in cmdline. - return False - - # Otherwise we need to update intel_pstate. - return True - - def UpdateKerncmdIntelPstate(self, intel_pstate): - """Update kernel command line. - - Args: - intel_pstate: kernel command line argument (active, passive, no_hwp) - """ - - good = 0 - - # First phase is to remove rootfs verification to allow cmdline change. - remove_verif_cmd = " ".join( - [ - "/usr/share/vboot/bin/make_dev_ssd.sh", - "--remove_rootfs_verification", - "--partition %d", - ] - ) - # Command for partition 2. - verif_part2_failed, _, _ = self.RunCommandOnDut( - remove_verif_cmd % 2, ignore_status=True - ) - # Command for partition 4 - # Some machines in the lab use partition 4 to boot from, - # so cmdline should be update for both partitions. - verif_part4_failed, _, _ = self.RunCommandOnDut( - remove_verif_cmd % 4, ignore_status=True - ) - if verif_part2_failed or verif_part4_failed: - self.logger.LogFatal( - "ERROR. Failed to update kernel cmdline on partition %d.\n" - "Remove verification failed with status %d" - % ( - 2 if verif_part2_failed else 4, - verif_part2_failed or verif_part4_failed, - ) - ) - - self.RunCommandOnDut("reboot && exit") - # Give enough time for dut to complete reboot - # TODO(denik): Replace with the function checking machine availability. - time.sleep(30) - - # Second phase to update intel_pstate in kernel cmdline. - kern_cmdline = "\n".join( - [ - "tmpfile=$(mktemp)", - "partnumb=%d", - "pstate=%s", - # Store kernel cmdline in a temp file. - "/usr/share/vboot/bin/make_dev_ssd.sh --partition ${partnumb}" - " --save_config ${tmpfile}", - # Remove intel_pstate argument if present. - "sed -i -r 's/ intel_pstate=[A-Za-z_]+//g' ${tmpfile}.${partnumb}", - # Insert intel_pstate with a new value if it is set. - "[[ -n ${pstate} ]] &&" - ' sed -i -e "s/ *$/ intel_pstate=${pstate}/" ${tmpfile}.${partnumb}', - # Save the change in kernel cmdline. - # After completion we have to reboot. - "/usr/share/vboot/bin/make_dev_ssd.sh --partition ${partnumb}" - " --set_config ${tmpfile}", - ] - ) - kern_part2_cmdline_cmd = kern_cmdline % (2, intel_pstate) - self.logger.LogOutput( - "Command to change kernel command line: %s" % kern_part2_cmdline_cmd - ) - upd_part2_failed, _, _ = self.RunCommandOnDut( - kern_part2_cmdline_cmd, ignore_status=True - ) - # Again here we are updating cmdline for partition 4 - # in addition to partition 2. Without this some machines - # in the lab might fail. - kern_part4_cmdline_cmd = kern_cmdline % (4, intel_pstate) - self.logger.LogOutput( - "Command to change kernel command line: %s" % kern_part4_cmdline_cmd - ) - upd_part4_failed, _, _ = self.RunCommandOnDut( - kern_part4_cmdline_cmd, ignore_status=True - ) - if upd_part2_failed or upd_part4_failed: - self.logger.LogFatal( - "ERROR. Failed to update kernel cmdline on partition %d.\n" - "intel_pstate update failed with status %d" - % ( - 2 if upd_part2_failed else 4, - upd_part2_failed or upd_part4_failed, - ) - ) - - self.RunCommandOnDut("reboot && exit") - # Wait 30s after reboot. - time.sleep(30) - - # Verification phase. - # Check that cmdline was updated. - # Throw an exception if not. - kern_cmdline_cmd = ( - 'grep -q "intel_pstate=%s" /proc/cmdline' % intel_pstate - ) - ret_code, _, _ = self.RunCommandOnDut( - kern_cmdline_cmd, ignore_status=True - ) - if ( - intel_pstate - and ret_code != good - or not intel_pstate - and ret_code == good - ): - # Kernel cmdline doesn't match input intel_pstate. - self.logger.LogFatal( - "ERROR. Failed to update kernel cmdline. " - "Final verification failed with status %d" % ret_code - ) - - self.logger.LogOutput("Kernel cmdline updated successfully.") - - @contextmanager - def PauseUI(self): - """Stop UI before and Start UI after the context block. - - Context manager will make sure UI is always resumed at the end. - """ - self.StopUI() - try: - yield - - finally: - self.StartUI() - - def SetupDevice(self): - """Setup device to get it ready for testing. - - @Returns Wait time of cool down for this benchmark run. - """ - self.logger.LogOutput("Update kernel cmdline if necessary and reboot") - intel_pstate = self.dut_config["intel_pstate"] - if intel_pstate and self.KerncmdUpdateNeeded(intel_pstate): - self.UpdateKerncmdIntelPstate(intel_pstate) - - wait_time = 0 - # Pause UI while configuring the DUT. - # This will accelerate setup (waiting for cooldown has x10 drop) - # and help to reset a Chrome state left after the previous test. - with self.PauseUI(): - # Unless the user turns on ASLR in the flag, we first disable ASLR - # before running the benchmarks - if not self.dut_config["enable_aslr"]: - self.DisableASLR() - - # CPU usage setup comes first where we enable/disable cores. - self.SetupCpuUsage() - cpu_online_status = self.GetCpuOnline() - # List of online cores of type int (core number). - online_cores = [ - core for core, status in cpu_online_status.items() if status - ] - if self.dut_config["cooldown_time"]: - # Setup power conservative mode for effective cool down. - # Set ignore status since powersave may no be available - # on all platforms and we are going to handle it. - ret = self.SetCpuGovernor("powersave", ignore_status=True) - if ret: - # "powersave" is not available, use "ondemand". - # Still not a fatal error if it fails. - ret = self.SetCpuGovernor("ondemand", ignore_status=True) - # TODO(denik): Run comparison test for 'powersave' and 'ondemand' - # on scarlet and kevin64. - # We might have to consider reducing freq manually to the min - # if it helps to reduce waiting time. - wait_time = self.WaitCooldown() - - # Setup CPU governor for the benchmark run. - # It overwrites the previous governor settings. - governor = self.dut_config["governor"] - # FIXME(denik): Pass online cores to governor setup. - self.SetCpuGovernor(governor) - - # Disable Turbo and Setup CPU freq should ALWAYS proceed governor setup - # since governor may change: - # - frequency; - # - turbo/boost. - self.DisableTurbo() - self.SetupCpuFreq(online_cores) - - self.DecreaseWaitTime() - # FIXME(denik): Currently we are not recovering the previous cpufreq - # settings since we do reboot/setup every time anyway. - # But it may change in the future and then we have to recover the - # settings. - return wait_time diff --git a/cros_utils/device_setup_utils_unittest.py b/cros_utils/device_setup_utils_unittest.py deleted file mode 100755 index d7339e25..00000000 --- a/cros_utils/device_setup_utils_unittest.py +++ /dev/null @@ -1,805 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2019 The ChromiumOS Authors -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Unittest for device_setup_utils.""" - - -import time -import unittest -from unittest import mock - -from cros_utils import command_executer -from cros_utils import logger -from cros_utils.device_setup_utils import DutWrapper - - -BIG_LITTLE_CPUINFO = """processor : 0 -model name : ARMv8 Processor rev 4 (v8l) -BogoMIPS : 48.00 -Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 -CPU implementer : 0x41 -CPU architecture: 8 -CPU variant : 0x0 -CPU part : 0xd03 -CPU revision : 4 - -processor : 1 -model name : ARMv8 Processor rev 4 (v8l) -BogoMIPS : 48.00 -Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 -CPU implementer : 0x41 -CPU architecture: 8 -CPU variant : 0x0 -CPU part : 0xd03 -CPU revision : 4 - -processor : 2 -model name : ARMv8 Processor rev 2 (v8l) -BogoMIPS : 48.00 -Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 -CPU implementer : 0x41 -CPU architecture: 8 -CPU variant : 0x0 -CPU part : 0xd08 -CPU revision : 2 -""" -LITTLE_ONLY_CPUINFO = """processor : 0 -model name : ARMv8 Processor rev 4 (v8l) -BogoMIPS : 48.00 -Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 -CPU implementer : 0x41 -CPU architecture: 8 -CPU variant : 0x0 -CPU part : 0xd03 -CPU revision : 4 - -processor : 1 -model name : ARMv8 Processor rev 4 (v8l) -BogoMIPS : 48.00 -Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 -CPU implementer : 0x41 -CPU architecture: 8 -CPU variant : 0x0 -CPU part : 0xd03 -CPU revision : 4 -""" - -NOT_BIG_LITTLE_CPUINFO = """processor : 0 -model name : ARMv7 Processor rev 1 (v7l) -Features : swp half thumb fastmult vfp edsp thumbee neon vfpv3 tls vfpv4 -CPU implementer : 0x41 -CPU architecture: 7 -CPU variant : 0x0 -CPU part : 0xc0d -CPU revision : 1 - -processor : 1 -model name : ARMv7 Processor rev 1 (v7l) -Features : swp half thumb fastmult vfp edsp thumbee neon vfpv3 tls vfpv4 -CPU implementer : 0x41 -CPU architecture: 7 -CPU variant : 0x0 -CPU part : 0xc0d -CPU revision : 1 - -Hardware : Rockchip (Device Tree) -Revision : 0000 -Serial : 0000000000000000 -""" - - -class DutWrapperTest(unittest.TestCase): - """Class of DutWrapper test.""" - - real_logger = logger.GetLogger() - - mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) - mock_logger = mock.Mock(spec=logger.Logger) - - def __init__(self, *args, **kwargs): - super(DutWrapperTest, self).__init__(*args, **kwargs) - - def setUp(self): - self.dw = DutWrapper( - "/tmp/chromeos", - "lumpy.cros2", - log_level="verbose", - logger=self.mock_logger, - ce=self.mock_cmd_exec, - dut_config={}, - ) - - @mock.patch.object( - command_executer.CommandExecuter, "CrosRunCommandWOutput" - ) - def test_run_command_on_dut(self, mock_cros_runcmd): - self.mock_cmd_exec.CrosRunCommandWOutput = mock_cros_runcmd - mock_cros_runcmd.return_value = (0, "", "") - mock_cros_runcmd.assert_not_called() - self.dw.RunCommandOnDut("run command;") - mock_cros_runcmd.assert_called_once_with( - "run command;", chromeos_root="/tmp/chromeos", machine="lumpy.cros2" - ) - - @mock.patch.object( - command_executer.CommandExecuter, "CrosRunCommandWOutput" - ) - def test_dut_wrapper_fatal_error(self, mock_cros_runcmd): - self.mock_cmd_exec.CrosRunCommandWOutput = mock_cros_runcmd - # Command returns error 1. - mock_cros_runcmd.return_value = (1, "", "Error!") - mock_cros_runcmd.assert_not_called() - self.dw.RunCommandOnDut("run command;") - mock_cros_runcmd.assert_called_once_with( - "run command;", chromeos_root="/tmp/chromeos", machine="lumpy.cros2" - ) - # Error status causes log fatal. - self.assertEqual( - self.mock_logger.method_calls[-1], - mock.call.LogFatal( - "Command execution on DUT lumpy.cros2 failed.\n" - "Failing command: run command;\nreturned 1\n" - "Error message: Error!" - ), - ) - - @mock.patch.object( - command_executer.CommandExecuter, "CrosRunCommandWOutput" - ) - def test_dut_wrapper_ignore_error(self, mock_cros_runcmd): - self.mock_cmd_exec.CrosRunCommandWOutput = mock_cros_runcmd - # Command returns error 1. - mock_cros_runcmd.return_value = (1, "", "Error!") - self.dw.RunCommandOnDut("run command;", ignore_status=True) - mock_cros_runcmd.assert_called_once_with( - "run command;", chromeos_root="/tmp/chromeos", machine="lumpy.cros2" - ) - # Error status is not fatal. LogError records the error message. - self.assertEqual( - self.mock_logger.method_calls[-1], - mock.call.LogError( - "Command execution on DUT lumpy.cros2 failed.\n" - "Failing command: run command;\nreturned 1\n" - "Error message: Error!\n" - "(Failure is considered non-fatal. Continue.)" - ), - ) - - def test_disable_aslr(self): - self.dw.RunCommandOnDut = mock.Mock(return_value=(0, "", "")) - self.dw.DisableASLR() - # pyformat: disable - set_cpu_cmd = ( - "set -e; " - "if [[ -e /proc/sys/kernel/randomize_va_space ]]; then " - " echo 0 > /proc/sys/kernel/randomize_va_space; " - "fi" - ) - self.dw.RunCommandOnDut.assert_called_once_with(set_cpu_cmd) - - def test_set_cpu_governor(self): - self.dw.RunCommandOnDut = mock.Mock(return_value=(0, "", "")) - self.dw.SetCpuGovernor("new_governor", ignore_status=False) - set_cpu_cmd = ( - "for f in `ls -d /sys/devices/system/cpu/cpu*/cpufreq 2>/dev/null`; do " - # Skip writing scaling_governor if cpu is offline. - " [[ -e ${f/cpufreq/online} ]] && grep -q 0 ${f/cpufreq/online} " - " && continue; " - " cd $f; " - " if [[ -e scaling_governor ]]; then " - " echo %s > scaling_governor; fi; " - "done; " - ) - self.dw.RunCommandOnDut.assert_called_once_with( - set_cpu_cmd % "new_governor", ignore_status=False - ) - - def test_set_cpu_governor_propagate_error(self): - self.dw.RunCommandOnDut = mock.Mock(return_value=(1, "", "Error.")) - self.dw.SetCpuGovernor("non-exist_governor") - set_cpu_cmd = ( - "for f in `ls -d /sys/devices/system/cpu/cpu*/cpufreq 2>/dev/null`; do " - # Skip writing scaling_governor if cpu is not online. - " [[ -e ${f/cpufreq/online} ]] && grep -q 0 ${f/cpufreq/online} " - " && continue; " - " cd $f; " - " if [[ -e scaling_governor ]]; then " - " echo %s > scaling_governor; fi; " - "done; " - ) - # By default error status is fatal. - self.dw.RunCommandOnDut.assert_called_once_with( - set_cpu_cmd % "non-exist_governor", ignore_status=False - ) - - def test_set_cpu_governor_ignore_status(self): - self.dw.RunCommandOnDut = mock.Mock(return_value=(1, "", "Error.")) - ret_code = self.dw.SetCpuGovernor( - "non-exist_governor", ignore_status=True - ) - set_cpu_cmd = ( - "for f in `ls -d /sys/devices/system/cpu/cpu*/cpufreq 2>/dev/null`; do " - # Skip writing scaling_governor if cpu is not online. - " [[ -e ${f/cpufreq/online} ]] && grep -q 0 ${f/cpufreq/online} " - " && continue; " - " cd $f; " - " if [[ -e scaling_governor ]]; then " - " echo %s > scaling_governor; fi; " - "done; " - ) - self.dw.RunCommandOnDut.assert_called_once_with( - set_cpu_cmd % "non-exist_governor", ignore_status=True - ) - self.assertEqual(ret_code, 1) - - def test_disable_turbo(self): - self.dw.RunCommandOnDut = mock.Mock(return_value=(0, "", "")) - self.dw.DisableTurbo() - set_cpu_cmd = ( - # Disable Turbo in Intel pstate driver - "if [[ -e /sys/devices/system/cpu/intel_pstate/no_turbo ]]; then " - " if grep -q 0 /sys/devices/system/cpu/intel_pstate/no_turbo; then " - " echo -n 1 > /sys/devices/system/cpu/intel_pstate/no_turbo; " - " fi; " - "fi; " - ) - self.dw.RunCommandOnDut.assert_called_once_with(set_cpu_cmd) - - def test_get_cpu_online_two(self): - """Test one digit CPU #.""" - self.dw.RunCommandOnDut = mock.Mock( - return_value=( - 0, - "/sys/devices/system/cpu/cpu0/online 0\n" - "/sys/devices/system/cpu/cpu1/online 1\n", - "", - ) - ) - cpu_online = self.dw.GetCpuOnline() - self.assertEqual(cpu_online, {0: 0, 1: 1}) - - def test_get_cpu_online_twelve(self): - """Test two digit CPU #.""" - self.dw.RunCommandOnDut = mock.Mock( - return_value=( - 0, - "/sys/devices/system/cpu/cpu0/online 1\n" - "/sys/devices/system/cpu/cpu1/online 0\n" - "/sys/devices/system/cpu/cpu10/online 1\n" - "/sys/devices/system/cpu/cpu11/online 1\n" - "/sys/devices/system/cpu/cpu2/online 1\n" - "/sys/devices/system/cpu/cpu3/online 0\n" - "/sys/devices/system/cpu/cpu4/online 1\n" - "/sys/devices/system/cpu/cpu5/online 0\n" - "/sys/devices/system/cpu/cpu6/online 1\n" - "/sys/devices/system/cpu/cpu7/online 0\n" - "/sys/devices/system/cpu/cpu8/online 1\n" - "/sys/devices/system/cpu/cpu9/online 0\n", - "", - ) - ) - cpu_online = self.dw.GetCpuOnline() - self.assertEqual( - cpu_online, - { - 0: 1, - 1: 0, - 2: 1, - 3: 0, - 4: 1, - 5: 0, - 6: 1, - 7: 0, - 8: 1, - 9: 0, - 10: 1, - 11: 1, - }, - ) - - def test_get_cpu_online_no_output(self): - """Test error case, no output.""" - self.dw.RunCommandOnDut = mock.Mock(return_value=(0, "", "")) - with self.assertRaises(AssertionError): - self.dw.GetCpuOnline() - - def test_get_cpu_online_command_error(self): - """Test error case, command error.""" - self.dw.RunCommandOnDut = mock.Mock(side_effect=AssertionError) - with self.assertRaises(AssertionError): - self.dw.GetCpuOnline() - - @mock.patch.object(DutWrapper, "SetupArmCores") - def test_setup_cpu_usage_little_on_arm(self, mock_setup_arm): - self.dw.SetupArmCores = mock_setup_arm - self.dw.RunCommandOnDut = mock.Mock(return_value=(0, "armv7l", "")) - self.dw.dut_config["cpu_usage"] = "little_only" - self.dw.SetupCpuUsage() - self.dw.SetupArmCores.assert_called_once_with() - - @mock.patch.object(DutWrapper, "SetupArmCores") - def test_setup_cpu_usage_big_on_aarch64(self, mock_setup_arm): - self.dw.SetupArmCores = mock_setup_arm - self.dw.RunCommandOnDut = mock.Mock(return_value=(0, "aarch64", "")) - self.dw.dut_config["cpu_usage"] = "big_only" - self.dw.SetupCpuUsage() - self.dw.SetupArmCores.assert_called_once_with() - - @mock.patch.object(DutWrapper, "SetupArmCores") - def test_setup_cpu_usage_big_on_intel(self, mock_setup_arm): - self.dw.SetupArmCores = mock_setup_arm - self.dw.RunCommandOnDut = mock.Mock(return_value=(0, "x86_64", "")) - self.dw.dut_config["cpu_usage"] = "big_only" - self.dw.SetupCpuUsage() - # Check that SetupArmCores not called with invalid setup. - self.dw.SetupArmCores.assert_not_called() - - @mock.patch.object(DutWrapper, "SetupArmCores") - def test_setup_cpu_usage_all_on_intel(self, mock_setup_arm): - self.dw.SetupArmCores = mock_setup_arm - self.dw.RunCommandOnDut = mock.Mock(return_value=(0, "x86_64", "")) - self.dw.dut_config["cpu_usage"] = "all" - self.dw.SetupCpuUsage() - # Check that SetupArmCores not called in general case. - self.dw.SetupArmCores.assert_not_called() - - def test_setup_arm_cores_big_on_big_little(self): - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[ - (0, BIG_LITTLE_CPUINFO, ""), - (0, "", ""), - ] - ) - self.dw.dut_config["cpu_usage"] = "big_only" - self.dw.SetupArmCores() - self.dw.RunCommandOnDut.assert_called_with( - "echo 1 | tee /sys/devices/system/cpu/cpu{2}/online; " - "echo 0 | tee /sys/devices/system/cpu/cpu{0,1}/online" - ) - - def test_setup_arm_cores_little_on_big_little(self): - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[ - (0, BIG_LITTLE_CPUINFO, ""), - (0, "", ""), - ] - ) - self.dw.dut_config["cpu_usage"] = "little_only" - self.dw.SetupArmCores() - self.dw.RunCommandOnDut.assert_called_with( - "echo 1 | tee /sys/devices/system/cpu/cpu{0,1}/online; " - "echo 0 | tee /sys/devices/system/cpu/cpu{2}/online" - ) - - def test_setup_arm_cores_invalid_config(self): - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[ - (0, LITTLE_ONLY_CPUINFO, ""), - (0, "", ""), - ] - ) - self.dw.dut_config["cpu_usage"] = "big_only" - self.dw.SetupArmCores() - # Check that setup command is not sent when trying - # to use 'big_only' on a platform with all little cores. - self.dw.RunCommandOnDut.assert_called_once_with("cat /proc/cpuinfo") - - def test_setup_arm_cores_not_big_little(self): - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[ - (0, NOT_BIG_LITTLE_CPUINFO, ""), - (0, "", ""), - ] - ) - self.dw.dut_config["cpu_usage"] = "big_only" - self.dw.SetupArmCores() - # Check that setup command is not sent when trying - # to use 'big_only' on a platform w/o support of big/little. - self.dw.RunCommandOnDut.assert_called_once_with("cat /proc/cpuinfo") - - def test_setup_arm_cores_unsupported_cpu_usage(self): - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[ - (0, BIG_LITTLE_CPUINFO, ""), - (0, "", ""), - ] - ) - self.dw.dut_config["cpu_usage"] = "exclusive_cores" - self.dw.SetupArmCores() - # Check that setup command is not sent when trying to use - # 'exclusive_cores' on ARM CPU setup. - self.dw.RunCommandOnDut.assert_called_once_with("cat /proc/cpuinfo") - - def test_setup_cpu_freq_single_full(self): - online = [0] - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[ - ( - 0, - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_available_frequencies\n", - "", - ), - (0, "1 2 3 4 5 6 7 8 9 10", ""), - (0, "", ""), - ] - ) - self.dw.dut_config["cpu_freq_pct"] = 100 - self.dw.SetupCpuFreq(online) - self.assertGreaterEqual(self.dw.RunCommandOnDut.call_count, 3) - self.assertEqual( - self.dw.RunCommandOnDut.call_args, - mock.call( - "echo 10 | tee " - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq " - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_min_freq" - ), - ) - - def test_setup_cpu_freq_middle(self): - online = [0] - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[ - ( - 0, - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_available_frequencies\n", - "", - ), - (0, "1 2 3 4 5 6 7 8 9 10", ""), - (0, "", ""), - ] - ) - self.dw.dut_config["cpu_freq_pct"] = 60 - self.dw.SetupCpuFreq(online) - self.assertGreaterEqual(self.dw.RunCommandOnDut.call_count, 2) - self.assertEqual( - self.dw.RunCommandOnDut.call_args, - mock.call( - "echo 6 | tee " - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq " - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_min_freq" - ), - ) - - def test_setup_cpu_freq_lowest(self): - online = [0] - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[ - ( - 0, - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_available_frequencies\n", - "", - ), - (0, "1 2 3 4 5 6 7 8 9 10", ""), - (0, "", ""), - ] - ) - self.dw.dut_config["cpu_freq_pct"] = 0 - self.dw.SetupCpuFreq(online) - self.assertGreaterEqual(self.dw.RunCommandOnDut.call_count, 2) - self.assertEqual( - self.dw.RunCommandOnDut.call_args, - mock.call( - "echo 1 | tee " - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq " - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_min_freq" - ), - ) - - def test_setup_cpu_freq_multiple_middle(self): - online = [0, 1] - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[ - ( - 0, - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_available_frequencies\n" - "/sys/devices/system/cpu/cpu1/cpufreq/scaling_available_frequencies\n", - "", - ), - (0, "1 2 3 4 5 6 7 8 9 10", ""), - (0, "", ""), - (0, "1 4 6 8 10 12 14 16 18 20", ""), - (0, "", ""), - ] - ) - self.dw.dut_config["cpu_freq_pct"] = 70 - self.dw.SetupCpuFreq(online) - self.assertEqual(self.dw.RunCommandOnDut.call_count, 5) - self.assertEqual( - self.dw.RunCommandOnDut.call_args_list[2], - mock.call( - "echo 7 | tee " - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq " - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_min_freq" - ), - ) - self.assertEqual( - self.dw.RunCommandOnDut.call_args_list[4], - mock.call( - "echo 14 | tee " - "/sys/devices/system/cpu/cpu1/cpufreq/scaling_max_freq " - "/sys/devices/system/cpu/cpu1/cpufreq/scaling_min_freq" - ), - ) - - def test_setup_cpu_freq_no_scaling_available(self): - online = [0, 1] - self.dw.RunCommandOnDut = mock.Mock( - return_value=(2, "", "No such file or directory") - ) - self.dw.dut_config["cpu_freq_pct"] = 50 - self.dw.SetupCpuFreq(online) - self.dw.RunCommandOnDut.assert_called_once() - self.assertNotRegex( - self.dw.RunCommandOnDut.call_args_list[0][0][0], - "^echo.*scaling_max_freq$", - ) - - def test_setup_cpu_freq_multiple_no_access(self): - online = [0, 1] - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[ - ( - 0, - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_available_frequencies\n" - "/sys/devices/system/cpu/cpu1/cpufreq/scaling_available_frequencies\n", - "", - ), - (0, "1 4 6 8 10 12 14 16 18 20", ""), - AssertionError(), - ] - ) - self.dw.dut_config["cpu_freq_pct"] = 30 - # Error status causes log fatal. - with self.assertRaises(AssertionError): - self.dw.SetupCpuFreq(online) - - @mock.patch.object(time, "sleep") - def test_wait_cooldown_nowait(self, mock_sleep): - mock_sleep.return_value = 0 - self.dw.RunCommandOnDut = mock.Mock(return_value=(0, "39000", "")) - self.dw.dut_config["cooldown_time"] = 10 - self.dw.dut_config["cooldown_temp"] = 40 - wait_time = self.dw.WaitCooldown() - # Send command to DUT only once to check temperature - # and make sure it does not exceed the threshold. - self.dw.RunCommandOnDut.assert_called_once() - mock_sleep.assert_not_called() - self.assertEqual(wait_time, 0) - - @mock.patch.object(time, "sleep") - def test_wait_cooldown_needwait_once(self, mock_sleep): - """Wait one iteration for cooldown. - - Set large enough timeout and changing temperature - output. Make sure it exits when expected value - received. - Expect that WaitCooldown check temp twice. - """ - mock_sleep.return_value = 0 - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[(0, "41000", ""), (0, "39999", "")] - ) - self.dw.dut_config["cooldown_time"] = 100 - self.dw.dut_config["cooldown_temp"] = 40 - wait_time = self.dw.WaitCooldown() - self.dw.RunCommandOnDut.assert_called() - self.assertEqual(self.dw.RunCommandOnDut.call_count, 2) - mock_sleep.assert_called() - self.assertGreater(wait_time, 0) - - @mock.patch.object(time, "sleep") - def test_wait_cooldown_needwait(self, mock_sleep): - """Test exit by timeout. - - Send command to DUT checking the temperature and - check repeatedly until timeout goes off. - Output from temperature sensor never changes. - """ - mock_sleep.return_value = 0 - self.dw.RunCommandOnDut = mock.Mock(return_value=(0, "41000", "")) - self.dw.dut_config["cooldown_time"] = 60 - self.dw.dut_config["cooldown_temp"] = 40 - wait_time = self.dw.WaitCooldown() - self.dw.RunCommandOnDut.assert_called() - self.assertGreater(self.dw.RunCommandOnDut.call_count, 2) - mock_sleep.assert_called() - self.assertGreater(wait_time, 0) - - @mock.patch.object(time, "sleep") - def test_wait_cooldown_needwait_multtemp(self, mock_sleep): - """Wait until all temps go down. - - Set large enough timeout and changing temperature - output. Make sure it exits when expected value - for all temperatures received. - Expect 3 checks. - """ - mock_sleep.return_value = 0 - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[ - (0, "41000\n20000\n30000\n45000", ""), - (0, "39000\n20000\n30000\n41000", ""), - (0, "39000\n20000\n30000\n31000", ""), - ] - ) - self.dw.dut_config["cooldown_time"] = 100 - self.dw.dut_config["cooldown_temp"] = 40 - wait_time = self.dw.WaitCooldown() - self.dw.RunCommandOnDut.assert_called() - self.assertEqual(self.dw.RunCommandOnDut.call_count, 3) - mock_sleep.assert_called() - self.assertGreater(wait_time, 0) - - @mock.patch.object(time, "sleep") - def test_wait_cooldown_thermal_error(self, mock_sleep): - """Handle error status. - - Any error should be considered non-fatal. - """ - mock_sleep.return_value = 0 - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[ - (1, "39000\n20000\n30000\n41000", "Thermal error"), - (1, "39000\n20000\n30000\n31000", "Thermal error"), - ] - ) - self.dw.dut_config["cooldown_time"] = 10 - self.dw.dut_config["cooldown_temp"] = 40 - wait_time = self.dw.WaitCooldown() - # Check that errors are ignored. - self.dw.RunCommandOnDut.assert_called_with( - "cat /sys/class/thermal/thermal_zone*/temp", ignore_status=True - ) - self.assertEqual(self.dw.RunCommandOnDut.call_count, 2) - # Check that we are waiting even when an error is returned - # as soon as data is coming. - mock_sleep.assert_called() - self.assertGreater(wait_time, 0) - - @mock.patch.object(time, "sleep") - def test_wait_cooldown_thermal_no_output(self, mock_sleep): - """Handle no output. - - Check handling of empty stdout. - """ - mock_sleep.return_value = 0 - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[(1, "", "Thermal error")] - ) - self.dw.dut_config["cooldown_time"] = 10 - self.dw.dut_config["cooldown_temp"] = 40 - wait_time = self.dw.WaitCooldown() - # Check that errors are ignored. - self.dw.RunCommandOnDut.assert_called_once_with( - "cat /sys/class/thermal/thermal_zone*/temp", ignore_status=True - ) - # No wait. - mock_sleep.assert_not_called() - self.assertEqual(wait_time, 0) - - @mock.patch.object(time, "sleep") - def test_wait_cooldown_thermal_ws_output(self, mock_sleep): - """Handle whitespace output. - - Check handling of whitespace only. - """ - mock_sleep.return_value = 0 - self.dw.RunCommandOnDut = mock.Mock( - side_effect=[(1, "\n", "Thermal error")] - ) - self.dw.dut_config["cooldown_time"] = 10 - self.dw.dut_config["cooldown_temp"] = 40 - wait_time = self.dw.WaitCooldown() - # Check that errors are ignored. - self.dw.RunCommandOnDut.assert_called_once_with( - "cat /sys/class/thermal/thermal_zone*/temp", ignore_status=True - ) - # No wait. - mock_sleep.assert_not_called() - self.assertEqual(wait_time, 0) - - def test_stop_ui(self): - self.dw.RunCommandOnDut = mock.Mock(return_value=(0, "", "")) - self.dw.StopUI() - self.dw.RunCommandOnDut.assert_called_once_with( - "stop ui", ignore_status=True - ) - - def test_start_ui(self): - self.dw.RunCommandOnDut = mock.Mock(return_value=(0, "", "")) - self.dw.StartUI() - self.dw.RunCommandOnDut.assert_called_once_with( - "start ui", ignore_status=True - ) - - def test_setup_device(self): - def FakeRunner(command, ignore_status=False): - # pylint fix for unused variable. - del command, ignore_status - return 0, "", "" - - def SetupMockFunctions(): - self.dw.RunCommandOnDut = mock.Mock(return_value=FakeRunner) - self.dw.KerncmdUpdateNeeded = mock.Mock(return_value=True) - self.dw.UpdateKerncmdIntelPstate = mock.Mock(return_value=0) - self.dw.DisableASLR = mock.Mock(return_value=0) - self.dw.SetupCpuUsage = mock.Mock(return_value=0) - self.dw.SetupCpuFreq = mock.Mock(return_value=0) - self.dw.GetCpuOnline = mock.Mock(return_value={0: 1, 1: 1, 2: 0}) - self.dw.SetCpuGovernor = mock.Mock(return_value=0) - self.dw.DisableTurbo = mock.Mock(return_value=0) - self.dw.StopUI = mock.Mock(return_value=0) - self.dw.StartUI = mock.Mock(return_value=0) - self.dw.WaitCooldown = mock.Mock(return_value=0) - self.dw.DecreaseWaitTime = mock.Mock(return_value=0) - - self.dw.dut_config["enable_aslr"] = False - self.dw.dut_config["cooldown_time"] = 0 - self.dw.dut_config["governor"] = "fake_governor" - self.dw.dut_config["cpu_freq_pct"] = 65 - self.dw.dut_config["intel_pstate"] = "no_hwp" - - SetupMockFunctions() - self.dw.SetupDevice() - - self.dw.KerncmdUpdateNeeded.assert_called_once() - self.dw.UpdateKerncmdIntelPstate.assert_called_once() - self.dw.DisableASLR.assert_called_once() - self.dw.SetupCpuUsage.assert_called_once_with() - self.dw.SetupCpuFreq.assert_called_once_with([0, 1]) - self.dw.GetCpuOnline.assert_called_once_with() - self.dw.SetCpuGovernor.assert_called_once_with("fake_governor") - self.dw.DisableTurbo.assert_called_once_with() - self.dw.DecreaseWaitTime.assert_called_once_with() - self.dw.StopUI.assert_called_once_with() - self.dw.StartUI.assert_called_once_with() - self.dw.WaitCooldown.assert_not_called() - - # Test SetupDevice with cooldown - self.dw.dut_config["cooldown_time"] = 10 - - SetupMockFunctions() - self.dw.GetCpuOnline = mock.Mock(return_value={0: 0, 1: 1}) - - self.dw.SetupDevice() - - self.dw.WaitCooldown.assert_called_once_with() - self.dw.DisableASLR.assert_called_once() - self.dw.DisableTurbo.assert_called_once_with() - self.dw.SetupCpuUsage.assert_called_once_with() - self.dw.SetupCpuFreq.assert_called_once_with([1]) - self.dw.SetCpuGovernor.assert_called() - self.dw.GetCpuOnline.assert_called_once_with() - self.dw.StopUI.assert_called_once_with() - self.dw.StartUI.assert_called_once_with() - self.assertGreater(self.dw.SetCpuGovernor.call_count, 1) - self.assertEqual( - self.dw.SetCpuGovernor.call_args, mock.call("fake_governor") - ) - - # Test SetupDevice with cooldown - SetupMockFunctions() - self.dw.SetupCpuUsage = mock.Mock(side_effect=RuntimeError()) - - with self.assertRaises(RuntimeError): - self.dw.SetupDevice() - - # This call injected an exception. - self.dw.SetupCpuUsage.assert_called_once_with() - # Calls following the expeption are skipped. - self.dw.WaitCooldown.assert_not_called() - self.dw.DisableTurbo.assert_not_called() - self.dw.SetupCpuFreq.assert_not_called() - self.dw.SetCpuGovernor.assert_not_called() - self.dw.GetCpuOnline.assert_not_called() - # Check that Stop/Start UI are always called. - self.dw.StopUI.assert_called_once_with() - self.dw.StartUI.assert_called_once_with() - - -if __name__ == "__main__": - unittest.main() diff --git a/cros_utils/email_sender.py b/cros_utils/email_sender.py index ccf4c1b4..b47c3beb 100755 --- a/cros_utils/email_sender.py +++ b/cros_utils/email_sender.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- - # Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -19,10 +17,9 @@ import getpass import json import os import smtplib +import subprocess import tempfile -from cros_utils import command_executer - X20_PATH = "/google/data/rw/teams/c-compiler-chrome/prod_emails" @@ -31,7 +28,7 @@ X20_PATH = "/google/data/rw/teams/c-compiler-chrome/prod_emails" def AtomicallyWriteFile(file_path): temp_path = file_path + ".in_progress" try: - with open(temp_path, "w") as f: + with open(temp_path, "w", encoding="utf-8") as f: yield f os.rename(temp_path, file_path) except: @@ -39,10 +36,10 @@ def AtomicallyWriteFile(file_path): raise -class EmailSender(object): +class EmailSender: """Utility class to send email through SMTP or SendGMR.""" - class Attachment(object): + class Attachment: """Small class to keep track of attachment info.""" def __init__(self, name, content): @@ -61,27 +58,29 @@ class EmailSender(object): """Enqueues an email in our x20 outbox. These emails ultimately get sent by the machinery in - //depot/google3/googleclient/chrome/chromeos_toolchain/mailer/mail.go. This - kind of sending is intended for accounts that don't have smtp or gmr access - (e.g., role accounts), but can be used by anyone with x20 access. + //depot/google3/googleclient/chrome/chromeos_toolchain/mailer/mail.go. + This kind of sending is intended for accounts that don't have smtp or + gmr access (e.g., role accounts), but can be used by anyone with x20 + access. - All emails are sent from `mdb.c-compiler-chrome+${identifier}@google.com`. + All emails are sent from + `mdb.c-compiler-chrome+${identifier}@google.com`. Args: - subject: email subject. Must be nonempty. - identifier: email identifier, or the text that lands after the `+` in the - "From" email address. Must be nonempty. - well_known_recipients: a list of well-known recipients for the email. - These are translated into addresses by our mailer. - Current potential values for this are ('detective', - 'cwp-team', 'cros-team', 'mage'). Either this or - direct_recipients must be a nonempty list. - direct_recipients: @google.com emails to send addresses to. Either this - or well_known_recipients must be a nonempty list. - text_body: a 'text/plain' email body to send. Either this or html_body - must be a nonempty string. Both may be specified - html_body: a 'text/html' email body to send. Either this or text_body - must be a nonempty string. Both may be specified + subject: email subject. Must be nonempty. + identifier: email identifier, or the text that lands after the + `+` in the "From" email address. Must be nonempty. + well_known_recipients: a list of well-known recipients for the + email. These are translated into addresses by our mailer. + Current potential values for this are ('detective', + 'cwp-team', 'cros-team', 'mage'). Either this or + direct_recipients must be a nonempty list. + direct_recipients: @google.com emails to send addresses to. Either + this or well_known_recipients must be a nonempty list. + text_body: a 'text/plain' email body to send. Either this or + html_body must be a nonempty string. Both may be specified + html_body: a 'text/html' email body to send. Either this or + text_body must be a nonempty string. Both may be specified """ # `str`s act a lot like tuples/lists. Ensure that we're not accidentally # iterating over one of those (or anything else that's sketchy, for that @@ -241,8 +240,6 @@ class EmailSender(object): attachments, ): """Send email via sendgmr program.""" - ce = command_executer.GetCommandExecuter(log_level="none") - if not email_from: email_from = getpass.getuser() + "@google.com" @@ -260,30 +257,29 @@ class EmailSender(object): f.flush() to_be_deleted.append(f.name) - # Fix single-quotes inside the subject. In bash, to escape a single quote - # (e.g 'don't') you need to replace it with '\'' (e.g. 'don'\''t'). To - # make Python read the backslash as a backslash rather than an escape - # character, you need to double it. So... + # Fix single-quotes inside the subject. In bash, to escape a single + # quote (e.g 'don't') you need to replace it with '\'' (e.g. + # 'don'\''t'). To make Python read the backslash as a backslash + # rather than an escape character, you need to double it. So... subject = subject.replace("'", "'\\''") + command = [ + "sendgmr", + f"--to={to_list}", + f"--from={email_from}", + f"--subject={subject}", + ] if msg_type == "html": - command = ( - "sendgmr --to='%s' --from='%s' --subject='%s' " - "--html_file='%s' --body_file=/dev/null" - % (to_list, email_from, subject, f.name) - ) + command += [f"--html_file={f.name}", "--body_file=/dev/null"] else: - command = ( - "sendgmr --to='%s' --from='%s' --subject='%s' " - "--body_file='%s'" % (to_list, email_from, subject, f.name) - ) + command.append(f"--body_file={f.name}") if email_cc: cc_list = ",".join(email_cc) - command += " --cc='%s'" % cc_list + command.append(f"--cc={cc_list}") if email_bcc: bcc_list = ",".join(email_bcc) - command += " --bcc='%s'" % bcc_list + command.append(f"--bcc={bcc_list}") if attachments: attachment_files = [] @@ -302,12 +298,12 @@ class EmailSender(object): f.flush() attachment_files.append(f.name) files = ",".join(attachment_files) - command += " --attachment_files='%s'" % files + command.append(f"--attachment_files={files}") to_be_deleted += attachment_files # Send the message via our own GMR server. - status = ce.RunCommand(command) - return status + completed_process = subprocess.run(command, check=False) + return completed_process.returncode finally: for f in to_be_deleted: diff --git a/cros_utils/logger.py b/cros_utils/logger.py index e9b9d1ba..7df3f8ff 100644 --- a/cros_utils/logger.py +++ b/cros_utils/logger.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,6 +9,7 @@ import os.path import sys import traceback +from typing import Union # TODO(yunlian@google.com): Use GetRoot from misc @@ -19,7 +19,7 @@ def GetRoot(scr_name): return (os.path.dirname(abs_path), os.path.basename(abs_path)) -class Logger(object): +class Logger: """Logging helper class.""" MAX_LOG_FILES = 10 @@ -67,7 +67,7 @@ class Logger(object): def _CreateLogFileHandle(self, name): fd = None try: - fd = open(name, "w") + fd = open(name, "w", encoding="utf-8") except IOError: print("Warning: could not open %s for writing." % name) return fd @@ -215,7 +215,7 @@ class Logger(object): self.stderr.flush() -class MockLogger(object): +class MockLogger: """Logging helper class.""" MAX_LOG_FILES = 10 @@ -363,7 +363,8 @@ def InitLogger(script_name, log_dir, print_console=True, mock=False): """Initialize a global logger. To be called only once.""" # pylint: disable=global-statement global main_logger - assert not main_logger, "The logger has already been initialized" + if main_logger: + return main_logger rootdir, basefilename = GetRoot(script_name) if not log_dir: log_dir = rootdir @@ -371,12 +372,11 @@ def InitLogger(script_name, log_dir, print_console=True, mock=False): main_logger = Logger(log_dir, basefilename, print_console) else: main_logger = MockLogger(log_dir, basefilename, print_console) + return main_logger -def GetLogger(log_dir="", mock=False): - if not main_logger: - InitLogger(sys.argv[0], log_dir, mock=mock) - return main_logger +def GetLogger(log_dir="", mock=False) -> Union[Logger, MockLogger]: + return InitLogger(sys.argv[0], log_dir, mock=mock) def HandleUncaughtExceptions(fun): diff --git a/cros_utils/misc.py b/cros_utils/misc.py index aabb5ad7..72cfb8b8 100644 --- a/cros_utils/misc.py +++ b/cros_utils/misc.py @@ -59,7 +59,7 @@ def ApplySubs(string, *substitutions): def UnitToNumber(unit_num, base=1000): """Convert a number with unit to float.""" - unit_dict = {"kilo": base, "mega": base ** 2, "giga": base ** 3} + unit_dict = {"kilo": base, "mega": base**2, "giga": base**3} unit_num = unit_num.lower() mo = re.search(r"(\d*)(.+)?", unit_num) number = mo.group(1) @@ -90,29 +90,26 @@ def GetRoot(scr_name): def GetChromeOSKeyFile(chromeos_root): return os.path.join( chromeos_root, - "src", - "scripts", - "mod_for_test_scripts", + "chromite", "ssh_keys", "testing_rsa", ) -def GetChrootPath(chromeos_root): - return os.path.join(chromeos_root, "chroot") +def GetInsideChrootPath(chromeos_root, file_path): + sys.path.insert(0, chromeos_root) + from chromite.lib import path_util -def GetInsideChrootPath(chromeos_root, file_path): - if not file_path.startswith(GetChrootPath(chromeos_root)): - raise RuntimeError( - "File: %s doesn't seem to be in the chroot: %s" - % (file_path, chromeos_root) - ) - return file_path[len(GetChrootPath(chromeos_root)) :] + return path_util.ToChrootPath(path=file_path, source_path=chromeos_root) def GetOutsideChrootPath(chromeos_root, file_path): - return os.path.join(GetChrootPath(chromeos_root), file_path.lstrip("/")) + sys.path.insert(0, chromeos_root) + + from chromite.lib import path_util + + return path_util.FromChrootPath(path=file_path, source_path=chromeos_root) def FormatQuotedCommand(command): @@ -126,7 +123,10 @@ def FormatCommands(commands): def GetImageDir(chromeos_root, board): - return os.path.join(chromeos_root, "src", "build", "images", board) + return GetOutsideChrootPath( + chromeos_root, + os.path.join(chromeos_root, "src", "build", "images", board), + ) def LabelLatestImage(chromeos_root, board, label, vanilla_path=None): @@ -305,16 +305,16 @@ def IsFloat(text): def RemoveChromeBrowserObjectFiles(chromeos_root, board): """Remove any object files from all the posible locations.""" - out_dir = os.path.join( - GetChrootPath(chromeos_root), - "var/cache/chromeos-chrome/chrome-src/src/out_%s" % board, + out_dir = GetOutsideChrootPath( + chromeos_root, + "/var/cache/chromeos-chrome/chrome-src/src/out_%s" % board, ) if os.path.exists(out_dir): shutil.rmtree(out_dir) logger.GetLogger().LogCmd("rm -rf %s" % out_dir) - out_dir = os.path.join( - GetChrootPath(chromeos_root), - "var/cache/chromeos-chrome/chrome-src-internal/src/out_%s" % board, + out_dir = GetOutsideChrootPath( + chromeos_root, + "/var/cache/chromeos-chrome/chrome-src-internal/src/out_%s" % board, ) if os.path.exists(out_dir): shutil.rmtree(out_dir) diff --git a/cros_utils/tabulator.py b/cros_utils/tabulator.py index d079ea22..2cfd5d35 100644 --- a/cros_utils/tabulator.py +++ b/cros_utils/tabulator.py @@ -67,13 +67,44 @@ import getpass import math import statistics import sys +from typing import Tuple, Union from cros_utils import misc from cros_utils.email_sender import EmailSender +import numpy as np -# TODO(crbug.com/980719): Drop scipy in the future. -# pylint: disable=import-error -import scipy + +def _ttest_ind( + sample: Union[np.ndarray, list], baseline: Union[np.ndarray, list] +) -> Tuple[float, float]: + """Independent, two-sided student's T test. + + Reimplementation of scipy.stats.ttest_ind. + """ + if isinstance(sample, list): + sample = np.asarray(sample) + if isinstance(baseline, list): + baseline = np.asarray(baseline) + diff = np.mean(sample) - np.mean(baseline) + diff_stderr = np.sqrt(sample.var(ddof=1) + baseline.var(ddof=1)) + t_value = np.mean(diff) / (diff_stderr / np.sqrt(len(sample))) + samples = _sample_student_t(len(sample), 1000) + # Assuming two-sided student's t + if t_value < 0: + # Lower tail + return t_value, 2 * np.sum(samples < t_value) / len(samples) + # Upper tail + return t_value, 2 * np.sum(samples > t_value) / len(samples) + + +def _sample_student_t( + dof: float, num_samples: int +) -> np.ndarray: + # In theory this probably should be memoized. However, + # that's a lot of data points to store in memory for + # the lifetime of the program? + sample_generator = np.random.default_rng() + return sample_generator.standard_t(dof, num_samples) def _AllFloat(values): @@ -141,6 +172,15 @@ class TableGenerator(object): values = _StripNone(values) if _AllFloat(values): values = _GetFloats(values) + values = [ + float(v) + for v in values + if isinstance(v, float) + or isinstance(v, int) + or v.lower() in ("nan", "inf") + ] + if not values: + return float("nan") return max(values) def _GetLowestValue(self, key): @@ -152,6 +192,15 @@ class TableGenerator(object): values = _StripNone(values) if _AllFloat(values): values = _GetFloats(values) + values = [ + float(v) + for v in values + if isinstance(v, float) + or isinstance(v, int) + or v.lower() in ("nan", "inf") + ] + if not values: + return float("nan") return min(values) def _SortKeys(self, keys): @@ -415,7 +464,7 @@ class SamplesTableGenerator(TableGenerator): # weighted_samples we added up. one_dict = {} if run: - one_dict[u"weighted_samples"] = [run, u"samples"] + one_dict["weighted_samples"] = [run, "samples"] one_dict["retval"] = 0 else: one_dict["retval"] = 1 @@ -682,7 +731,7 @@ class PValueResult(ComparisonResult): if len(values) < 2 or len(baseline_values) < 2: cell.value = float("nan") return - _, cell.value = scipy.stats.ttest_ind(values, baseline_values) + _, cell.value = _ttest_ind(values, baseline_values) def _ComputeString(self, cell, values, baseline_values): return float("nan") @@ -940,7 +989,7 @@ class StorageFormat(Format): current += 1 if current: - divisor = base ** current + divisor = base**current cell.string_value = "%1.1f%s" % ( (v / divisor), suffices[current - 1], diff --git a/crosperf/README.md b/crosperf/README.md index 18601b67..f1429513 100644 --- a/crosperf/README.md +++ b/crosperf/README.md @@ -3,8 +3,7 @@ To use these experiment files, replace the board, remote and images placeholders and run crosperf on them. -Further information about crosperf: -https://sites.google.com/a/google.com/chromeos-toolchain-team-home2/home/team-tools-and-scripts/crosperf-cros-image-performance-comparison-tool +Further information about crosperf: https://goto.google.com/crostc-crosperf The final experiment file should look something like the following (but with different actual values for the fields): diff --git a/crosperf/benchmark.py b/crosperf/benchmark.py index f9de0cf3..eb8661e9 100644 --- a/crosperf/benchmark.py +++ b/crosperf/benchmark.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2013 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,10 +6,10 @@ import math +import statistics +from typing import Any -# FIXME(denik): Fix the import in chroot. -# pylint: disable=import-error -from scipy import stats +import numpy as np # See crbug.com/673558 for how these are estimated. @@ -26,10 +25,49 @@ _estimated_stddev = { "loading.desktop": 0.021, # Copied from page_cycler initially } +# Numpy makes it hard to know the real type of some inputs +# and outputs, so this type alias is just for docs. +FloatLike = Any + + +def isf(x: FloatLike, mu=0.0, sigma=1.0, pitch=0.01) -> FloatLike: + """Compute the inverse survival function for value x. + + In the abscence of using scipy.stats.norm's isf(), this function + attempts to re-implement the inverse survival function by calculating + the numerical inverse of the survival function, interpolating between + table values. See bug b/284489250 for details. + + Survival function as defined by: + https://en.wikipedia.org/wiki/Survival_function + + Examples: + >>> -2.0e-16 < isf(0.5) < 2.0e-16 + True + + Args: + x: float or numpy array-like to compute the ISF for. + mu: Center of the underlying normal distribution. + sigma: Spread of the underlying normal distribution. + pitch: Absolute spacing between y-value interpolation points. + + Returns: + float or numpy array-like representing the ISF of `x`. + """ + norm = statistics.NormalDist(mu, sigma) + # np.interp requires a monotonically increasing x table. + # Because the survival table is monotonically decreasing, we have to + # reverse the y_vals too. + y_vals = np.flip(np.arange(-4.0, 4.0, pitch)) + survival_table = np.fromiter( + (1.0 - norm.cdf(y) for y in y_vals), y_vals.dtype + ) + return np.interp(x, survival_table, y_vals) + # Get #samples needed to guarantee a given confidence interval, assuming the # samples follow normal distribution. -def _samples(b): +def _samples(b: str) -> int: # TODO: Make this an option # CI = (0.9, 0.02), i.e., 90% chance that |sample mean - true mean| < 2%. p = 0.9 @@ -39,7 +77,7 @@ def _samples(b): d = _estimated_stddev[b] # Get at least 2 samples so as to calculate standard deviation, which is # needed in T-test for p-value. - n = int(math.ceil((stats.norm.isf((1 - p) / 2) * d / e) ** 2)) + n = int(math.ceil((isf((1 - p) / 2) * d / e) ** 2)) return n if n > 1 else 2 diff --git a/crosperf/crosperf_unittest.py b/crosperf/crosperf_unittest.py index 7b52f2e0..bbcb1712 100755 --- a/crosperf/crosperf_unittest.py +++ b/crosperf/crosperf_unittest.py @@ -67,7 +67,7 @@ class CrosperfTest(unittest.TestCase): settings = crosperf.ConvertOptionsToSettings(options) self.assertIsNotNone(settings) self.assertIsInstance(settings, settings_factory.GlobalSettings) - self.assertEqual(len(settings.fields), 40) + self.assertEqual(len(settings.fields), 42) self.assertTrue(settings.GetField("rerun")) argv = ["crosperf/crosperf.py", "temp.exp"] options, _ = parser.parse_known_args(argv) diff --git a/crosperf/default_remotes b/crosperf/default_remotes index 714385e7..80563664 100644 --- a/crosperf/default_remotes +++ b/crosperf/default_remotes @@ -1,5 +1,5 @@ bob : chromeos8-row12-rack16-host2 -chell : chromeos2-row1-rack10-host2 chromeos2-row1-rack10-host4 +chell : chromeos6-row16-rack5-host6 chromeos6-row16-rack5-host7 coral : chromeos6-row5-rack6-host1 chromeos6-row5-rack6-host3 chromeos6-row5-rack6-host5 elm : chromeos6-row14-rack15-host21 nautilus : chromeos6-row5-rack10-host1 chromeos6-row5-rack10-host3 diff --git a/crosperf/download_images.py b/crosperf/download_images.py index 9a46280d..38d5d68a 100644 --- a/crosperf/download_images.py +++ b/crosperf/download_images.py @@ -8,8 +8,10 @@ import ast import os +import shlex from cros_utils import command_executer +from cros_utils import misc import test_flag @@ -79,7 +81,9 @@ class ImageDownloader(object): ) # Make sure the directory for downloading the image exists. - download_path = os.path.join(chromeos_root, "chroot/tmp", build_id) + download_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", build_id) + ) image_path = os.path.join(download_path, "chromiumos_test_image.bin") if not os.path.exists(download_path): os.makedirs(download_path) @@ -105,18 +109,19 @@ class ImageDownloader(object): def UncompressImage(self, chromeos_root, build_id): # Check to see if the file has already been uncompresssed, etc. - if os.path.exists( + download_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join( - chromeos_root, - "chroot/tmp", + "/tmp", build_id, - "chromiumos_test_image.bin", - ) + ), + ) + if os.path.exists( + os.path.join(download_path, "chromiumos_test_image.bin") ): return # Uncompress and untar the downloaded image. - download_path = os.path.join(chromeos_root, "chroot/tmp", build_id) command = ( "cd %s ; tar -Jxf chromiumos_test_image.tar.xz " % download_path ) @@ -178,7 +183,9 @@ class ImageDownloader(object): ) # Make sure the directory for downloading the package exists. - download_path = os.path.join(chromeos_root, "chroot/tmp", build_id) + download_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", build_id) + ) package_path = os.path.join(download_path, package_file_name) if not os.path.exists(download_path): os.makedirs(download_path) @@ -204,7 +211,9 @@ class ImageDownloader(object): self, chromeos_root, build_id, package_file_name, uncompress_cmd ): # Uncompress file - download_path = os.path.join(chromeos_root, "chroot/tmp", build_id) + download_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", build_id) + ) command = "cd %s ; %s %s" % ( download_path, uncompress_cmd, @@ -252,13 +261,13 @@ class ImageDownloader(object): autotest_server_package_name = "autotest_server_package.tar.bz2" autotest_control_files_name = "control_files.tar" - download_path = os.path.join(chromeos_root, "chroot/tmp", build_id) + download_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", build_id) + ) # Autotest directory relative path wrt chroot autotest_rel_path = os.path.join("/tmp", build_id, "autotest_files") # Absolute Path to download files - autotest_path = os.path.join( - chromeos_root, "chroot/tmp", build_id, "autotest_files" - ) + autotest_path = os.path.join(download_path, "autotest_files") if not os.path.exists(autotest_path): # Quickly verify if the files are present on server @@ -315,12 +324,14 @@ class ImageDownloader(object): # Download autest package files (3 files) debug_archive_name = "debug.tgz" - download_path = os.path.join(chromeos_root, "chroot/tmp", build_id) + download_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", build_id) + ) # Debug directory relative path wrt chroot debug_rel_path = os.path.join("/tmp", build_id, "debug_files") # Debug path to download files - debug_path = os.path.join( - chromeos_root, "chroot/tmp", build_id, "debug_files" + debug_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", build_id, "debug_files") ) if not os.path.exists(debug_path): @@ -338,15 +349,11 @@ class ImageDownloader(object): # File exists on server, download and uncompress it self.DownloadSingleFile(chromeos_root, build_id, debug_archive_name) - self.UncompressSingleFile( - chromeos_root, build_id, debug_archive_name, "tar -xf " - ) # Extract and move debug files into the proper location. - debug_dir = "debug_files/usr/lib" - command = "cd %s ; mkdir -p %s; mv debug %s" % ( - download_path, - debug_dir, - debug_dir, + debug_dir = "debug_files/usr/lib/debug" + command = ( + f"cd {shlex.quote(download_path)}; " + f"mkdir -p {shlex.quote(debug_dir)}" ) if self.log_level != "verbose": self._logger.LogOutput("CMD: %s" % command) @@ -357,6 +364,12 @@ class ImageDownloader(object): "Could not create directory %s" % os.path.join(debug_dir, "debug") ) + self.UncompressSingleFile( + chromeos_root, + build_id, + debug_archive_name, + f"tar -C {shlex.quote(debug_dir)} -xf ", + ) return debug_rel_path diff --git a/crosperf/download_images_unittest.py b/crosperf/download_images_unittest.py index 6a640f80..0e47e757 100755 --- a/crosperf/download_images_unittest.py +++ b/crosperf/download_images_unittest.py @@ -8,6 +8,7 @@ import os +import re import unittest import unittest.mock as mock @@ -20,6 +21,16 @@ import test_flag MOCK_LOGGER = logger.GetLogger(log_dir="", mock=True) +class RegexMatcher: + """A regex matcher, for passing to mocks.""" + + def __init__(self, regex): + self._regex = re.compile(regex) + + def __eq__(self, string): + return self._regex.search(string) is not None + + class ImageDownloaderTestcast(unittest.TestCase): """The image downloader test class.""" @@ -34,7 +45,6 @@ class ImageDownloaderTestcast(unittest.TestCase): @mock.patch.object(os, "makedirs") @mock.patch.object(os.path, "exists") def test_download_image(self, mock_path_exists, mock_mkdirs): - # Set mock and test values. mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) test_chroot = "/usr/local/home/chromeos" @@ -59,50 +69,63 @@ class ImageDownloaderTestcast(unittest.TestCase): image_path, ) - # Verify os.path.exists was called twice, with proper arguments. - self.assertEqual(mock_path_exists.call_count, 2) - mock_path_exists.assert_called_with( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/" - "R36-5814.0.0/chromiumos_test_image.bin" + # Verify os.path.exists was called thrice, with proper arguments. + self.assertEqual(mock_path_exists.call_count, 3) + mock_path_exists.assert_any_call( + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/" + "R36-5814.0.0/chromiumos_test_image.bin" + ) ) mock_path_exists.assert_any_call( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0" + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0" + ) ) + mock_path_exists.assert_any_call("/etc/cros_chroot_version") # Verify we called os.mkdirs self.assertEqual(mock_mkdirs.call_count, 1) mock_mkdirs.assert_called_with( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0" + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0" + ) ) # Verify we called RunCommand once, with proper arguments. self.assertEqual(mock_cmd_exec.RunCommand.call_count, 1) - expected_args = ( + expected_args = RegexMatcher( "/usr/local/home/chromeos/src/chromium/depot_tools/gsutil.py " "cp gs://chromeos-image-archive/lumpy-release/R36-5814.0.0/" "chromiumos_test_image.tar.xz " - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0" + "/usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0" ) mock_cmd_exec.RunCommand.assert_called_with(expected_args) - # Reset the velues in the mocks; set os.path.exists to always return True. + # Reset the velues in the mocks; set os.path.exists to always return + # True (except for "inside chroot" check). mock_path_exists.reset_mock() mock_cmd_exec.reset_mock() - mock_path_exists.return_value = True + mock_path_exists.side_effect = lambda x: x != "/etc/cros_chroot_version" # Run downloader downloader.DownloadImage(test_chroot, test_build_id, image_path) - # Verify os.path.exists was called twice, with proper arguments. - self.assertEqual(mock_path_exists.call_count, 2) + # Verify os.path.exists was called thrice, with proper arguments. + self.assertEqual(mock_path_exists.call_count, 3) mock_path_exists.assert_called_with( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/" - "R36-5814.0.0/chromiumos_test_image.bin" + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/" + "R36-5814.0.0/chromiumos_test_image.bin" + ) ) mock_path_exists.assert_any_call( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0" + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0" + ) ) + mock_path_exists.assert_any_call("/etc/cros_chroot_version") # Verify we made no RunCommand or ChrootRunCommand calls (since # os.path.exists returned True, there was no work do be done). @@ -111,7 +134,6 @@ class ImageDownloaderTestcast(unittest.TestCase): @mock.patch.object(os.path, "exists") def test_uncompress_image(self, mock_path_exists): - # set mock and test values. mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) test_chroot = "/usr/local/home/chromeos" @@ -130,12 +152,15 @@ class ImageDownloaderTestcast(unittest.TestCase): test_build_id, ) - # Verify os.path.exists was called once, with correct arguments. - self.assertEqual(mock_path_exists.call_count, 1) + # Verify os.path.exists was called twice, with correct arguments. + self.assertEqual(mock_path_exists.call_count, 2) mock_path_exists.assert_called_with( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/" - "R36-5814.0.0/chromiumos_test_image.bin" + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/" + "R36-5814.0.0/chromiumos_test_image.bin" + ) ) + mock_path_exists.assert_any_call("/etc/cros_chroot_version") # Verify RunCommand was called twice with correct arguments. self.assertEqual(mock_cmd_exec.RunCommand.call_count, 2) @@ -143,8 +168,10 @@ class ImageDownloaderTestcast(unittest.TestCase): self.assertEqual(len(mock_cmd_exec.RunCommand.call_args_list[0]), 2) actual_arg = mock_cmd_exec.RunCommand.call_args_list[0][0] expected_arg = ( - "cd /usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0 ; " - "tar -Jxf chromiumos_test_image.tar.xz ", + RegexMatcher( + "cd /usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0 ; " + "tar -Jxf chromiumos_test_image.tar.xz " + ), ) self.assertEqual(expected_arg, actual_arg) # 2nd arg must be exception handler @@ -158,8 +185,10 @@ class ImageDownloaderTestcast(unittest.TestCase): self.assertEqual(len(mock_cmd_exec.RunCommand.call_args_list[1]), 2) actual_arg = mock_cmd_exec.RunCommand.call_args_list[1][0] expected_arg = ( - "cd /usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0 ; " - "rm -f chromiumos_test_image.bin ", + RegexMatcher( + "cd /usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0 ; " + "rm -f chromiumos_test_image.bin " + ), ) self.assertEqual(expected_arg, actual_arg) # 2nd arg must be empty @@ -167,24 +196,27 @@ class ImageDownloaderTestcast(unittest.TestCase): "{}" in repr(mock_cmd_exec.RunCommand.call_args_list[1][1]) ) - # Set os.path.exists to always return True and run uncompress. + # Set os.path.exists to always return True (except for "inside chroot" + # check) and run uncompress. mock_path_exists.reset_mock() mock_cmd_exec.reset_mock() - mock_path_exists.return_value = True + mock_path_exists.side_effect = lambda x: x != "/etc/cros_chroot_version" downloader.UncompressImage(test_chroot, test_build_id) # Verify os.path.exists was called once, with correct arguments. - self.assertEqual(mock_path_exists.call_count, 1) + self.assertEqual(mock_path_exists.call_count, 2) mock_path_exists.assert_called_with( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/" - "R36-5814.0.0/chromiumos_test_image.bin" + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/" + "R36-5814.0.0/chromiumos_test_image.bin" + ) ) + mock_path_exists.assert_any_call("/etc/cros_chroot_version") # Verify RunCommand was not called. self.assertEqual(mock_cmd_exec.RunCommand.call_count, 0) def test_run(self): - # Set test arguments test_chroot = "/usr/local/home/chromeos" test_build_id = "remote/lumpy/latest-dev" diff --git a/crosperf/experiment.py b/crosperf/experiment.py index 9973f7e9..7d35e319 100644 --- a/crosperf/experiment.py +++ b/crosperf/experiment.py @@ -44,6 +44,7 @@ class Experiment(object): ignore_min_max, crosfleet, dut_config, + keep_stateful: bool, no_lock: bool, ): self.name = name @@ -101,7 +102,11 @@ class Experiment(object): if test_flag.GetTestMode(): machine_manager_fn = MockMachineManager self.machine_manager = machine_manager_fn( - chromeos_root, acquire_timeout, log_level, locks_directory + chromeos_root, + acquire_timeout, + log_level, + locks_directory, + keep_stateful=keep_stateful, ) self.l = logger.GetLogger(log_dir) @@ -137,7 +142,6 @@ class Experiment(object): for label in self.labels: for benchmark in self.benchmarks: for iteration in range(1, benchmark.iterations + 1): - benchmark_run_name = "%s: %s (%s)" % ( label.name, benchmark.name, diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py index c71981ab..e89adb87 100644 --- a/crosperf/experiment_factory.py +++ b/crosperf/experiment_factory.py @@ -28,33 +28,8 @@ import config # specified sets. Here we define sets of tests that users may want # to run together. -telemetry_perfv2_tests = [ - "kraken", - "octane", -] - -telemetry_pagecycler_tests = [ - "page_cycler_v2.intl_ar_fa_he", - "page_cycler_v2.intl_es_fr_pt-BR", - "page_cycler_v2.intl_hi_ru", - "page_cycler_v2.intl_ja_zh", - "page_cycler_v2.intl_ko_th_vi", - "page_cycler_v2.typical_25", -] - -telemetry_toolchain_old_perf_tests = [ - "page_cycler_v2.intl_es_fr_pt-BR", - "page_cycler_v2.intl_hi_ru", - "page_cycler_v2.intl_ja_zh", - "page_cycler_v2.intl_ko_th_vi", - "page_cycler_v2.netsim.top_10", - "page_cycler_v2.typical_25", - "spaceport", - "tab_switching.top_10", -] telemetry_toolchain_perf_tests = [ "octane", - "kraken", "speedometer", "speedometer2", "jetstream2", @@ -68,13 +43,10 @@ graphics_perf_tests = [ ] # TODO: disable rendering.desktop by default as the benchmark is # currently in a bad state -# page_cycler_v2.typical_25 is deprecated and the recommend replacement is -# loading.desktop@@typical (crbug.com/916340) telemetry_crosbolt_perf_tests = [ "octane", - "kraken", "speedometer2", - "jetstream", + "jetstream2", "loading.desktop", # 'rendering.desktop', ] @@ -84,12 +56,6 @@ crosbolt_perf_tests = [ "tast.video.PlaybackPerfVP91080P30FPS", ] -# 'cheets_AntutuTest', -# 'cheets_PerfBootServer', -# 'cheets_CandyCrushTest', -# 'cheets_LinpackTest', -# ] - dso_list = [ "all", "chrome", @@ -211,6 +177,7 @@ class ExperimentFactory(object): "turbostat": global_settings.GetField("turbostat"), "top_interval": global_settings.GetField("top_interval"), } + keep_stateful = global_settings.GetField("keep_stateful") # Default cache hit conditions. The image checksum in the cache and the # computed checksum of the image must match. Also a cache file must exist. @@ -220,7 +187,9 @@ class ExperimentFactory(object): ] if global_settings.GetField("rerun_if_failed"): cache_conditions.append(CacheConditions.RUN_SUCCEEDED) - if global_settings.GetField("rerun"): + if global_settings.GetField("rerun") or global_settings.GetField( + "ignore_cache" + ): cache_conditions.append(CacheConditions.FALSE) if global_settings.GetField("same_machine"): cache_conditions.append(CacheConditions.SAME_MACHINE_MATCH) @@ -305,37 +274,7 @@ class ExperimentFactory(object): ) if suite == "telemetry_Crosperf": - if test_name == "all_perfv2": - self.AppendBenchmarkSet( - benchmarks, - telemetry_perfv2_tests, - test_args, - iterations, - rm_chroot_tmp, - perf_args, - suite, - show_all_results, - retries, - run_local, - cwp_dso, - weight, - ) - elif test_name == "all_pagecyclers": - self.AppendBenchmarkSet( - benchmarks, - telemetry_pagecycler_tests, - test_args, - iterations, - rm_chroot_tmp, - perf_args, - suite, - show_all_results, - retries, - run_local, - cwp_dso, - weight, - ) - elif test_name == "all_crosbolt_perf": + if test_name == "all_crosbolt_perf": self.AppendBenchmarkSet( benchmarks, telemetry_crosbolt_perf_tests, @@ -381,21 +320,6 @@ class ExperimentFactory(object): ) # Add non-telemetry toolchain-perf benchmarks: - # Tast test platform.ReportDiskUsage for image size. - benchmarks.append( - Benchmark( - "platform.ReportDiskUsage", - "platform.ReportDiskUsage", - "", - 1, # This is not a performance benchmark, only run once. - rm_chroot_tmp, - "", - "tast", # Specify the suite to be 'tast' - show_all_results, - retries, - ) - ) - # TODO: crbug.com/1057755 Do not enable graphics_WebGLAquarium until # it gets fixed. # @@ -413,21 +337,6 @@ class ExperimentFactory(object): # run_local=False, # cwp_dso=cwp_dso, # weight=weight)) - elif test_name == "all_toolchain_perf_old": - self.AppendBenchmarkSet( - benchmarks, - telemetry_toolchain_old_perf_tests, - test_args, - iterations, - rm_chroot_tmp, - perf_args, - suite, - show_all_results, - retries, - run_local, - cwp_dso, - weight, - ) else: benchmark = Benchmark( benchmark_name, @@ -613,6 +522,7 @@ class ExperimentFactory(object): ignore_min_max, crosfleet, dut_config, + keep_stateful, no_lock=no_lock, ) @@ -623,7 +533,7 @@ class ExperimentFactory(object): os.path.dirname(__file__), "default_remotes" ) try: - with open(default_remotes_file) as f: + with open(default_remotes_file, encoding="utf-8") as f: for line in f: key, v = line.split(":") if key.strip() == board: @@ -632,15 +542,15 @@ class ExperimentFactory(object): return remotes else: raise RuntimeError( - "There is no remote for {0}".format(board) + f"There is no remote for {board}" ) except IOError: # TODO: rethrow instead of throwing different exception. raise RuntimeError( - "IOError while reading file {0}".format(default_remotes_file) + f"IOError while reading file {default_remotes_file}" ) else: - raise RuntimeError("There is no remote for {0}".format(board)) + raise RuntimeError(f"There is no remote for {board}") def CheckRemotesInCrosfleet(self, remote): # TODO: (AI:zhizhouy) need to check whether a remote is a local or lab diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py index 0541bb9b..87e8c4f9 100755 --- a/crosperf/experiment_factory_unittest.py +++ b/crosperf/experiment_factory_unittest.py @@ -20,6 +20,7 @@ from cros_utils.file_utils import FileUtils import experiment_factory from experiment_factory import ExperimentFactory from experiment_file import ExperimentFile +from results_cache import CacheConditions import settings_factory import test_flag @@ -258,7 +259,7 @@ class ExperimentFactoryTest(unittest.TestCase): bench_list = [] ef.AppendBenchmarkSet( bench_list, - experiment_factory.telemetry_perfv2_tests, + experiment_factory.telemetry_crosbolt_perf_tests, "", 1, False, @@ -271,14 +272,15 @@ class ExperimentFactoryTest(unittest.TestCase): 0, ) self.assertEqual( - len(bench_list), len(experiment_factory.telemetry_perfv2_tests) + len(bench_list), + len(experiment_factory.telemetry_crosbolt_perf_tests), ) self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark)) bench_list = [] ef.AppendBenchmarkSet( bench_list, - experiment_factory.telemetry_pagecycler_tests, + experiment_factory.telemetry_toolchain_perf_tests, "", 1, False, @@ -291,7 +293,8 @@ class ExperimentFactoryTest(unittest.TestCase): 0, ) self.assertEqual( - len(bench_list), len(experiment_factory.telemetry_pagecycler_tests) + len(bench_list), + len(experiment_factory.telemetry_toolchain_perf_tests), ) self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark)) @@ -318,7 +321,6 @@ class ExperimentFactoryTest(unittest.TestCase): @mock.patch.object(socket, "gethostname") def test_get_experiment(self, mock_socket): - test_flag.SetTestMode(False) self.append_benchmark_call_args = [] @@ -418,10 +420,13 @@ class ExperimentFactoryTest(unittest.TestCase): ) self.assertEqual(exp.labels[0].autotest_path, "/tmp/autotest") self.assertEqual(exp.labels[0].board, "lumpy") + self.assertEqual(exp.machine_manager.keep_stateful, False) # Second test: Remotes listed in labels. test_flag.SetTestMode(True) label_settings.SetField("remote", "chromeos1.cros chromeos2.cros") + # Also verify keep_stateful. + global_settings.SetField("keep_stateful", "true") exp = ef.GetExperiment(mock_experiment_file, "", "") self.assertCountEqual( exp.remote, @@ -432,6 +437,9 @@ class ExperimentFactoryTest(unittest.TestCase): "chromeos2.cros", ], ) + # keep_stateful is propagated to machine_manager which flashes the + # images. + self.assertEqual(exp.machine_manager.keep_stateful, True) # Third test: Automatic fixing of bad logging_level param: global_settings.SetField("logging_level", "really loud!") @@ -450,7 +458,47 @@ class ExperimentFactoryTest(unittest.TestCase): label_settings.SetField("remote", "") global_settings.SetField("remote", "123.45.67.89") exp = ef.GetExperiment(mock_experiment_file, "", "") - self.assertEqual(exp.cache_conditions, [0, 2, 3, 4, 6, 1]) + self.assertEqual( + exp.cache_conditions, + [ + CacheConditions.CACHE_FILE_EXISTS, + CacheConditions.CHECKSUMS_MATCH, + CacheConditions.RUN_SUCCEEDED, + CacheConditions.FALSE, + CacheConditions.SAME_MACHINE_MATCH, + CacheConditions.MACHINES_MATCH, + ], + ) + + # Check the alias option to ignore cache. + global_settings.SetField("rerun", "false") + global_settings.SetField("ignore_cache", "true") + exp = ef.GetExperiment(mock_experiment_file, "", "") + self.assertEqual( + exp.cache_conditions, + [ + CacheConditions.CACHE_FILE_EXISTS, + CacheConditions.CHECKSUMS_MATCH, + CacheConditions.RUN_SUCCEEDED, + CacheConditions.FALSE, + CacheConditions.SAME_MACHINE_MATCH, + CacheConditions.MACHINES_MATCH, + ], + ) + # Check without cache use. + global_settings.SetField("rerun", "false") + global_settings.SetField("ignore_cache", "false") + exp = ef.GetExperiment(mock_experiment_file, "", "") + self.assertEqual( + exp.cache_conditions, + [ + CacheConditions.CACHE_FILE_EXISTS, + CacheConditions.CHECKSUMS_MATCH, + CacheConditions.RUN_SUCCEEDED, + CacheConditions.SAME_MACHINE_MATCH, + CacheConditions.MACHINES_MATCH, + ], + ) # Fifth Test: Adding a second label; calling GetXbuddyPath; omitting all # remotes (Call GetDefaultRemotes). diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py index 1f78dcc0..c41459a1 100644 --- a/crosperf/experiment_runner.py +++ b/crosperf/experiment_runner.py @@ -298,7 +298,6 @@ class ExperimentRunner(object): experiment_file_path = os.path.join(results_directory, "experiment.exp") FileUtils().WriteFile(experiment_file_path, experiment.experiment_file) - has_failure = False all_failed = True topstats_file = os.path.join(results_directory, "topstats.log") @@ -306,17 +305,21 @@ class ExperimentRunner(object): "Storing top statistics of each benchmark run into %s." % topstats_file ) + # Track if any iterations for a given benchmark has passed for each + # label. + benchmarks_passes = {} with open(topstats_file, "w") as top_fd: for benchmark_run in experiment.benchmark_runs: + benchmarks_passes.setdefault( + benchmark_run.label.name, + {benchmark_run.benchmark.name: False}, + ) if benchmark_run.result: - # FIXME: Pylint has a bug suggesting the following change, which - # should be fixed in pylint 2.0. Resolve this after pylint >= 2.0. - # Bug: https://github.com/PyCQA/pylint/issues/1984 - # pylint: disable=simplifiable-if-statement - if benchmark_run.result.retval: - has_failure = True - else: + if not benchmark_run.result.retval: all_failed = False + benchmarks_passes[benchmark_run.label.name][ + benchmark_run.benchmark.name + ] = True # Header with benchmark run name. top_fd.write("%s\n" % str(benchmark_run)) # Formatted string with top statistics. @@ -325,6 +328,11 @@ class ExperimentRunner(object): if all_failed: return self.ALL_FAILED + # Set has_passes if atleast one iteration of all benchmarks has passed + # for every label. + has_passes = True + for benchmarks in benchmarks_passes.values(): + has_passes = has_passes and all(benchmarks.values()) self.l.LogOutput("Storing results of each benchmark run.") for benchmark_run in experiment.benchmark_runs: @@ -339,8 +347,10 @@ class ExperimentRunner(object): benchmark_run.result.CompressResultsTo(benchmark_run_path) else: benchmark_run.result.CopyResultsTo(benchmark_run_path) + # Don't remove benchmark tmp if it was a cache hit. benchmark_run.result.CleanUp( benchmark_run.benchmark.rm_chroot_tmp + and not benchmark_run.cache_hit ) self.l.LogOutput("Storing results report in %s." % results_directory) @@ -367,7 +377,7 @@ class ExperimentRunner(object): msg_body = "<pre style='font-size: 13px'>%s</pre>" % text_report FileUtils().WriteFile(msg_file_path, msg_body) - return self.SUCCEEDED if not has_failure else self.HAS_FAILURE + return self.SUCCEEDED if has_passes else self.HAS_FAILURE def Run(self): try: diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py index ffb0b5e6..17db64f5 100644 --- a/crosperf/machine_manager.py +++ b/crosperf/machine_manager.py @@ -221,6 +221,7 @@ class MachineManager(object): locks_dir, cmd_exec=None, lgr=None, + keep_stateful: bool = False, ): self._lock = threading.RLock() self._all_machines = [] @@ -233,6 +234,7 @@ class MachineManager(object): self.acquire_timeout = acquire_timeout self.log_level = log_level self.locks_dir = locks_dir + self.keep_stateful = keep_stateful self.ce = cmd_exec or command_executer.GetCommandExecuter( log_level=self.log_level ) @@ -282,14 +284,16 @@ class MachineManager(object): image_chromeos_args = [ image_chromeos.__file__, "--no_lock", - "--chromeos_root=%s" % chromeos_root, - "--image=%s" % label.chromeos_image, - "--image_args=%s" % label.image_args, - "--remote=%s" % machine.name, - "--logging_level=%s" % self.log_level, + f"--chromeos_root={chromeos_root}", + f"--image={label.chromeos_image}", + f"--image_args={label.image_args}", + f"--remote={machine.name}", + f"--logging_level={self.log_level}", ] if label.board: - image_chromeos_args.append("--board=%s" % label.board) + image_chromeos_args.append(f"--board={label.board}") + if self.keep_stateful: + image_chromeos_args.append("--keep_stateful") # Currently can't image two machines at once. # So have to serialized on this lock. @@ -729,9 +733,20 @@ power management: class MockMachineManager(MachineManager): """Mock machine manager class.""" - def __init__(self, chromeos_root, acquire_timeout, log_level, locks_dir): + def __init__( + self, + chromeos_root, + acquire_timeout, + log_level, + locks_dir, + keep_stateful: bool = False, + ): super(MockMachineManager, self).__init__( - chromeos_root, acquire_timeout, log_level, locks_dir + chromeos_root, + acquire_timeout, + log_level, + locks_dir, + keep_stateful=keep_stateful, ) def _TryToLockMachine(self, cros_machine): diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py index 043da990..b08fde48 100644 --- a/crosperf/results_cache.py +++ b/crosperf/results_cache.py @@ -185,13 +185,16 @@ class Result(object): # Otherwise get the base filename and create the correct # path for it. _, f_base = misc.GetRoot(f) - data_filename = os.path.join( - self.chromeos_root, "chroot/tmp", self.temp_dir, f_base + data_filename = misc.GetOutsideChrootPath( + self.chromeos_root, + os.path.join("/tmp", self.temp_dir, f_base), ) if data_filename.find(".json") > 0: raw_dict = dict() if os.path.exists(data_filename): - with open(data_filename, "r") as data_file: + with open( + data_filename, "r", encoding="utf-8" + ) as data_file: raw_dict = json.load(data_file) if "charts" in raw_dict: @@ -220,7 +223,9 @@ class Result(object): units_dict[key] = result_dict["units"] else: if os.path.exists(data_filename): - with open(data_filename, "r") as data_file: + with open( + data_filename, "r", encoding="utf-8" + ) as data_file: lines = data_file.readlines() for line in lines: tmp_dict = json.loads(line) @@ -258,22 +263,24 @@ class Result(object): return results_dict def GetKeyvals(self): - results_in_chroot = os.path.join(self.chromeos_root, "chroot", "tmp") + results_in_chroot = misc.GetOutsideChrootPath( + self.chromeos_root, "/tmp" + ) if not self.temp_dir: self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot) command = f"cp -r {self.results_dir}/* {self.temp_dir}" self.ce.RunCommand(command, print_to_console=False) + tmp_dir_in_chroot = misc.GetInsideChrootPath( + self.chromeos_root, self.temp_dir + ) command = "./generate_test_report --no-color --csv %s" % ( - os.path.join("/tmp", os.path.basename(self.temp_dir)) + tmp_dir_in_chroot ) _, out, _ = self.ce.ChrootRunCommandWOutput( self.chromeos_root, command, print_to_console=False ) keyvals_dict = {} - tmp_dir_in_chroot = misc.GetInsideChrootPath( - self.chromeos_root, self.temp_dir - ) for line in out.splitlines(): tokens = re.split("=|,", line) key = tokens[-2] @@ -297,8 +304,8 @@ class Result(object): chroot_perf_data_file = misc.GetInsideChrootPath( self.chromeos_root, perf_data_file ) - perf_path = os.path.join( - self.chromeos_root, "chroot", "usr/bin/perf" + perf_path = misc.GetOutsideChrootPath( + self.chromeos_root, "/usr/bin/perf" ) perf_file = "/usr/sbin/perf" if os.path.exists(perf_path): @@ -308,20 +315,19 @@ class Result(object): # We specify exact match for known DSO type, and every sample for `all`. exact_match = "" if self.cwp_dso == "all": - exact_match = '""' + exact_match = "" elif self.cwp_dso == "chrome": - exact_match = '" chrome "' + exact_match = "chrome" elif self.cwp_dso == "kallsyms": - exact_match = '"[kernel.kallsyms]"' + exact_match = "[kernel.kallsyms]" else: # This will need to be updated once there are more DSO types supported, # if user want an exact match for the field they want. - exact_match = '"%s"' % self.cwp_dso + exact_match = self.cwp_dso - command = "%s report -n -s dso -i %s 2> /dev/null | grep %s" % ( - perf_file, - chroot_perf_data_file, - exact_match, + command = ( + f"{perf_file} report -n -s dso -i " + f"{chroot_perf_data_file} 2> /dev/null" ) _, result, _ = self.ce.ChrootRunCommandWOutput( self.chromeos_root, command @@ -335,6 +341,8 @@ class Result(object): for line in result.split("\n"): attr = line.split() if len(attr) == 3 and "%" in attr[0]: + if exact_match and exact_match != attr[2]: + continue samples += int(attr[1]) except: raise RuntimeError("Cannot parse perf dso result") @@ -357,11 +365,12 @@ class Result(object): "default_idle", "cpu_idle_loop", "do_idle", + "cpuidle_enter_state", ), } idle_samples = 0 - with open(perf_report_file) as f: + with open(perf_report_file, encoding="utf-8") as f: try: for line in f: line = line.strip() @@ -462,10 +471,7 @@ class Result(object): return self.FindFilesInResultsDir("-name wait_time.log").split("\n")[0] def _CheckDebugPath(self, option, path): - relative_path = path[1:] - out_chroot_path = os.path.join( - self.chromeos_root, "chroot", relative_path - ) + out_chroot_path = misc.GetOutsideChrootPath(self.chromeos_root, path) if os.path.exists(out_chroot_path): if option == "kallsyms": path = os.path.join(path, "System.map-*") @@ -493,8 +499,8 @@ class Result(object): chroot_perf_report_file = misc.GetInsideChrootPath( self.chromeos_root, perf_report_file ) - perf_path = os.path.join( - self.chromeos_root, "chroot", "usr/bin/perf" + perf_path = misc.GetOutsideChrootPath( + self.chromeos_root, "/usr/bin/perf" ) perf_file = "/usr/sbin/perf" @@ -562,7 +568,7 @@ class Result(object): def GatherPerfResults(self): report_id = 0 for perf_report_file in self.perf_report_files: - with open(perf_report_file, "r") as f: + with open(perf_report_file, "r", encoding="utf-8") as f: report_contents = f.read() for group in re.findall( r"Events: (\S+) (\S+)", report_contents @@ -612,7 +618,7 @@ class Result(object): raise IOError("%s does not exist" % filename) keyvals = {} - with open(filename, "r") as f: + with open(filename, "r", encoding="utf-8") as f: raw_dict = json.load(f) if "charts" in raw_dict: raw_dict = raw_dict["charts"] @@ -660,7 +666,7 @@ class Result(object): """ cpustats = {} read_data = "" - with open(self.turbostat_log_file) as f: + with open(self.turbostat_log_file, encoding="utf-8") as f: read_data = f.readlines() if not read_data: @@ -732,7 +738,7 @@ class Result(object): 121 root 20 0 0 0 0 S 1.0 0.0 0:00.45 spi5 """ all_data = "" - with open(self.top_log_file) as f: + with open(self.top_log_file, encoding="utf-8") as f: all_data = f.read() if not all_data: @@ -874,7 +880,7 @@ class Result(object): cpustats = {} read_data = "" - with open(self.cpustats_log_file) as f: + with open(self.cpustats_log_file, encoding="utf-8") as f: read_data = f.readlines() if not read_data: @@ -935,7 +941,7 @@ class Result(object): raise IOError("%s does not exist" % filename) keyvals = {} - with open(filename) as f: + with open(filename, encoding="utf-8") as f: histograms = json.load(f) value_map = {} # Gets generic set values. @@ -1129,7 +1135,7 @@ class Result(object): if self.perf_data_files and self.top_cmds: self.VerifyPerfDataPID() if self.wait_time_log_file: - with open(self.wait_time_log_file) as f: + with open(self.wait_time_log_file, encoding="utf-8") as f: wait_time = f.readline().strip() try: wait_time = float(wait_time) @@ -1162,7 +1168,7 @@ class Result(object): chrome_version = "" keys_file = os.path.join(cache_dir, CACHE_KEYS_FILE) if os.path.exists(keys_file): - with open(keys_file, "r") as f: + with open(keys_file, "r", encoding="utf-8") as f: lines = f.readlines() for l in lines: if l.startswith("Google Chrome "): @@ -1184,7 +1190,7 @@ class Result(object): # Untar the tarball to a temporary directory self.temp_dir = tempfile.mkdtemp( - dir=os.path.join(self.chromeos_root, "chroot", "tmp") + dir=misc.GetOutsideChrootPath(self.chromeos_root, "/tmp") ) command = "cd %s && tar xf %s" % ( @@ -1202,7 +1208,11 @@ class Result(object): self.ProcessResults(use_cache=True) def CleanUp(self, rm_chroot_tmp): - if rm_chroot_tmp and self.results_dir: + if ( + rm_chroot_tmp + and self.results_dir + and self.results_dir != self.temp_dir + ): dirname, basename = misc.GetRoot(self.results_dir) if basename.find("test_that_results_") != -1: command = "rm -rf %s" % self.results_dir @@ -1240,7 +1250,9 @@ class Result(object): pickle.dump(self.retval, f) if not test_flag.GetTestMode(): - with open(os.path.join(temp_dir, CACHE_KEYS_FILE), "w") as f: + with open( + os.path.join(temp_dir, CACHE_KEYS_FILE), "w", encoding="utf-8" + ) as f: f.write("%s\n" % self.label.name) f.write("%s\n" % self.label.chrome_version) f.write("%s\n" % self.machine.checksum_string) @@ -1255,7 +1267,9 @@ class Result(object): # Store machine info. # TODO(asharif): Make machine_manager a singleton, and don't pass it into # this function. - with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f: + with open( + os.path.join(temp_dir, MACHINE_FILE), "w", encoding="utf-8" + ) as f: f.write(machine_manager.machine_checksum_string[self.label.name]) if os.path.exists(cache_dir): diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py index cad149e0..06a8b94d 100755 --- a/crosperf/results_cache_unittest.py +++ b/crosperf/results_cache_unittest.py @@ -11,6 +11,7 @@ import io import os import pickle +import re import shutil import tempfile import unittest @@ -449,6 +450,16 @@ class MockResult(Result): return keyvals +class RegexMatcher: + """A regex matcher, for passing to mocks.""" + + def __init__(self, regex): + self._regex = re.compile(regex) + + def __eq__(self, string): + return self._regex.search(string) is not None + + class ResultTest(unittest.TestCase): """Result test class.""" @@ -490,7 +501,9 @@ class ResultTest(unittest.TestCase): None, ) - def testCreateFromRun(self): + @mock.patch.object(os.path, "exists") + def testCreateFromRun(self, mock_path_exists): + mock_path_exists.side_effect = lambda x: x != "/etc/cros_chroot_version" result = MockResult.CreateFromRun( logger.GetLogger(), "average", @@ -508,7 +521,7 @@ class ResultTest(unittest.TestCase): ) self.assertEqual( result.results_dir, - "/tmp/chroot/tmp/test_that.PO1234567/platform_LibCBench", + RegexMatcher("/tmp/.*tmp/test_that.PO1234567/platform_LibCBench"), ) self.assertEqual(result.retval, 0) @@ -517,12 +530,12 @@ class ResultTest(unittest.TestCase): self.mock_logger, self.mock_label, "average", self.mock_cmd_exec ) self.result.chromeos_root = "/tmp/chromeos" + self.orig_exists = os.path.exists @mock.patch.object(os.path, "isdir") @mock.patch.object(command_executer.CommandExecuter, "RunCommand") @mock.patch.object(command_executer.CommandExecuter, "CopyFiles") def test_copy_files_to(self, mock_copyfiles, mock_runcmd, mock_isdir): - files = ["src_file_1", "src_file_2", "src_file_3"] dest_dir = "/tmp/test" self.mock_cmd_exec.RunCommand = mock_runcmd @@ -614,118 +627,118 @@ class ResultTest(unittest.TestCase): self.assertEqual( kv_dict2, { - u"Box2D__Box2D": 4775, - u"Mandreel__Mandreel": 6620, - u"Gameboy__Gameboy": 9901, - u"Crypto__Crypto": 8737, - u"telemetry_page_measurement_results__num_errored": 0, - u"telemetry_page_measurement_results__num_failed": 0, - u"PdfJS__PdfJS": 6455, - u"Total__Score": 7918, - u"EarleyBoyer__EarleyBoyer": 14340, - u"MandreelLatency__MandreelLatency": 5188, - u"CodeLoad__CodeLoad": 6271, - u"DeltaBlue__DeltaBlue": 14401, - u"Typescript__Typescript": 9815, - u"SplayLatency__SplayLatency": 7653, - u"zlib__zlib": 16094, - u"Richards__Richards": 10358, - u"RegExp__RegExp": 1765, - u"NavierStokes__NavierStokes": 9815, - u"Splay__Splay": 4425, - u"RayTrace__RayTrace": 16600, + "Box2D__Box2D": 4775, + "Mandreel__Mandreel": 6620, + "Gameboy__Gameboy": 9901, + "Crypto__Crypto": 8737, + "telemetry_page_measurement_results__num_errored": 0, + "telemetry_page_measurement_results__num_failed": 0, + "PdfJS__PdfJS": 6455, + "Total__Score": 7918, + "EarleyBoyer__EarleyBoyer": 14340, + "MandreelLatency__MandreelLatency": 5188, + "CodeLoad__CodeLoad": 6271, + "DeltaBlue__DeltaBlue": 14401, + "Typescript__Typescript": 9815, + "SplayLatency__SplayLatency": 7653, + "zlib__zlib": 16094, + "Richards__Richards": 10358, + "RegExp__RegExp": 1765, + "NavierStokes__NavierStokes": 9815, + "Splay__Splay": 4425, + "RayTrace__RayTrace": 16600, }, ) self.assertEqual( udict, { - u"Box2D__Box2D": u"score", - u"Mandreel__Mandreel": u"score", - u"Gameboy__Gameboy": u"score", - u"Crypto__Crypto": u"score", - u"telemetry_page_measurement_results__num_errored": u"count", - u"telemetry_page_measurement_results__num_failed": u"count", - u"PdfJS__PdfJS": u"score", - u"Total__Score": u"score", - u"EarleyBoyer__EarleyBoyer": u"score", - u"MandreelLatency__MandreelLatency": u"score", - u"CodeLoad__CodeLoad": u"score", - u"DeltaBlue__DeltaBlue": u"score", - u"Typescript__Typescript": u"score", - u"SplayLatency__SplayLatency": u"score", - u"zlib__zlib": u"score", - u"Richards__Richards": u"score", - u"RegExp__RegExp": u"score", - u"NavierStokes__NavierStokes": u"score", - u"Splay__Splay": u"score", - u"RayTrace__RayTrace": u"score", + "Box2D__Box2D": "score", + "Mandreel__Mandreel": "score", + "Gameboy__Gameboy": "score", + "Crypto__Crypto": "score", + "telemetry_page_measurement_results__num_errored": "count", + "telemetry_page_measurement_results__num_failed": "count", + "PdfJS__PdfJS": "score", + "Total__Score": "score", + "EarleyBoyer__EarleyBoyer": "score", + "MandreelLatency__MandreelLatency": "score", + "CodeLoad__CodeLoad": "score", + "DeltaBlue__DeltaBlue": "score", + "Typescript__Typescript": "score", + "SplayLatency__SplayLatency": "score", + "zlib__zlib": "score", + "Richards__Richards": "score", + "RegExp__RegExp": "score", + "NavierStokes__NavierStokes": "score", + "Splay__Splay": "score", + "RayTrace__RayTrace": "score", }, ) def test_append_telemetry_units(self): kv_dict = { - u"Box2D__Box2D": 4775, - u"Mandreel__Mandreel": 6620, - u"Gameboy__Gameboy": 9901, - u"Crypto__Crypto": 8737, - u"PdfJS__PdfJS": 6455, - u"Total__Score": 7918, - u"EarleyBoyer__EarleyBoyer": 14340, - u"MandreelLatency__MandreelLatency": 5188, - u"CodeLoad__CodeLoad": 6271, - u"DeltaBlue__DeltaBlue": 14401, - u"Typescript__Typescript": 9815, - u"SplayLatency__SplayLatency": 7653, - u"zlib__zlib": 16094, - u"Richards__Richards": 10358, - u"RegExp__RegExp": 1765, - u"NavierStokes__NavierStokes": 9815, - u"Splay__Splay": 4425, - u"RayTrace__RayTrace": 16600, + "Box2D__Box2D": 4775, + "Mandreel__Mandreel": 6620, + "Gameboy__Gameboy": 9901, + "Crypto__Crypto": 8737, + "PdfJS__PdfJS": 6455, + "Total__Score": 7918, + "EarleyBoyer__EarleyBoyer": 14340, + "MandreelLatency__MandreelLatency": 5188, + "CodeLoad__CodeLoad": 6271, + "DeltaBlue__DeltaBlue": 14401, + "Typescript__Typescript": 9815, + "SplayLatency__SplayLatency": 7653, + "zlib__zlib": 16094, + "Richards__Richards": 10358, + "RegExp__RegExp": 1765, + "NavierStokes__NavierStokes": 9815, + "Splay__Splay": 4425, + "RayTrace__RayTrace": 16600, } units_dict = { - u"Box2D__Box2D": u"score", - u"Mandreel__Mandreel": u"score", - u"Gameboy__Gameboy": u"score", - u"Crypto__Crypto": u"score", - u"PdfJS__PdfJS": u"score", - u"Total__Score": u"score", - u"EarleyBoyer__EarleyBoyer": u"score", - u"MandreelLatency__MandreelLatency": u"score", - u"CodeLoad__CodeLoad": u"score", - u"DeltaBlue__DeltaBlue": u"score", - u"Typescript__Typescript": u"score", - u"SplayLatency__SplayLatency": u"score", - u"zlib__zlib": u"score", - u"Richards__Richards": u"score", - u"RegExp__RegExp": u"score", - u"NavierStokes__NavierStokes": u"score", - u"Splay__Splay": u"score", - u"RayTrace__RayTrace": u"score", + "Box2D__Box2D": "score", + "Mandreel__Mandreel": "score", + "Gameboy__Gameboy": "score", + "Crypto__Crypto": "score", + "PdfJS__PdfJS": "score", + "Total__Score": "score", + "EarleyBoyer__EarleyBoyer": "score", + "MandreelLatency__MandreelLatency": "score", + "CodeLoad__CodeLoad": "score", + "DeltaBlue__DeltaBlue": "score", + "Typescript__Typescript": "score", + "SplayLatency__SplayLatency": "score", + "zlib__zlib": "score", + "Richards__Richards": "score", + "RegExp__RegExp": "score", + "NavierStokes__NavierStokes": "score", + "Splay__Splay": "score", + "RayTrace__RayTrace": "score", } results_dict = self.result.AppendTelemetryUnits(kv_dict, units_dict) self.assertEqual( results_dict, { - u"Box2D__Box2D": [4775, u"score"], - u"Splay__Splay": [4425, u"score"], - u"Gameboy__Gameboy": [9901, u"score"], - u"Crypto__Crypto": [8737, u"score"], - u"PdfJS__PdfJS": [6455, u"score"], - u"Total__Score": [7918, u"score"], - u"EarleyBoyer__EarleyBoyer": [14340, u"score"], - u"MandreelLatency__MandreelLatency": [5188, u"score"], - u"DeltaBlue__DeltaBlue": [14401, u"score"], - u"SplayLatency__SplayLatency": [7653, u"score"], - u"Mandreel__Mandreel": [6620, u"score"], - u"Richards__Richards": [10358, u"score"], - u"zlib__zlib": [16094, u"score"], - u"CodeLoad__CodeLoad": [6271, u"score"], - u"Typescript__Typescript": [9815, u"score"], - u"RegExp__RegExp": [1765, u"score"], - u"RayTrace__RayTrace": [16600, u"score"], - u"NavierStokes__NavierStokes": [9815, u"score"], + "Box2D__Box2D": [4775, "score"], + "Splay__Splay": [4425, "score"], + "Gameboy__Gameboy": [9901, "score"], + "Crypto__Crypto": [8737, "score"], + "PdfJS__PdfJS": [6455, "score"], + "Total__Score": [7918, "score"], + "EarleyBoyer__EarleyBoyer": [14340, "score"], + "MandreelLatency__MandreelLatency": [5188, "score"], + "DeltaBlue__DeltaBlue": [14401, "score"], + "SplayLatency__SplayLatency": [7653, "score"], + "Mandreel__Mandreel": [6620, "score"], + "Richards__Richards": [10358, "score"], + "zlib__zlib": [16094, "score"], + "CodeLoad__CodeLoad": [6271, "score"], + "Typescript__Typescript": [9815, "score"], + "RegExp__RegExp": [1765, "score"], + "RayTrace__RayTrace": [16600, "score"], + "NavierStokes__NavierStokes": [9815, "score"], }, ) @@ -738,7 +751,6 @@ class ResultTest(unittest.TestCase): def test_get_keyvals( self, mock_chrootruncmd, mock_runcmd, mock_mkdtemp, mock_getpath ): - self.kv_dict = {} self.callGetNewKeyvals = False @@ -821,9 +833,9 @@ class ResultTest(unittest.TestCase): # Test 3. suite != telemetry_Crosperf. Normally this would be for # running non-Telemetry autotests, such as BootPerfServer. In this test - # case, the keyvals we have set up were returned from a Telemetry test run; - # so this pass is basically testing that we don't append the units to the - # test results (which we do for Telemetry autotest runs). + # case, the keyvals we have set up were returned from a Telemetry test + # run; so this pass is basically testing that we don't append the units + # to the test results (which we do for Telemetry autotest runs). reset() self.result.suite = "" res = self.result.GetKeyvals() @@ -839,10 +851,15 @@ class ResultTest(unittest.TestCase): ): self.result.perf_data_files = ["/tmp/results/perf.data"] self.result.board = "samus" + self.result.cwp_dso = "kallsyms" mock_getpath.return_value = "/usr/chromeos/chroot/tmp/results/perf.data" mock_get_total_samples.return_value = [ "", - "45.42% 237210 chrome ", + ( + "45.42% 53721 chrome \n" + "10.01% 12345 [kernel.kallsyms] \n" + "1.42% 1234 ssh " + ), "", ] mock_exists.return_value = True @@ -855,10 +872,9 @@ class ResultTest(unittest.TestCase): with mock.patch("builtins.open", return_value=io.StringIO(content)): samples = self.result.GetSamples() - self.assertEqual(samples, [237210 - 60, u"samples"]) + self.assertEqual(samples, [12345 - 60, "samples"]) def test_get_results_dir(self): - self.result.out = "" self.assertRaises(Exception, self.result.GetResultsDir) @@ -868,7 +884,6 @@ class ResultTest(unittest.TestCase): @mock.patch.object(command_executer.CommandExecuter, "RunCommandGeneric") def test_find_files_in_results_dir(self, mock_runcmd): - self.result.results_dir = None res = self.result.FindFilesInResultsDir("-name perf.data") self.assertEqual(res, "") @@ -1149,11 +1164,12 @@ class ResultTest(unittest.TestCase): """Normal case when log exists and contains valid data.""" self.result.turbostat_log_file = "/tmp/somelogfile.log" with mock.patch( - "builtins.open", mock.mock_open(read_data=TURBOSTAT_LOG_OUTPUT) + "builtins.open", + mock.mock_open(read_data=TURBOSTAT_LOG_OUTPUT), ) as mo: cpustats = self.result.ProcessTurbostatResults() # Check that the log got opened and data were read/parsed. - calls = [mock.call("/tmp/somelogfile.log")] + calls = [mock.call("/tmp/somelogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(cpustats, TURBOSTAT_DATA) @@ -1162,9 +1178,9 @@ class ResultTest(unittest.TestCase): self.result.turbostat_log_file = "/tmp/emptylogfile.log" with mock.patch("builtins.open", mock.mock_open(read_data="")) as mo: cpustats = self.result.ProcessTurbostatResults() - # Check that the log got opened and parsed successfully and empty data - # returned. - calls = [mock.call("/tmp/emptylogfile.log")] + # Check that the log got opened and parsed successfully and empty + # data returned. + calls = [mock.call("/tmp/emptylogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(cpustats, {}) @@ -1189,11 +1205,12 @@ class ResultTest(unittest.TestCase): """ self.result.cpustats_log_file = "/tmp/somelogfile.log" with mock.patch( - "builtins.open", mock.mock_open(read_data=CPUSTATS_UNIQ_OUTPUT) + "builtins.open", + mock.mock_open(read_data=CPUSTATS_UNIQ_OUTPUT), ) as mo: cpustats = self.result.ProcessCpustatsResults() # Check that the log got opened and data were read/parsed. - calls = [mock.call("/tmp/somelogfile.log")] + calls = [mock.call("/tmp/somelogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(cpustats, CPUSTATS_UNIQ_DATA) @@ -1207,11 +1224,12 @@ class ResultTest(unittest.TestCase): """ self.result.cpustats_log_file = "/tmp/somelogfile.log" with mock.patch( - "builtins.open", mock.mock_open(read_data=CPUSTATS_DUPL_OUTPUT) + "builtins.open", + mock.mock_open(read_data=CPUSTATS_DUPL_OUTPUT), ) as mo: cpustats = self.result.ProcessCpustatsResults() # Check that the log got opened and data were read/parsed. - calls = [mock.call("/tmp/somelogfile.log")] + calls = [mock.call("/tmp/somelogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(cpustats, CPUSTATS_DUPL_DATA) @@ -1220,9 +1238,9 @@ class ResultTest(unittest.TestCase): self.result.cpustats_log_file = "/tmp/emptylogfile.log" with mock.patch("builtins.open", mock.mock_open(read_data="")) as mo: cpustats = self.result.ProcessCpustatsResults() - # Check that the log got opened and parsed successfully and empty data - # returned. - calls = [mock.call("/tmp/emptylogfile.log")] + # Check that the log got opened and parsed successfully and empty + # data returned. + calls = [mock.call("/tmp/emptylogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(cpustats, {}) @@ -1235,7 +1253,7 @@ class ResultTest(unittest.TestCase): ) as mo: topproc = self.result.ProcessTopResults() # Check that the log got opened and data were read/parsed. - calls = [mock.call("/tmp/fakelogfile.log")] + calls = [mock.call("/tmp/fakelogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(topproc, TOP_DATA) @@ -1244,9 +1262,9 @@ class ResultTest(unittest.TestCase): self.result.top_log_file = "/tmp/emptylogfile.log" with mock.patch("builtins.open", mock.mock_open(read_data="")) as mo: topcalls = self.result.ProcessTopResults() - # Check that the log got opened and parsed successfully and empty data - # returned. - calls = [mock.call("/tmp/emptylogfile.log")] + # Check that the log got opened and parsed successfully and empty + # data returned. + calls = [mock.call("/tmp/emptylogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(topcalls, []) @@ -1342,10 +1360,18 @@ class ResultTest(unittest.TestCase): ), ) + @mock.patch.object(os.path, "exists") @mock.patch.object(misc, "GetInsideChrootPath") @mock.patch.object(command_executer.CommandExecuter, "ChrootRunCommand") - def test_generate_perf_report_files(self, mock_chrootruncmd, mock_getpath): - fake_file = "/usr/chromeos/chroot/tmp/results/fake_file" + def test_generate_perf_report_files( + self, mock_chrootruncmd, mock_getpath, mock_pathexists + ): + mock_pathexists.side_effect = ( + lambda x: self.orig_exists(x) + if x != "/etc/cros_chroot_version" + else False + ) + fake_file = "/tmp/results/perf.data.report" self.result.perf_data_files = ["/tmp/results/perf.data"] self.result.board = "lumpy" mock_getpath.return_value = fake_file @@ -1354,7 +1380,8 @@ class ResultTest(unittest.TestCase): # Debug path not found self.result.label.debug_path = "" tmp = self.result.GeneratePerfReportFiles() - self.assertEqual(tmp, ["/tmp/chromeos/chroot%s" % fake_file]) + self.assertEqual(len(tmp), 1) + self.assertEqual(tmp[0], RegexMatcher("/tmp/chromeos.*%s" % fake_file)) self.assertEqual( mock_chrootruncmd.call_args_list[0][0], ( @@ -1364,12 +1391,18 @@ class ResultTest(unittest.TestCase): ), ) + @mock.patch.object(os.path, "exists") @mock.patch.object(misc, "GetInsideChrootPath") @mock.patch.object(command_executer.CommandExecuter, "ChrootRunCommand") def test_generate_perf_report_files_debug( - self, mock_chrootruncmd, mock_getpath + self, mock_chrootruncmd, mock_getpath, mock_pathexists ): - fake_file = "/usr/chromeos/chroot/tmp/results/fake_file" + mock_pathexists.side_effect = ( + lambda x: self.orig_exists(x) + if x != "/etc/cros_chroot_version" + else False + ) + fake_file = "/tmp/results/perf.data.report" self.result.perf_data_files = ["/tmp/results/perf.data"] self.result.board = "lumpy" mock_getpath.return_value = fake_file @@ -1378,7 +1411,8 @@ class ResultTest(unittest.TestCase): # Debug path found self.result.label.debug_path = "/tmp/debug" tmp = self.result.GeneratePerfReportFiles() - self.assertEqual(tmp, ["/tmp/chromeos/chroot%s" % fake_file]) + self.assertEqual(len(tmp), 1) + self.assertEqual(tmp[0], RegexMatcher("/tmp/chromeos.*%s" % fake_file)) self.assertEqual( mock_chrootruncmd.call_args_list[0][0], ( @@ -1521,27 +1555,27 @@ class ResultTest(unittest.TestCase): # format self.result.suite = "telemetry_Crosperf" self.result.results_file = [tempfile.mkdtemp() + "/histograms.json"] - with open(self.result.results_file[0], "w") as f: + with open(self.result.results_file[0], "w", encoding="utf-8") as f: f.write(HISTOGRAMSET) self.result.ProcessResults() shutil.rmtree(os.path.dirname(self.result.results_file[0])) # Verify the summary for the story is correct self.assertEqual( self.result.keyvals["timeToFirstContentfulPaint__typical"], - [880.000, u"ms_smallerIsBetter"], + [880.000, "ms_smallerIsBetter"], ) # Veirfy the summary for a certain stroy tag is correct self.assertEqual( self.result.keyvals[ "timeToFirstContentfulPaint__cache_temperature:cold" ], - [1000.000, u"ms_smallerIsBetter"], + [1000.000, "ms_smallerIsBetter"], ) self.assertEqual( self.result.keyvals[ "timeToFirstContentfulPaint__cache_temperature:warm" ], - [800.000, u"ms_smallerIsBetter"], + [800.000, "ms_smallerIsBetter"], ) @mock.patch.object(Result, "ProcessCpustatsResults") @@ -1664,7 +1698,6 @@ class ResultTest(unittest.TestCase): command_executer.CommandExecuter, "ChrootRunCommandWOutput" ) def test_populate_from_cache_dir(self, mock_runchrootcmd, mock_getpath): - # pylint: disable=redefined-builtin def FakeMkdtemp(dir=None): if dir: @@ -1672,7 +1705,7 @@ class ResultTest(unittest.TestCase): return self.tmpdir def FakeGetSamples(): - return [1, u"samples"] + return [1, "samples"] current_path = os.getcwd() cache_dir = os.path.join(current_path, "test_cache/test_input") @@ -1696,46 +1729,46 @@ class ResultTest(unittest.TestCase): self.assertEqual( self.result.keyvals, { - u"Total__Total": [444.0, u"ms"], - u"regexp-dna__regexp-dna": [16.2, u"ms"], - u"telemetry_page_measurement_results__num_failed": [ + "Total__Total": [444.0, "ms"], + "regexp-dna__regexp-dna": [16.2, "ms"], + "telemetry_page_measurement_results__num_failed": [ 0, - u"count", + "count", ], - u"telemetry_page_measurement_results__num_errored": [ + "telemetry_page_measurement_results__num_errored": [ 0, - u"count", + "count", ], - u"string-fasta__string-fasta": [23.2, u"ms"], - u"crypto-sha1__crypto-sha1": [11.6, u"ms"], - u"bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte": [ + "string-fasta__string-fasta": [23.2, "ms"], + "crypto-sha1__crypto-sha1": [11.6, "ms"], + "bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte": [ 3.2, - u"ms", + "ms", ], - u"access-nsieve__access-nsieve": [7.9, u"ms"], - u"bitops-nsieve-bits__bitops-nsieve-bits": [9.4, u"ms"], - u"string-validate-input__string-validate-input": [19.3, u"ms"], - u"3d-raytrace__3d-raytrace": [24.7, u"ms"], - u"3d-cube__3d-cube": [28.0, u"ms"], - u"string-unpack-code__string-unpack-code": [46.7, u"ms"], - u"date-format-tofte__date-format-tofte": [26.3, u"ms"], - u"math-partial-sums__math-partial-sums": [22.0, u"ms"], + "access-nsieve__access-nsieve": [7.9, "ms"], + "bitops-nsieve-bits__bitops-nsieve-bits": [9.4, "ms"], + "string-validate-input__string-validate-input": [19.3, "ms"], + "3d-raytrace__3d-raytrace": [24.7, "ms"], + "3d-cube__3d-cube": [28.0, "ms"], + "string-unpack-code__string-unpack-code": [46.7, "ms"], + "date-format-tofte__date-format-tofte": [26.3, "ms"], + "math-partial-sums__math-partial-sums": [22.0, "ms"], "\telemetry_Crosperf": ["PASS", ""], - u"crypto-aes__crypto-aes": [15.2, u"ms"], - u"bitops-bitwise-and__bitops-bitwise-and": [8.4, u"ms"], - u"crypto-md5__crypto-md5": [10.5, u"ms"], - u"string-tagcloud__string-tagcloud": [52.8, u"ms"], - u"access-nbody__access-nbody": [8.5, u"ms"], + "crypto-aes__crypto-aes": [15.2, "ms"], + "bitops-bitwise-and__bitops-bitwise-and": [8.4, "ms"], + "crypto-md5__crypto-md5": [10.5, "ms"], + "string-tagcloud__string-tagcloud": [52.8, "ms"], + "access-nbody__access-nbody": [8.5, "ms"], "retval": 0, - u"math-spectral-norm__math-spectral-norm": [6.6, u"ms"], - u"math-cordic__math-cordic": [8.7, u"ms"], - u"access-binary-trees__access-binary-trees": [4.5, u"ms"], - u"controlflow-recursive__controlflow-recursive": [4.4, u"ms"], - u"access-fannkuch__access-fannkuch": [17.8, u"ms"], - u"string-base64__string-base64": [16.0, u"ms"], - u"date-format-xparb__date-format-xparb": [20.9, u"ms"], - u"3d-morph__3d-morph": [22.1, u"ms"], - u"bitops-bits-in-byte__bitops-bits-in-byte": [9.1, u"ms"], + "math-spectral-norm__math-spectral-norm": [6.6, "ms"], + "math-cordic__math-cordic": [8.7, "ms"], + "access-binary-trees__access-binary-trees": [4.5, "ms"], + "controlflow-recursive__controlflow-recursive": [4.4, "ms"], + "access-fannkuch__access-fannkuch": [17.8, "ms"], + "string-base64__string-base64": [16.0, "ms"], + "date-format-xparb__date-format-xparb": [20.9, "ms"], + "3d-morph__3d-morph": [22.1, "ms"], + "bitops-bits-in-byte__bitops-bits-in-byte": [9.1, "ms"], }, ) @@ -1746,47 +1779,47 @@ class ResultTest(unittest.TestCase): self.assertEqual( self.result.keyvals, { - u"Total__Total": [444.0, u"ms"], - u"regexp-dna__regexp-dna": [16.2, u"ms"], - u"telemetry_page_measurement_results__num_failed": [ + "Total__Total": [444.0, "ms"], + "regexp-dna__regexp-dna": [16.2, "ms"], + "telemetry_page_measurement_results__num_failed": [ 0, - u"count", + "count", ], - u"telemetry_page_measurement_results__num_errored": [ + "telemetry_page_measurement_results__num_errored": [ 0, - u"count", + "count", ], - u"string-fasta__string-fasta": [23.2, u"ms"], - u"crypto-sha1__crypto-sha1": [11.6, u"ms"], - u"bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte": [ + "string-fasta__string-fasta": [23.2, "ms"], + "crypto-sha1__crypto-sha1": [11.6, "ms"], + "bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte": [ 3.2, - u"ms", + "ms", ], - u"access-nsieve__access-nsieve": [7.9, u"ms"], - u"bitops-nsieve-bits__bitops-nsieve-bits": [9.4, u"ms"], - u"string-validate-input__string-validate-input": [19.3, u"ms"], - u"3d-raytrace__3d-raytrace": [24.7, u"ms"], - u"3d-cube__3d-cube": [28.0, u"ms"], - u"string-unpack-code__string-unpack-code": [46.7, u"ms"], - u"date-format-tofte__date-format-tofte": [26.3, u"ms"], - u"math-partial-sums__math-partial-sums": [22.0, u"ms"], + "access-nsieve__access-nsieve": [7.9, "ms"], + "bitops-nsieve-bits__bitops-nsieve-bits": [9.4, "ms"], + "string-validate-input__string-validate-input": [19.3, "ms"], + "3d-raytrace__3d-raytrace": [24.7, "ms"], + "3d-cube__3d-cube": [28.0, "ms"], + "string-unpack-code__string-unpack-code": [46.7, "ms"], + "date-format-tofte__date-format-tofte": [26.3, "ms"], + "math-partial-sums__math-partial-sums": [22.0, "ms"], "\telemetry_Crosperf": ["PASS", ""], - u"crypto-aes__crypto-aes": [15.2, u"ms"], - u"bitops-bitwise-and__bitops-bitwise-and": [8.4, u"ms"], - u"crypto-md5__crypto-md5": [10.5, u"ms"], - u"string-tagcloud__string-tagcloud": [52.8, u"ms"], - u"access-nbody__access-nbody": [8.5, u"ms"], + "crypto-aes__crypto-aes": [15.2, "ms"], + "bitops-bitwise-and__bitops-bitwise-and": [8.4, "ms"], + "crypto-md5__crypto-md5": [10.5, "ms"], + "string-tagcloud__string-tagcloud": [52.8, "ms"], + "access-nbody__access-nbody": [8.5, "ms"], "retval": 0, - u"math-spectral-norm__math-spectral-norm": [6.6, u"ms"], - u"math-cordic__math-cordic": [8.7, u"ms"], - u"access-binary-trees__access-binary-trees": [4.5, u"ms"], - u"controlflow-recursive__controlflow-recursive": [4.4, u"ms"], - u"access-fannkuch__access-fannkuch": [17.8, u"ms"], - u"string-base64__string-base64": [16.0, u"ms"], - u"date-format-xparb__date-format-xparb": [20.9, u"ms"], - u"3d-morph__3d-morph": [22.1, u"ms"], - u"bitops-bits-in-byte__bitops-bits-in-byte": [9.1, u"ms"], - u"samples": [1, u"samples"], + "math-spectral-norm__math-spectral-norm": [6.6, "ms"], + "math-cordic__math-cordic": [8.7, "ms"], + "access-binary-trees__access-binary-trees": [4.5, "ms"], + "controlflow-recursive__controlflow-recursive": [4.4, "ms"], + "access-fannkuch__access-fannkuch": [17.8, "ms"], + "string-base64__string-base64": [16.0, "ms"], + "date-format-xparb__date-format-xparb": [20.9, "ms"], + "3d-morph__3d-morph": [22.1, "ms"], + "bitops-bits-in-byte__bitops-bits-in-byte": [9.1, "ms"], + "samples": [1, "samples"], }, ) @@ -1798,7 +1831,6 @@ class ResultTest(unittest.TestCase): @mock.patch.object(misc, "GetRoot") @mock.patch.object(command_executer.CommandExecuter, "RunCommand") def test_cleanup(self, mock_runcmd, mock_getroot): - # Test 1. 'rm_chroot_tmp' is True; self.results_dir exists; # self.temp_dir exists; results_dir name contains 'test_that_results_'. mock_getroot.return_value = [ @@ -2031,7 +2063,6 @@ class TelemetryResultTest(unittest.TestCase): self.assertEqual(self.result.retval, 3) def test_populate_from_cache_dir_and_process_results(self): - self.result = TelemetryResult( self.mock_logger, self.mock_label, "average", self.mock_machine ) @@ -2244,7 +2275,6 @@ class ResultsCacheTest(unittest.TestCase): @mock.patch.object(os.path, "isdir") @mock.patch.object(Result, "CreateFromCacheHit") def test_read_result(self, mock_create, mock_isdir, mock_runcmd): - self.fakeCacheReturnResult = None def FakeGetCacheDirForRead(): @@ -2303,7 +2333,8 @@ class ResultsCacheTest(unittest.TestCase): self.assertIsNone(res) # Test 5. os.path.isdir returns true, but mock_create now returns None - # (the call to CreateFromCacheHit returns None), so overal result is None. + # (the call to CreateFromCacheHit returns None), so overal result is + # None. mock_isdir.return_value = True mock_create.return_value = None res = self.results_cache.ReadResult() diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py index 6382bba7..b34f0b16 100644 --- a/crosperf/settings_factory.py +++ b/crosperf/settings_factory.py @@ -206,7 +206,7 @@ class GlobalSettings(Settings): self.AddField( BooleanField( "rerun_if_failed", - description="Whether to re-run failed test runs " "or not.", + description="Whether to re-run failed test runs or not.", default=False, ) ) @@ -235,6 +235,13 @@ class GlobalSettings(Settings): ) self.AddField( BooleanField( + "ignore_cache", + description='Alias of "rerun" to ignore cache.', + default=False, + ) + ) + self.AddField( + BooleanField( "same_specs", default=True, description="Ensure cached runs are run on the " @@ -246,7 +253,7 @@ class GlobalSettings(Settings): BooleanField( "same_machine", default=False, - description="Ensure cached runs are run on the " "same remote.", + description="Ensure cached runs are run on the same remote.", ) ) self.AddField( @@ -552,6 +559,15 @@ class GlobalSettings(Settings): " Useful when lock is held externally, say with crosfleet.", ) ) + self.AddField( + BooleanField( + "keep_stateful", + default=False, + description="When flashing a ChromeOS image keep the stateful" + " partition, i.e. don't use --clobber-stateful. This option" + " is useful to keep ssh keys, wi-fi settings and so on.", + ) + ) class SettingsFactory(object): diff --git a/crosperf/settings_factory_unittest.py b/crosperf/settings_factory_unittest.py index 93d3bd6d..a6771c03 100755 --- a/crosperf/settings_factory_unittest.py +++ b/crosperf/settings_factory_unittest.py @@ -49,7 +49,7 @@ class GlobalSettingsTest(unittest.TestCase): def test_init(self): res = settings_factory.GlobalSettings("g_settings") self.assertIsNotNone(res) - self.assertEqual(len(res.fields), 40) + self.assertEqual(len(res.fields), 42) self.assertEqual(res.GetField("name"), "") self.assertEqual(res.GetField("board"), "") self.assertEqual(res.GetField("crosfleet"), False) @@ -58,6 +58,7 @@ class GlobalSettingsTest(unittest.TestCase): self.assertEqual(res.GetField("rm_chroot_tmp"), False) self.assertEqual(res.GetField("email"), None) self.assertEqual(res.GetField("rerun"), False) + self.assertEqual(res.GetField("ignore_cache"), False) self.assertEqual(res.GetField("same_specs"), True) self.assertEqual(res.GetField("same_machine"), False) self.assertEqual(res.GetField("iterations"), 0) @@ -114,7 +115,7 @@ class SettingsFactoryTest(unittest.TestCase): "global", "global" ) self.assertIsInstance(g_settings, settings_factory.GlobalSettings) - self.assertEqual(len(g_settings.fields), 40) + self.assertEqual(len(g_settings.fields), 42) if __name__ == "__main__": diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py index e777a57f..f5566f51 100644 --- a/crosperf/suite_runner.py +++ b/crosperf/suite_runner.py @@ -17,6 +17,7 @@ import subprocess import time from cros_utils import command_executer +from cros_utils import misc # sshwatcher path, relative to ChromiumOS source root. @@ -152,7 +153,9 @@ class SuiteRunner(object): def RemoveTelemetryTempFile(self, machine, chromeos_root): filename = "telemetry@%s" % machine - fullname = os.path.join(chromeos_root, "chroot", "tmp", filename) + fullname = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", filename) + ) if os.path.exists(fullname): os.remove(fullname) @@ -286,7 +289,7 @@ class SuiteRunner(object): def DownloadResult(self, label, task_id): gsutil_cmd = os.path.join(label.chromeos_root, GS_UTIL) result_dir = "gs://chromeos-autotest-results/swarming-%s" % task_id - download_path = os.path.join(label.chromeos_root, "chroot/tmp") + download_path = misc.GetOutsideChrootPath(label.chromeos_root, "/tmp") ls_command = "%s ls %s" % ( gsutil_cmd, os.path.join(result_dir, "autoserv_test"), diff --git a/heatmaps/heat_map.py b/heatmaps/heat_map.py deleted file mode 100755 index a3c52369..00000000 --- a/heatmaps/heat_map.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2015 The ChromiumOS Authors -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Wrapper to generate heat maps for chrome.""" - - -import argparse -import os -import shutil -import sys -import tempfile - -from cros_utils import command_executer -from heatmaps import heatmap_generator - - -def IsARepoRoot(directory): - """Returns True if directory is the root of a repo checkout.""" - return os.path.exists( - os.path.join(os.path.realpath(os.path.expanduser(directory)), ".repo") - ) - - -class HeatMapProducer(object): - """Class to produce heat map.""" - - def __init__( - self, chromeos_root, perf_data, hugepage, binary, title, logger=None - ): - self.chromeos_root = os.path.realpath(os.path.expanduser(chromeos_root)) - self.perf_data = os.path.realpath(os.path.expanduser(perf_data)) - self.hugepage = hugepage - self.dir = os.path.dirname(os.path.realpath(__file__)) - self.binary = binary - self.ce = command_executer.GetCommandExecuter() - self.temp_dir = "" - self.temp_perf_inchroot = "" - self.temp_dir_created = False - self.perf_report = "" - self.title = title - self.logger = logger - - def _EnsureFileInChroot(self): - chroot_prefix = os.path.join(self.chromeos_root, "chroot") - if self.perf_data.startswith(chroot_prefix): - # If the path to perf_data starts with the same chromeos_root, assume - # it's in the chromeos_root so no need for temporary directory and copy. - self.temp_dir = self.perf_data.replace("perf.data", "") - self.temp_perf_inchroot = self.temp_dir.replace(chroot_prefix, "") - - else: - # Otherwise, create a temporary directory and copy perf.data into chroot. - self.temp_dir = tempfile.mkdtemp( - prefix=os.path.join(self.chromeos_root, "src/") - ) - temp_perf = os.path.join(self.temp_dir, "perf.data") - shutil.copy2(self.perf_data, temp_perf) - self.temp_perf_inchroot = os.path.join( - "~/trunk/src", os.path.basename(self.temp_dir) - ) - self.temp_dir_created = True - - def _GeneratePerfReport(self): - cmd = ( - "cd %s && perf report -D -i perf.data > perf_report.txt" - % self.temp_perf_inchroot - ) - retval = self.ce.ChrootRunCommand(self.chromeos_root, cmd) - if retval: - raise RuntimeError("Failed to generate perf report") - self.perf_report = os.path.join(self.temp_dir, "perf_report.txt") - - def _GetHeatMap(self, top_n_pages): - generator = heatmap_generator.HeatmapGenerator( - perf_report=self.perf_report, - page_size=4096, - hugepage=self.hugepage, - title=self.title, - ) - generator.draw() - # Analyze top N hottest symbols with the binary, if provided - if self.binary: - generator.analyze(self.binary, top_n_pages) - - def _RemoveFiles(self): - files = [ - "out.txt", - "inst-histo.txt", - "inst-histo-hp.txt", - "inst-histo-sp.txt", - ] - for f in files: - if os.path.exists(f): - os.remove(f) - - def Run(self, top_n_pages): - try: - self._EnsureFileInChroot() - self._GeneratePerfReport() - self._GetHeatMap(top_n_pages) - finally: - self._RemoveFiles() - msg = ( - "heat map and time histogram genereated in the current " - "directory with name heat_map.png and timeline.png " - "accordingly." - ) - if self.binary: - msg += ( - "\nThe hottest %d pages inside and outside hugepage " - "is symbolized and saved to addr2symbol.txt" % top_n_pages - ) - if self.logger: - self.logger.LogOutput(msg) - else: - print(msg) - - -def main(argv): - """Parse the options. - - Args: - argv: The options with which this script was invoked. - - Returns: - 0 unless an exception is raised. - """ - parser = argparse.ArgumentParser() - - parser.add_argument( - "--chromeos_root", - dest="chromeos_root", - required=True, - help="ChromeOS root to use for generate heatmaps.", - ) - parser.add_argument( - "--perf_data", - dest="perf_data", - required=True, - help="The raw perf data. Must be collected with -e instructions while " - "disabling ASLR.", - ) - parser.add_argument( - "--binary", - dest="binary", - help="The path to the Chrome binary. Only useful if want to print " - "symbols on hottest pages", - default=None, - ) - parser.add_argument( - "--top_n", - dest="top_n", - type=int, - default=10, - help="Print out top N hottest pages within/outside huge page range. " - "Must be used with --hugepage and --binary. (Default: %(default)s)", - ) - parser.add_argument( - "--title", dest="title", help="Title of the heatmap", default="" - ) - parser.add_argument( - "--hugepage", - dest="hugepage", - help="A range of addresses (start,end) where huge page starts and ends" - " in text section, separated by a comma." - " Used to differentiate regions in heatmap." - " Example: --hugepage=0,4096" - " If not specified, no effect on the heatmap.", - default=None, - ) - - options = parser.parse_args(argv) - - if not IsARepoRoot(options.chromeos_root): - parser.error("%s does not contain .repo dir." % options.chromeos_root) - - if not os.path.isfile(options.perf_data): - parser.error("Cannot find perf_data: %s." % options.perf_data) - - hugepage_range = None - if options.hugepage: - hugepage_range = options.hugepage.split(",") - if len(hugepage_range) != 2 or int(hugepage_range[0]) > int( - hugepage_range[1] - ): - parser.error( - "Wrong format of hugepage range: %s" % options.hugepage - ) - hugepage_range = [int(x) for x in hugepage_range] - - heatmap_producer = HeatMapProducer( - options.chromeos_root, - options.perf_data, - hugepage_range, - options.binary, - options.title, - ) - - heatmap_producer.Run(options.top_n) - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/heatmaps/heat_map_test.py b/heatmaps/heat_map_test.py deleted file mode 100755 index 96300bb4..00000000 --- a/heatmaps/heat_map_test.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2019 The ChromiumOS Authors -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Tests for heat_map.py.""" - - -import os -import unittest -import unittest.mock as mock - -from cros_utils import command_executer -from heatmaps import heat_map -from heatmaps import heatmap_generator - - -def make_heatmap( - chromeos_root="/path/to/fake/chromeos_root/", - perf_data="/any_path/perf.data", -): - return heat_map.HeatMapProducer(chromeos_root, perf_data, None, None, "") - - -def fake_mkdtemp(prefix): - """Mock tempfile.mkdtemp() by just create a pathname.""" - return prefix + "random_dir" - - -def fake_parser_error(_, msg): - """Redirect parser.error() to exception.""" - raise Exception(msg) - - -def fake_generate_perf_report_exception(_): - raise Exception - - -class HeatmapTest(unittest.TestCase): - """All of our tests for heat_map.""" - - # pylint: disable=protected-access - @mock.patch("shutil.copy2") - @mock.patch("tempfile.mkdtemp") - def test_EnsureFileInChrootAlreadyInside(self, mock_mkdtemp, mock_copy): - perf_data_inchroot = ( - "/path/to/fake/chromeos_root/chroot/inchroot_path/perf.data" - ) - heatmap = make_heatmap(perf_data=perf_data_inchroot) - heatmap._EnsureFileInChroot() - self.assertFalse(heatmap.temp_dir_created) - self.assertEqual( - heatmap.temp_dir, - "/path/to/fake/chromeos_root/chroot/inchroot_path/", - ) - self.assertEqual(heatmap.temp_perf_inchroot, "/inchroot_path/") - mock_mkdtemp.assert_not_called() - mock_copy.assert_not_called() - - @mock.patch("shutil.copy2") - @mock.patch("tempfile.mkdtemp", fake_mkdtemp) - def test_EnsureFileInChrootOutsideNeedCopy(self, mock_copy): - heatmap = make_heatmap() - heatmap._EnsureFileInChroot() - self.assertTrue(heatmap.temp_dir_created) - self.assertEqual(mock_copy.call_count, 1) - self.assertEqual( - heatmap.temp_dir, "/path/to/fake/chromeos_root/src/random_dir" - ) - self.assertEqual(heatmap.temp_perf_inchroot, "~/trunk/src/random_dir") - - @mock.patch.object(command_executer.CommandExecuter, "ChrootRunCommand") - def test_GeneratePerfReport(self, mock_ChrootRunCommand): - heatmap = make_heatmap() - heatmap.temp_dir = "/fake/chroot/inchroot_path/" - heatmap.temp_perf_inchroot = "/inchroot_path/" - mock_ChrootRunCommand.return_value = 0 - heatmap._GeneratePerfReport() - cmd = ( - "cd %s && perf report -D -i perf.data > perf_report.txt" - % heatmap.temp_perf_inchroot - ) - mock_ChrootRunCommand.assert_called_with(heatmap.chromeos_root, cmd) - self.assertEqual(mock_ChrootRunCommand.call_count, 1) - self.assertEqual( - heatmap.perf_report, "/fake/chroot/inchroot_path/perf_report.txt" - ) - - @mock.patch.object(heatmap_generator, "HeatmapGenerator") - def test_GetHeatMap(self, mock_heatmap_generator): - heatmap = make_heatmap() - heatmap._GetHeatMap(10) - self.assertTrue(mock_heatmap_generator.called) - - @mock.patch.object(heat_map.HeatMapProducer, "_EnsureFileInChroot") - @mock.patch.object(heat_map.HeatMapProducer, "_GeneratePerfReport") - @mock.patch.object(heat_map.HeatMapProducer, "_GetHeatMap") - @mock.patch.object(heat_map.HeatMapProducer, "_RemoveFiles") - def test_Run( - self, - mock_remove_files, - mock_get_heatmap, - mock_generate_perf_report, - mock_ensure_file_in_chroot, - ): - heatmap = make_heatmap() - heatmap.Run(10) - mock_ensure_file_in_chroot.assert_called_once_with() - mock_generate_perf_report.assert_called_once_with() - mock_get_heatmap.assert_called_once_with(10) - mock_remove_files.assert_called_once_with() - - @mock.patch.object(heat_map.HeatMapProducer, "_EnsureFileInChroot") - @mock.patch.object( - heat_map.HeatMapProducer, - "_GeneratePerfReport", - new=fake_generate_perf_report_exception, - ) - @mock.patch.object(heat_map.HeatMapProducer, "_GetHeatMap") - @mock.patch.object(heat_map.HeatMapProducer, "_RemoveFiles") - @mock.patch("builtins.print") - def test_Run_with_exception( - self, - mock_print, - mock_remove_files, - mock_get_heatmap, - mock_ensure_file_in_chroot, - ): - heatmap = make_heatmap() - with self.assertRaises(Exception): - heatmap.Run(10) - mock_ensure_file_in_chroot.assert_called_once_with() - mock_get_heatmap.assert_not_called() - mock_remove_files.assert_called_once_with() - mock_print.assert_not_called() - - @mock.patch("argparse.ArgumentParser.error", fake_parser_error) - @mock.patch.object(os.path, "isfile") - @mock.patch.object(heat_map, "IsARepoRoot") - def test_main_arg_format(self, mock_IsARepoRoot, mock_isfile): - """Test wrong arg format are detected.""" - args = [ - "--chromeos_root=/fake/chroot/", - "--perf_data=/path/to/perf.data", - ] - - # Test --chromeos_root format - mock_IsARepoRoot.return_value = False - with self.assertRaises(Exception) as msg: - heat_map.main(args) - self.assertIn("does not contain .repo dir.", str(msg.exception)) - - # Test --perf_data format - mock_IsARepoRoot.return_value = True - mock_isfile.return_value = False - with self.assertRaises(Exception) as msg: - heat_map.main(args) - self.assertIn("Cannot find perf_data", str(msg.exception)) - - # Test --hugepage format - mock_isfile.return_value = True - args.append("--hugepage=0") - with self.assertRaises(Exception) as msg: - heat_map.main(args) - self.assertIn("Wrong format of hugepage range", str(msg.exception)) - - # Test --hugepage parse - args[-1] = "--hugepage=0,4096" - heat_map.HeatMapProducer = mock.MagicMock() - heat_map.main(args) - heat_map.HeatMapProducer.assert_called_with( - "/fake/chroot/", "/path/to/perf.data", [0, 4096], None, "" - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/heatmaps/heatmap_generator.py b/heatmaps/heatmap_generator.py deleted file mode 100644 index 703c37d4..00000000 --- a/heatmaps/heatmap_generator.py +++ /dev/null @@ -1,524 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2018 The ChromiumOS Authors -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Python module to draw heat map for Chrome - -heat map is a histogram used to analyze the locality of function layout. - -This module is used by heat_map.py. HeatmapGenerator is a class to -generate data for drawing heat maps (the actual drawing of heat maps is -performed by another script perf-to-inst-page.sh). It can also analyze -the symbol names in hot pages. -""" - - -import bisect -import collections -import os -import pipes -import subprocess - -from cros_utils import command_executer - - -HugepageRange = collections.namedtuple("HugepageRange", ["start", "end"]) - - -class MMap(object): - """Class to store mmap information in perf report. - - We assume ASLR is disabled, so MMap for all Chrome is assumed to be - the same. This class deals with the case hugepage creates several - mmaps for Chrome but should be merged together. In these case, we - assume the first MMAP is not affected by the bug and use the MMAP. - """ - - def __init__(self, addr, size, offset): - self.start_address = addr - self.size = size - self.offset = offset - - def __str__(self): - return "(%x, %x, %x)" % (self.start_address, self.size, self.offset) - - def merge(self, mmap): - # This function should not be needed, since we should only have - # one MMAP on Chrome of each process. This function only deals with - # images that is affected by http://crbug.com/931465. - - # This function is only checking a few conditions to make sure - # the bug is within our expectation. - - if self.start_address == mmap.start_address: - assert ( - self.size >= mmap.size - ), "Original MMAP size(%x) is smaller than the forked process(%x)." % ( - self.size, - mmap.size, - ) - # The case that the MMAP is forked from the previous process - # No need to do anything, OR - # The case where hugepage causes a small Chrome mmap. - # In this case, we use the prior MMAP for the whole Chrome - return - - assert self.start_address < mmap.start_address, ( - "Original MMAP starting address(%x) is larger than the forked" - "process(%x)." % (self.start_address, mmap.start_address) - ) - - assert ( - self.start_address + self.size >= mmap.start_address + mmap.size - ), "MMAP of the forked process exceeds the end of original MMAP." - - -class HeatmapGenerator(object): - """Class to generate heat map with a perf report, containing mmaps and - - samples. This class contains two interfaces with other modules: - draw() and analyze(). - - draw() draws a heatmap with the sample information given in the perf report - analyze() prints out the symbol names in hottest pages with the given - chrome binary - """ - - def __init__( - self, perf_report, page_size, hugepage, title, log_level="verbose" - ): - self.perf_report = perf_report - # Pick 1G as a relatively large number. All addresses less than it will - # be recorded. The actual heatmap will show up to a boundary of the - # largest address in text segment. - self.max_addr = 1024 * 1024 * 1024 - self.ce = command_executer.GetCommandExecuter(log_level=log_level) - self.dir = os.path.dirname(os.path.realpath(__file__)) - with open(perf_report, "r", encoding="utf-8") as f: - self.perf_report_contents = f.readlines() - # Write histogram results to a text file, in order to use gnu plot to draw - self.hist_temp_output = open("out.txt", "w", encoding="utf-8") - self.processes = {} - self.deleted_processes = {} - self.count = 0 - if hugepage: - self.hugepage = HugepageRange(start=hugepage[0], end=hugepage[1]) - else: - self.hugepage = None - self.title = title - self.symbol_addresses = [] - self.symbol_names = [] - self.page_size = page_size - - def _parse_perf_sample(self, line): - # In a perf report, generated with -D, a PERF_RECORD_SAMPLE command should - # look like this: TODO: some arguments are unknown - # - # cpuid cycle unknown [unknown]: PERF_RECORD_SAMPLE(IP, 0x2): pid/tid: - # 0xaddr period: period addr: addr - # ... thread: threadname:tid - # ...... dso: process - # - # This is an example: - # 1 136712833349 0x6a558 [0x30]: PERF_RECORD_SAMPLE(IP, 0x2): 5227/5227: - # 0x55555683b810 period: 372151 addr: 0 - # ... thread: chrome:5227 - # ...... dso: /opt/google/chrome/chrome - # - # For this function, the 7th argument (args[6]) after spltting with spaces - # is pid/tid. We use the combination of the two as the pid. - # Also, we add an assertion here to check the tid in the 7th argument( - # args[6]) and the 15th argument(arg[14]) are the same - # - # The function returns the ((pid,tid), address) pair if the sampling - # is on Chrome. Otherwise, return (None, None) pair. - - if ( - "thread: chrome" not in line - or "dso: /opt/google/chrome/chrome" not in line - ): - return None, None - args = line.split(" ") - pid_raw = args[6].split("/") - assert ( - pid_raw[1][:-1] == args[14].split(":")[1][:-1] - ), "TID in %s of sample is not the same: %s/%s" % ( - line[:-1], - pid_raw[1][:-1], - args[14].split(":")[1][:-1], - ) - key = (int(pid_raw[0]), int(pid_raw[1][:-1])) - address = int(args[7], base=16) - return key, address - - def _parse_perf_record(self, line): - # In a perf report, generated with -D, a PERF_RECORD_MMAP2 command should - # look like this: TODO: some arguments are unknown - # - # cpuid cycle unknown [unknown]: PERF_RECORD_MMAP2 pid/tid: - # [0xaddr(0xlength) @ pageoffset maj:min ino ino_generation]: - # permission process - # - # This is an example. - # 2 136690556823 0xa6898 [0x80]: PERF_RECORD_MMAP2 5227/5227: - # [0x555556496000(0x8d1b000) @ 0xf42000 b3:03 92844 1892514370]: - # r-xp /opt/google/chrome/chrome - # - # For this function, the 6th argument (args[5]) after spltting with spaces - # is pid/tid. We use the combination of the two as the pid. - # The 7th argument (args[6]) is the [0xaddr(0xlength). We can peel the - # string to get the address and size of the mmap. - # The 9th argument (args[8]) is the page offset. - # The function returns the ((pid,tid), mmap) pair if the mmap is for Chrome - # is on Chrome. Otherwise, return (None, None) pair. - - if "chrome/chrome" not in line: - return None, None - args = line.split(" ") - pid_raw = args[5].split("/") - assert ( - pid_raw[0] == pid_raw[1][:-1] - ), "PID in %s of mmap is not the same: %s/%s" % ( - line[:-1], - pid_raw[0], - pid_raw[1], - ) - pid = (int(pid_raw[0]), int(pid_raw[1][:-1])) - address_raw = args[6].split("(") - start_address = int(address_raw[0][1:], base=16) - size = int(address_raw[1][:-1], base=16) - offset = int(args[8], base=16) - # Return an mmap object instead of only starting address, - # in case there are many mmaps for the sample PID - return pid, MMap(start_address, size, offset) - - def _parse_pair_event(self, arg): - # This function is called by the _parse_* functions that has a pattern of - # pids like: (pid:tid):(pid:tid), i.e. - # PERF_RECORD_FORK and PERF_RECORD_COMM - _, remain = arg.split("(", 1) - pid1, remain = remain.split(":", 1) - pid2, remain = remain.split(")", 1) - _, remain = remain.split("(", 1) - pid3, remain = remain.split(":", 1) - pid4, remain = remain.split(")", 1) - return (int(pid1), int(pid2)), (int(pid3), int(pid4)) - - def _process_perf_record(self, line): - # This function calls _parse_perf_record() to get information from - # PERF_RECORD_MMAP2. It records the mmap object for each pid (a pair of - # pid,tid), into a dictionary. - pid, mmap = self._parse_perf_record(line) - if pid is None: - # PID = None meaning the mmap is not for chrome - return - if pid in self.processes: - # This should never happen for a correct profiling result, as we - # should only have one MMAP for Chrome for each process. - # If it happens, see http://crbug.com/931465 - self.processes[pid].merge(mmap) - else: - self.processes[pid] = mmap - - def _process_perf_fork(self, line): - # In a perf report, generated with -D, a PERF_RECORD_FORK command should - # look like this: - # - # cpuid cycle unknown [unknown]: - # PERF_RECORD_FORK(pid_to:tid_to):(pid_from:tid_from) - # - # This is an example. - # 0 0 0x22a8 [0x38]: PERF_RECORD_FORK(1:1):(0:0) - # - # In this function, we need to peel the information of pid:tid pairs - # So we get the last argument and send it to function _parse_pair_event() - # for analysis. - # We use (pid, tid) as the pid. - args = line.split(" ") - pid_to, pid_from = self._parse_pair_event(args[-1]) - if pid_from in self.processes: - assert pid_to not in self.processes - self.processes[pid_to] = MMap( - self.processes[pid_from].start_address, - self.processes[pid_from].size, - self.processes[pid_from].offset, - ) - - def _process_perf_exit(self, line): - # In a perf report, generated with -D, a PERF_RECORD_EXIT command should - # look like this: - # - # cpuid cycle unknown [unknown]: - # PERF_RECORD_EXIT(pid1:tid1):(pid2:tid2) - # - # This is an example. - # 1 136082505621 0x30810 [0x38]: PERF_RECORD_EXIT(3851:3851):(3851:3851) - # - # In this function, we need to peel the information of pid:tid pairs - # So we get the last argument and send it to function _parse_pair_event() - # for analysis. - # We use (pid, tid) as the pid. - args = line.split(" ") - pid_to, pid_from = self._parse_pair_event(args[-1]) - assert pid_to == pid_from, "(%d, %d) (%d, %d)" % ( - pid_to[0], - pid_to[1], - pid_from[0], - pid_from[1], - ) - if pid_to in self.processes: - # Don't delete the process yet - self.deleted_processes[pid_from] = self.processes[pid_from] - - def _process_perf_sample(self, line): - # This function calls _parse_perf_sample() to get information from - # the perf report. - # It needs to check the starting address of allocated mmap from - # the dictionary (self.processes) to calculate the offset within - # the text section of the sampling. - # The offset is calculated into pages (4KB or 2MB) and writes into - # out.txt together with the total counts, which will be used to - # calculate histogram. - pid, addr = self._parse_perf_sample(line) - if pid is None: - return - - assert ( - pid in self.processes and pid not in self.deleted_processes - ), "PID %d not found mmap and not forked from another process" - - start_address = self.processes[pid].start_address - address = addr - start_address - assert ( - address >= 0 - and "addresses accessed in PERF_RECORD_SAMPLE should be larger than" - " the starting address of Chrome" - ) - if address < self.max_addr: - self.count += 1 - line = "%d/%d: %d %d" % ( - pid[0], - pid[1], - self.count, - address // self.page_size * self.page_size, - ) - if self.hugepage: - if self.hugepage.start <= address < self.hugepage.end: - line += " hugepage" - else: - line += " smallpage" - print(line, file=self.hist_temp_output) - - def _read_perf_report(self): - # Serve as main function to read perf report, generated by -D - lines = iter(self.perf_report_contents) - for line in lines: - if "PERF_RECORD_MMAP" in line: - self._process_perf_record(line) - elif "PERF_RECORD_FORK" in line: - self._process_perf_fork(line) - elif "PERF_RECORD_EXIT" in line: - self._process_perf_exit(line) - elif "PERF_RECORD_SAMPLE" in line: - # Perf sample is multi-line - self._process_perf_sample(line + next(lines) + next(lines)) - self.hist_temp_output.close() - - def _draw_heat_map(self): - # Calls a script (perf-to-inst-page.sh) to calculate histogram - # of results written in out.txt and also generate pngs for - # heat maps. - heatmap_script = os.path.join(self.dir, "perf-to-inst-page.sh") - if self.hugepage: - hp_arg = "hugepage" - else: - hp_arg = "none" - - cmd = "{0} {1} {2}".format( - heatmap_script, pipes.quote(self.title), hp_arg - ) - retval = self.ce.RunCommand(cmd) - if retval: - raise RuntimeError("Failed to run script to generate heatmap") - - def _restore_histogram(self): - # When hugepage is used, there are two files inst-histo-{hp,sp}.txt - # So we need to read in all the files. - names = [ - x for x in os.listdir(".") if "inst-histo" in x and ".txt" in x - ] - hist = {} - for n in names: - with open(n, encoding="utf-8") as f: - for l in f.readlines(): - num, addr = l.strip().split(" ") - assert int(addr) not in hist - hist[int(addr)] = int(num) - return hist - - def _read_symbols_from_binary(self, binary): - # FIXME: We are using nm to read symbol names from Chrome binary - # for now. Can we get perf to hand us symbol names, instead of - # using nm in the future? - # - # Get all the symbols (and their starting addresses) that fall into - # the page. Will be used to print out information of hot pages - # Each line shows the information of a symbol: - # [symbol value (0xaddr)] [symbol type] [symbol name] - # For some symbols, the [symbol name] field might be missing. - # e.g. - # 0000000001129da0 t Builtins_LdaNamedPropertyHandler - - # Generate a list of symbols from nm tool and check each line - # to extract symbols names - text_section_start = 0 - for l in subprocess.check_output(["nm", "-n", binary]).split("\n"): - args = l.strip().split(" ") - if len(args) < 3: - # No name field - continue - addr_raw, symbol_type, name = args - addr = int(addr_raw, base=16) - if "t" not in symbol_type and "T" not in symbol_type: - # Filter out symbols not in text sections - continue - if not self.symbol_addresses: - # The first symbol in text sections - text_section_start = addr - self.symbol_addresses.append(0) - self.symbol_names.append(name) - else: - assert ( - text_section_start != 0 - ), "The starting address of text section has not been found" - if addr == self.symbol_addresses[-1]: - # if the same address has multiple symbols, put them together - # and separate symbol names with '/' - self.symbol_names[-1] += "/" + name - else: - # The output of nm -n command is already sorted by address - # Insert to the end will result in a sorted array for bisect - self.symbol_addresses.append(addr - text_section_start) - self.symbol_names.append(name) - - def _map_addr_to_symbol(self, addr): - # Find out the symbol name - assert self.symbol_addresses - index = bisect.bisect(self.symbol_addresses, addr) - assert ( - 0 < index <= len(self.symbol_names) - ), "Failed to find an index (%d) in the list (len=%d)" % ( - index, - len(self.symbol_names), - ) - return self.symbol_names[index - 1] - - def _print_symbols_in_hot_pages(self, fp, pages_to_show): - # Print symbols in all the pages of interest - for page_num, sample_num in pages_to_show: - print( - "----------------------------------------------------------", - file=fp, - ) - print( - "Page Offset: %d MB, Count: %d" - % (page_num // 1024 // 1024, sample_num), - file=fp, - ) - - symbol_counts = collections.Counter() - # Read Sample File and find out the occurance of symbols in the page - lines = iter(self.perf_report_contents) - for line in lines: - if "PERF_RECORD_SAMPLE" in line: - pid, addr = self._parse_perf_sample( - line + next(lines) + next(lines) - ) - if pid is None: - # The sampling is not on Chrome - continue - if ( - addr // self.page_size - != (self.processes[pid].start_address + page_num) - // self.page_size - ): - # Sampling not in the current page - continue - - name = self._map_addr_to_symbol( - addr - self.processes[pid].start_address - ) - assert name, "Failed to find symbol name of addr %x" % addr - symbol_counts[name] += 1 - - assert ( - sum(symbol_counts.values()) == sample_num - ), "Symbol name matching missing for some addresses: %d vs %d" % ( - sum(symbol_counts.values()), - sample_num, - ) - - # Print out the symbol names sorted by the number of samples in - # the page - for name, count in sorted( - symbol_counts.items(), key=lambda kv: kv[1], reverse=True - ): - if count == 0: - break - print("> %s : %d" % (name, count), file=fp) - print("\n\n", file=fp) - - def draw(self): - # First read perf report to process information and save histogram - # into a text file - self._read_perf_report() - # Then use gnu plot to draw heat map - self._draw_heat_map() - - def analyze(self, binary, top_n): - # Read histogram from histo.txt - hist = self._restore_histogram() - # Sort the pages in histogram - sorted_hist = sorted( - hist.items(), key=lambda value: value[1], reverse=True - ) - - # Generate symbolizations - self._read_symbols_from_binary(binary) - - # Write hottest pages - with open("addr2symbol.txt", "w", encoding="utf-8") as fp: - if self.hugepage: - # Print hugepage region first - print( - "Hugepage top %d hot pages (%d MB - %d MB):" - % ( - top_n, - self.hugepage.start // 1024 // 1024, - self.hugepage.end // 1024 // 1024, - ), - file=fp, - ) - pages_to_print = [ - (k, v) - for k, v in sorted_hist - if self.hugepage.start <= k < self.hugepage.end - ][:top_n] - self._print_symbols_in_hot_pages(fp, pages_to_print) - print("==========================================", file=fp) - print( - "Top %d hot pages landed outside of hugepage:" % top_n, - file=fp, - ) - # Then print outside pages - pages_to_print = [ - (k, v) - for k, v in sorted_hist - if k < self.hugepage.start or k >= self.hugepage.end - ][:top_n] - self._print_symbols_in_hot_pages(fp, pages_to_print) - else: - # Print top_n hottest pages. - pages_to_print = sorted_hist[:top_n] - self._print_symbols_in_hot_pages(fp, pages_to_print) diff --git a/heatmaps/heatmap_generator_test.py b/heatmaps/heatmap_generator_test.py deleted file mode 100755 index 898c7370..00000000 --- a/heatmaps/heatmap_generator_test.py +++ /dev/null @@ -1,326 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2018 The ChromiumOS Authors -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Tests for heatmap_generator.py.""" - - -import os -import unittest -import unittest.mock as mock - -from heatmaps import heatmap_generator - - -def _write_perf_mmap(pid, tid, addr, size, fp): - print( - "0 0 0 0 PERF_RECORD_MMAP2 %d/%d: " - "[%x(%x) @ 0x0 0:0 0 0] " - "r-xp /opt/google/chrome/chrome\n" % (pid, tid, addr, size), - file=fp, - ) - - -def _write_perf_fork(pid_from, tid_from, pid_to, tid_to, fp): - print( - "0 0 0 0 PERF_RECORD_FORK(%d:%d):(%d:%d)\n" - % (pid_to, tid_to, pid_from, tid_from), - file=fp, - ) - - -def _write_perf_exit(pid_from, tid_from, pid_to, tid_to, fp): - print( - "0 0 0 0 PERF_RECORD_EXIT(%d:%d):(%d:%d)\n" - % (pid_to, tid_to, pid_from, tid_from), - file=fp, - ) - - -def _write_perf_sample(pid, tid, addr, fp): - print( - "0 0 0 0 PERF_RECORD_SAMPLE(IP, 0x2): " - "%d/%d: %x period: 100000 addr: 0" % (pid, tid, addr), - file=fp, - ) - print(" ... thread: chrome:%d" % tid, file=fp) - print(" ...... dso: /opt/google/chrome/chrome\n", file=fp) - - -def _heatmap(file_name, page_size=4096, hugepage=None, analyze=False, top_n=10): - generator = heatmap_generator.HeatmapGenerator( - file_name, page_size, hugepage, "", log_level="none" - ) # Don't log to stdout - generator.draw() - if analyze: - generator.analyze("/path/to/chrome", top_n) - - -def _cleanup(file_name): - files = [ - file_name, - "out.txt", - "inst-histo.txt", - "inst-histo-hp.txt", - "inst-histo-sp.txt", - "heat_map.png", - "timeline.png", - "addr2symbol.txt", - ] - for f in files: - if os.path.exists(f): - os.remove(f) - - -class HeatmapGeneratorDrawTests(unittest.TestCase): - """All of our tests for heatmap_generator.draw() and related.""" - - def test_with_one_mmap_one_sample(self): - """Tests one perf record and one sample.""" - fname = "test.txt" - with open(fname, "w") as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - _write_perf_sample(101, 101, 0xABCD101, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname) - self.assertIn("out.txt", os.listdir(".")) - with open("out.txt") as f: - lines = f.readlines() - self.assertEqual(len(lines), 1) - self.assertIn("101/101: 1 0", lines[0]) - - def test_with_one_mmap_multiple_samples(self): - """Tests one perf record and three samples.""" - fname = "test.txt" - with open(fname, "w") as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - _write_perf_sample(101, 101, 0xABCD101, f) - _write_perf_sample(101, 101, 0xABCD102, f) - _write_perf_sample(101, 101, 0xABCE102, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname) - self.assertIn("out.txt", os.listdir(".")) - with open("out.txt") as f: - lines = f.readlines() - self.assertEqual(len(lines), 3) - self.assertIn("101/101: 1 0", lines[0]) - self.assertIn("101/101: 2 0", lines[1]) - self.assertIn("101/101: 3 4096", lines[2]) - - def test_with_fork_and_exit(self): - """Tests perf fork and perf exit.""" - fname = "test_fork.txt" - with open(fname, "w") as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - _write_perf_fork(101, 101, 202, 202, f) - _write_perf_sample(101, 101, 0xABCD101, f) - _write_perf_sample(202, 202, 0xABCE101, f) - _write_perf_exit(202, 202, 202, 202, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname) - self.assertIn("out.txt", os.listdir(".")) - with open("out.txt") as f: - lines = f.readlines() - self.assertEqual(len(lines), 2) - self.assertIn("101/101: 1 0", lines[0]) - self.assertIn("202/202: 2 4096", lines[1]) - - def test_hugepage_creates_two_chrome_mmaps(self): - """Test two chrome mmaps for the same process.""" - fname = "test_hugepage.txt" - with open(fname, "w") as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x1000, f) - _write_perf_fork(101, 101, 202, 202, f) - _write_perf_mmap(202, 202, 0xABCD000, 0x100, f) - _write_perf_mmap(202, 202, 0xABCD300, 0xD00, f) - _write_perf_sample(101, 101, 0xABCD102, f) - _write_perf_sample(202, 202, 0xABCD102, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname) - self.assertIn("out.txt", os.listdir(".")) - with open("out.txt") as f: - lines = f.readlines() - self.assertEqual(len(lines), 2) - self.assertIn("101/101: 1 0", lines[0]) - self.assertIn("202/202: 2 0", lines[1]) - - def test_hugepage_creates_two_chrome_mmaps_fail(self): - """Test two chrome mmaps for the same process.""" - fname = "test_hugepage.txt" - # Cases where first_mmap.size < second_mmap.size - with open(fname, "w") as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x1000, f) - _write_perf_fork(101, 101, 202, 202, f) - _write_perf_mmap(202, 202, 0xABCD000, 0x10000, f) - self.addCleanup(_cleanup, fname) - with self.assertRaises(AssertionError) as msg: - _heatmap(fname) - self.assertIn("Original MMAP size", str(msg.exception)) - - # Cases where first_mmap.address > second_mmap.address - with open(fname, "w") as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x1000, f) - _write_perf_fork(101, 101, 202, 202, f) - _write_perf_mmap(202, 202, 0xABCC000, 0x10000, f) - with self.assertRaises(AssertionError) as msg: - _heatmap(fname) - self.assertIn("Original MMAP starting address", str(msg.exception)) - - # Cases where first_mmap.address + size < - # second_mmap.address + second_mmap.size - with open(fname, "w") as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x1000, f) - _write_perf_fork(101, 101, 202, 202, f) - _write_perf_mmap(202, 202, 0xABCD100, 0x10000, f) - with self.assertRaises(AssertionError) as msg: - _heatmap(fname) - self.assertIn("exceeds the end of original MMAP", str(msg.exception)) - - def test_histogram(self): - """Tests if the tool can generate correct histogram. - - In the tool, histogram is generated from statistics - of perf samples (saved to out.txt). The histogram is - generated by perf-to-inst-page.sh and saved to - inst-histo.txt. It will be used to draw heat maps. - """ - fname = "test_histo.txt" - with open(fname, "w") as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - for i in range(100): - _write_perf_sample(101, 101, 0xABCD000 + i, f) - _write_perf_sample(101, 101, 0xABCE000 + i, f) - _write_perf_sample(101, 101, 0xABFD000 + i, f) - _write_perf_sample(101, 101, 0xAFCD000 + i, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname) - self.assertIn("inst-histo.txt", os.listdir(".")) - with open("inst-histo.txt") as f: - lines = f.readlines() - self.assertEqual(len(lines), 4) - self.assertIn("100 0", lines[0]) - self.assertIn("100 4096", lines[1]) - self.assertIn("100 196608", lines[2]) - self.assertIn("100 4194304", lines[3]) - - def test_histogram_two_mb_page(self): - """Tests handling of 2MB page.""" - fname = "test_histo.txt" - with open(fname, "w") as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - for i in range(100): - _write_perf_sample(101, 101, 0xABCD000 + i, f) - _write_perf_sample(101, 101, 0xABCE000 + i, f) - _write_perf_sample(101, 101, 0xABFD000 + i, f) - _write_perf_sample(101, 101, 0xAFCD000 + i, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname, page_size=2 * 1024 * 1024) - self.assertIn("inst-histo.txt", os.listdir(".")) - with open("inst-histo.txt") as f: - lines = f.readlines() - self.assertEqual(len(lines), 2) - self.assertIn("300 0", lines[0]) - self.assertIn("100 4194304", lines[1]) - - def test_histogram_in_and_out_hugepage(self): - """Tests handling the case of separating samples in and out huge page.""" - fname = "test_histo.txt" - with open(fname, "w") as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - for i in range(100): - _write_perf_sample(101, 101, 0xABCD000 + i, f) - _write_perf_sample(101, 101, 0xABCE000 + i, f) - _write_perf_sample(101, 101, 0xABFD000 + i, f) - _write_perf_sample(101, 101, 0xAFCD000 + i, f) - self.addCleanup(_cleanup, fname) - _heatmap(fname, hugepage=[0, 8192]) - file_list = os.listdir(".") - self.assertNotIn("inst-histo.txt", file_list) - self.assertIn("inst-histo-hp.txt", file_list) - self.assertIn("inst-histo-sp.txt", file_list) - with open("inst-histo-hp.txt") as f: - lines = f.readlines() - self.assertEqual(len(lines), 2) - self.assertIn("100 0", lines[0]) - self.assertIn("100 4096", lines[1]) - with open("inst-histo-sp.txt") as f: - lines = f.readlines() - self.assertEqual(len(lines), 2) - self.assertIn("100 196608", lines[0]) - self.assertIn("100 4194304", lines[1]) - - -class HeatmapGeneratorAnalyzeTests(unittest.TestCase): - """All of our tests for heatmap_generator.analyze() and related.""" - - def setUp(self): - # Use the same perf report for testing - self.fname = "test_histo.txt" - with open(self.fname, "w") as f: - _write_perf_mmap(101, 101, 0xABCD000, 0x100, f) - for i in range(10): - _write_perf_sample(101, 101, 0xABCD000 + i, f) - _write_perf_sample(101, 101, 0xABCE000 + i, f) - _write_perf_sample(101, 101, 0xABFD000 + i, f) - self.nm = ( - "000000000abcd000 t Func1@Page1\n" - "000000000abcd001 t Func2@Page1\n" - "000000000abcd0a0 t Func3@Page1andFunc1@Page2\n" - "000000000abce010 t Func2@Page2\n" - "000000000abfd000 t Func1@Page3\n" - ) - - def tearDown(self): - _cleanup(self.fname) - - @mock.patch("subprocess.check_output") - def test_analyze_hot_pages_with_hp_top(self, mock_nm): - """Test if the analyze() can print the top page with hugepage.""" - mock_nm.return_value = self.nm - _heatmap(self.fname, hugepage=[0, 8192], analyze=True, top_n=1) - file_list = os.listdir(".") - self.assertIn("addr2symbol.txt", file_list) - with open("addr2symbol.txt") as f: - contents = f.read() - self.assertIn("Func2@Page1 : 9", contents) - self.assertIn("Func1@Page1 : 1", contents) - self.assertIn("Func1@Page3 : 10", contents) - # Only displaying one page in hugepage - self.assertNotIn("Func3@Page1andFunc1@Page2 : 10", contents) - - @mock.patch("subprocess.check_output") - def test_analyze_hot_pages_without_hp_top(self, mock_nm): - """Test if the analyze() can print the top page without hugepage.""" - mock_nm.return_value = self.nm - _heatmap(self.fname, analyze=True, top_n=1) - file_list = os.listdir(".") - self.assertIn("addr2symbol.txt", file_list) - with open("addr2symbol.txt") as f: - contents = f.read() - self.assertIn("Func2@Page1 : 9", contents) - self.assertIn("Func1@Page1 : 1", contents) - # Only displaying one page - self.assertNotIn("Func3@Page1andFunc1@Page2 : 10", contents) - self.assertNotIn("Func1@Page3 : 10", contents) - - @mock.patch("subprocess.check_output") - def test_analyze_hot_pages_with_hp_top10(self, mock_nm): - """Test if the analyze() can print with default top 10.""" - mock_nm.return_value = self.nm - _heatmap(self.fname, analyze=True) - # Make sure nm command is called correctly. - mock_nm.assert_called_with(["nm", "-n", "/path/to/chrome"]) - file_list = os.listdir(".") - self.assertIn("addr2symbol.txt", file_list) - with open("addr2symbol.txt") as f: - contents = f.read() - self.assertIn("Func2@Page1 : 9", contents) - self.assertIn("Func1@Page1 : 1", contents) - self.assertIn("Func3@Page1andFunc1@Page2 : 10", contents) - self.assertIn("Func1@Page3 : 10", contents) - - -if __name__ == "__main__": - unittest.main() diff --git a/heatmaps/perf-to-inst-page.sh b/heatmaps/perf-to-inst-page.sh deleted file mode 100755 index 5be1a2b9..00000000 --- a/heatmaps/perf-to-inst-page.sh +++ /dev/null @@ -1,68 +0,0 @@ -#! /bin/bash -u -# Copyright 2015 The ChromiumOS Authors -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# This script takes the out.txt, generated by heatmap_generator.py -# and sorted into a heatmap data file (inst-histo.txt) and then -# call gnu plot to draw the heat map and the time map. -# A heat map shows the overall hotness of instructions being executed -# while the time map shows the hotness of instruction at different time. -# -# Usage: -# ./perf-to-inst-page.sh HEATMAP_TITLE -# HEATMAP_TITLE: the title to display on the heatmap - -HEAT_PNG="heat_map.png" -TIMELINE_PNG="timeline.png" -HEATMAP_TITLE=$1 -ENABLE_HUGEPAGE=$2 - -heatmap_command=" -set terminal png size 600,450 -set xlabel \"Instruction Virtual Address (MB)\" -set ylabel \"Sample Occurance\" -set grid - -set output \"${HEAT_PNG}\" -set title \"${HEATMAP_TITLE}\" -" - -if [[ "${ENABLE_HUGEPAGE}" = "hugepage" ]]; then - hugepage_hist="inst-histo-hp.txt" - smallpage_hist="inst-histo-sp.txt" - cat out.txt | grep hugepage | awk '{print $3}' \ - | sort -n | uniq -c > "${hugepage_hist}" - cat out.txt | grep smallpage | awk '{print $3}' \ - | sort -n | uniq -c > "${smallpage_hist}" - # generate inst heat map - heatmap_in_hugepage=("'${hugepage_hist}' using \ -(\$2/1024/1024):1 with impulses notitle lt rgb 'red'") - heatmap_outside_hugepage=("'${smallpage_hist}' using \ -(\$2/1024/1024):1 with impulses notitle lt rgb 'blue'") - echo " - ${heatmap_command} - plot ${heatmap_in_hugepage}, ${heatmap_outside_hugepage} - " | gnuplot -else - awk '{print $3}' out.txt | sort -n | uniq -c > inst-histo.txt - # generate inst heat map - echo " - ${heatmap_command} - plot 'inst-histo.txt' using (\$2/1024/1024):1 with impulses notitle - " | gnuplot -fi - -# generate instruction page access timeline -num=$(awk 'END {print NR+1}' out.txt) - -echo " -set terminal png size 600,450 -set xlabel \"time (sec)\" -set ylabel \"Instruction Virtual Address (MB)\" - -set output \"${TIMELINE_PNG}\" -set title \"instruction page accessd timeline\" - -plot 'out.txt' using (\$0/$num*10):(\$3/1024/1024) with dots notitle -" | gnuplot diff --git a/image_chromeos.py b/image_chromeos.py index 36965d05..38300591 100755 --- a/image_chromeos.py +++ b/image_chromeos.py @@ -15,7 +15,6 @@ __author__ = "asharif@google.com (Ahmad Sharif)" import argparse import filecmp -import getpass import glob import os import re @@ -50,12 +49,12 @@ def CheckForCrosFlash(chromeos_root, remote, log_level): command, chromeos_root=chromeos_root, machine=remote ) logger.GetLogger().LogFatalIf( - ret == 255, "Failed ssh to %s (for checking cherrypy)" % remote + ret == 255, f"Failed ssh to {remote} (for checking cherrypy)" ) logger.GetLogger().LogFatalIf( ret != 0, - "Failed to find cherrypy or ctypes on remote '{}', " - "cros flash cannot work.".format(remote), + f"Failed to find cherrypy or ctypes on '{remote}', " + "cros flash cannot work.", ) @@ -63,7 +62,7 @@ def DisableCrosBeeps(chromeos_root, remote, log_level): """Disable annoying chromebooks beeps after reboots.""" cmd_executer = command_executer.GetCommandExecuter(log_level=log_level) - command = "/usr/share/vboot/bin/set_gbb_flags.sh 0x1" + command = "/usr/bin/futility gbb --set --flash --flags=0x1" logger.GetLogger().LogOutput("Trying to disable beeping.") ret, o, _ = cmd_executer.CrosRunCommandWOutput( @@ -81,37 +80,21 @@ def FindChromeOSImage(image_file, chromeos_root): or outside the chroot. In either case the path needs to be translated to an real/absolute path inside the chroot. Example input paths: - /usr/local/google/home/uname/chromeos/chroot/tmp/my-test-images/image - ~/trunk/src/build/images/board/latest/image + /usr/local/google/home/uname/chromeos/out/tmp/my-test-images/image + ~/chromiumos/src/build/images/board/latest/image /tmp/peppy-release/R67-1235.0.0/image Corresponding example output paths: /tmp/my-test-images/image - /home/uname/trunk/src/build/images/board/latest/image + /mnt/host/source/src/build/images/board/latest/image /tmp/peppy-release/R67-1235.0,0/image """ - # Get the name of the user, for "/home/<user>" part of the path. - whoami = getpass.getuser() - # Get the full path for the chroot dir, including 'chroot' - real_chroot_dir = os.path.join(os.path.realpath(chromeos_root), "chroot") - # Get the full path for the chromeos root, excluding 'chroot' - real_chromeos_root = os.path.realpath(chromeos_root) - - # If path name starts with real_chroot_dir, remove that piece, but assume - # the rest of the path is correct. - if image_file.find(real_chroot_dir) != -1: - chroot_image = image_file[len(real_chroot_dir) :] - # If path name starts with chromeos_root, excluding 'chroot', replace the - # chromeos_root with the prefix: '/home/<username>/trunk'. - elif image_file.find(real_chromeos_root) != -1: - chroot_image = image_file[len(real_chromeos_root) :] - chroot_image = "/home/%s/trunk%s" % (whoami, chroot_image) - # Else assume the path is already internal, so leave it alone. - else: - chroot_image = image_file + sys.path.insert(0, chromeos_root) + + from chromite.lib import path_util - return chroot_image + return path_util.ToChrootPath(image_file, source_path=chromeos_root) def DoImage(argv): @@ -159,6 +142,13 @@ def DoImage(argv): "'quiet', 'average', and 'verbose'.", ) parser.add_argument("-a", "--image_args", dest="image_args") + parser.add_argument( + "--keep_stateful", + dest="keep_stateful", + default=False, + action="store_true", + help="Do not clobber the stateful partition.", + ) options = parser.parse_args(argv[1:]) @@ -278,9 +268,13 @@ def DoImage(argv): "cros", "flash", "--board=%s" % board, - "--clobber-stateful", - options.remote, ] + if not options.keep_stateful: + cros_flash_args.append("--clobber-stateful") + # New arguments should be added here. + + # The last two arguments are positional and have to be at the end. + cros_flash_args.append(options.remote) if local_image: cros_flash_args.append(chroot_image) else: diff --git a/llvm_tools/atomic_write_file.py b/llvm_tools/atomic_write_file.py new file mode 100644 index 00000000..aa6f112e --- /dev/null +++ b/llvm_tools/atomic_write_file.py @@ -0,0 +1,67 @@ +# Copyright 2023 The ChromiumOS Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Atomic file writing utilities. + +Provides atomic_write(...), which allows atomically replacing the contents +of a file. +""" + +import contextlib +import logging +import os +from pathlib import Path +import tempfile +from typing import Union + + +@contextlib.contextmanager +def atomic_write(fp: Union[Path, str], mode="w", *args, **kwargs): + """Write to a filepath atomically. + + This works by a temp file swap, created with a .tmp suffix in + the same directory briefly until being renamed to the desired + filepath. + + In the event an exception is raised during the write, the + temporary file is deleted and the original filepath is untouched. + + Examples: + >>> with atomic_write("my_file.txt", encoding="utf-8") as f: + >>> f.write("Hello world!") + >>> # my_file.txt is still unmodified + >>> # "f" is closed here, and my_file.txt is written to. + + Args: + fp: Filepath to open. + mode: File mode; can be 'w', 'wb'. Default 'w'. + *args: Passed to Path.open as nargs. + **kwargs: Passed to Path.open as kwargs. + + Raises: + ValueError when the mode is invalid. + """ + if isinstance(fp, str): + fp = Path(fp) + if mode not in ("w", "wb"): + raise ValueError(f"mode {mode} not accepted") + + # We use mkstemp here because we want to handle the closing and + # replacement ourselves. + (fd, tmp_path) = tempfile.mkstemp( + prefix=fp.name, + suffix=".tmp", + dir=fp.parent, + ) + tmp_path = Path(tmp_path) + try: + with os.fdopen(fd, mode=mode, *args, **kwargs) as f: + yield f + except: + try: + tmp_path.unlink() + except Exception as e: + logging.exception("unexpected error removing temporary file %s", e) + raise + tmp_path.replace(fp) diff --git a/llvm_tools/atomic_write_file_unittest.py b/llvm_tools/atomic_write_file_unittest.py new file mode 100644 index 00000000..78115569 --- /dev/null +++ b/llvm_tools/atomic_write_file_unittest.py @@ -0,0 +1,48 @@ +# Copyright 2023 The ChromiumOS Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from pathlib import Path +import tempfile +import unittest + +import atomic_write_file + + +class TestAtomicWrite(unittest.TestCase): + """Test atomic_write.""" + + def test_atomic_write(self): + """Test that atomic write safely writes.""" + prior_contents = "This is a test written by patch_utils_unittest.py\n" + new_contents = "I am a test written by patch_utils_unittest.py\n" + with tempfile.TemporaryDirectory( + prefix="patch_utils_unittest" + ) as dirname: + dirpath = Path(dirname) + filepath = dirpath / "test_atomic_write.txt" + with filepath.open("w", encoding="utf-8") as f: + f.write(prior_contents) + + def _t(): + with atomic_write_file.atomic_write( + filepath, encoding="utf-8" + ) as f: + f.write(new_contents) + raise Exception("Expected failure") + + self.assertRaises(Exception, _t) + with filepath.open(encoding="utf-8") as f: + lines = f.readlines() + self.assertEqual(lines[0], prior_contents) + with atomic_write_file.atomic_write( + filepath, encoding="utf-8" + ) as f: + f.write(new_contents) + with filepath.open(encoding="utf-8") as f: + lines = f.readlines() + self.assertEqual(lines[0], new_contents) + + +if __name__ == "__main__": + unittest.main() diff --git a/llvm_tools/auto_llvm_bisection.py b/llvm_tools/auto_llvm_bisection.py index 3640abae..9b4b8559 100755 --- a/llvm_tools/auto_llvm_bisection.py +++ b/llvm_tools/auto_llvm_bisection.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -49,10 +48,15 @@ class BuilderStatus(enum.Enum): RUNNING = "running" +# Writing a dict with `.value`s spelled out makes `black`'s style conflict with +# `cros lint`'s diagnostics. builder_status_mapping = { - BuilderStatus.PASS.value: update_tryjob_status.TryjobStatus.GOOD.value, - BuilderStatus.FAIL.value: update_tryjob_status.TryjobStatus.BAD.value, - BuilderStatus.RUNNING.value: update_tryjob_status.TryjobStatus.PENDING.value, + a.value: b.value + for a, b in ( + (BuilderStatus.PASS, update_tryjob_status.TryjobStatus.GOOD), + (BuilderStatus.FAIL, update_tryjob_status.TryjobStatus.BAD), + (BuilderStatus.RUNNING, update_tryjob_status.TryjobStatus.PENDING), + ) } @@ -63,8 +67,6 @@ def GetBuildResult(chroot_path, buildbucket_id): try: tryjob_json = subprocess.check_output( [ - "cros_sdk", - "--", "cros", "buildresult", "--buildbucket-id", @@ -98,13 +100,15 @@ def main(): """Bisects LLVM using the result of `cros buildresult` of each tryjob. Raises: - AssertionError: The script was run inside the chroot. + AssertionError: The script was run inside the chroot. """ chroot.VerifyOutsideChroot() args_output = llvm_bisection.GetCommandLineArgs() + chroot.VerifyChromeOSRoot(args_output.chroot_path) + if os.path.isfile(args_output.last_tested): print("Resuming bisection for %s" % args_output.last_tested) else: @@ -114,7 +118,7 @@ def main(): # Update the status of existing tryjobs if os.path.isfile(args_output.last_tested): update_start_time = time.time() - with open(args_output.last_tested) as json_file: + with open(args_output.last_tested, encoding="utf-8") as json_file: json_dict = json.load(json_file) while True: print( @@ -139,15 +143,16 @@ def main(): print("-" * 40) - # Proceed to the next step if all the existing tryjobs have completed. + # Proceed to the next step if all the existing tryjobs have + # completed. if completed: break delta_time = time.time() - update_start_time if delta_time > POLLING_LIMIT_SECS: - # Something is wrong with updating the tryjobs's 'status' via - # `cros buildresult` (e.g. network issue, etc.). + # Something is wrong with updating the tryjobs's 'status' + # via `cros buildresult` (e.g. network issue, etc.). sys.exit("Failed to update pending tryjobs.") print("-" * 40) @@ -157,7 +162,7 @@ def main(): # There should always be update from the tryjobs launched in the # last iteration. temp_filename = "%s.new" % args_output.last_tested - with open(temp_filename, "w") as temp_file: + with open(temp_filename, "w", encoding="utf-8") as temp_file: json.dump( json_dict, temp_file, indent=4, separators=(",", ": ") ) diff --git a/llvm_tools/auto_llvm_bisection_unittest.py b/llvm_tools/auto_llvm_bisection_unittest.py index c70ddee5..30e7dfb3 100755 --- a/llvm_tools/auto_llvm_bisection_unittest.py +++ b/llvm_tools/auto_llvm_bisection_unittest.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -13,7 +12,7 @@ import subprocess import time import traceback import unittest -import unittest.mock as mock +from unittest import mock import auto_llvm_bisection import chroot @@ -25,6 +24,7 @@ import update_tryjob_status class AutoLLVMBisectionTest(unittest.TestCase): """Unittests for auto bisection of LLVM.""" + @mock.patch.object(chroot, "VerifyChromeOSRoot") @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) @mock.patch.object( llvm_bisection, @@ -53,6 +53,7 @@ class AutoLLVMBisectionTest(unittest.TestCase): mock_sleep, mock_get_args, mock_outside_chroot, + mock_chromeos_root, ): mock_isfile.side_effect = [False, False, True, True] @@ -76,8 +77,8 @@ class AutoLLVMBisectionTest(unittest.TestCase): update_tryjob_status.TryjobStatus.GOOD.value ) - # Verify the excpetion is raised when successfully found the bad revision. - # Uses `sys.exit(0)` to indicate success. + # Verify the excpetion is raised when successfully found the bad + # revision. Uses `sys.exit(0)` to indicate success. with self.assertRaises(SystemExit) as err: auto_llvm_bisection.main() @@ -90,6 +91,7 @@ class AutoLLVMBisectionTest(unittest.TestCase): mock_traceback.assert_called_once() mock_sleep.assert_called_once() + @mock.patch.object(chroot, "VerifyChromeOSRoot") @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) @mock.patch.object(time, "sleep") @mock.patch.object(traceback, "print_exc") @@ -108,6 +110,7 @@ class AutoLLVMBisectionTest(unittest.TestCase): mock_traceback, mock_sleep, mock_outside_chroot, + mock_chromeos_root, ): mock_isfile.return_value = False @@ -123,6 +126,7 @@ class AutoLLVMBisectionTest(unittest.TestCase): self.assertEqual(err.exception.code, "Unable to continue bisection.") + mock_chromeos_root.assert_called_once() mock_outside_chroot.assert_called_once() mock_get_args.assert_called_once() self.assertEqual(mock_isfile.call_count, 2) @@ -130,6 +134,7 @@ class AutoLLVMBisectionTest(unittest.TestCase): self.assertEqual(mock_traceback.call_count, 3) self.assertEqual(mock_sleep.call_count, 2) + @mock.patch.object(chroot, "VerifyChromeOSRoot") @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) @mock.patch.object( llvm_bisection, @@ -153,6 +158,7 @@ class AutoLLVMBisectionTest(unittest.TestCase): mock_time, mock_get_args, mock_outside_chroot, + mock_chromeos_root, ): # Simulate behavior of `time.time()` for time passed. @@ -210,8 +216,6 @@ class AutoLLVMBisectionTest(unittest.TestCase): mock_chroot_command.assert_called_once_with( [ - "cros_sdk", - "--", "cros", "buildresult", "--buildbucket-id", @@ -234,8 +238,6 @@ class AutoLLVMBisectionTest(unittest.TestCase): auto_llvm_bisection.GetBuildResult(chroot_path, buildbucket_id) mock_chroot_command.assert_called_once_with( [ - "cros_sdk", - "--", "cros", "buildresult", "--buildbucket-id", @@ -258,8 +260,8 @@ class AutoLLVMBisectionTest(unittest.TestCase): tryjob_contents = {buildbucket_id: {"status": invalid_build_status}} mock_chroot_command.return_value = json.dumps(tryjob_contents) - # Verify the exception is raised when the return value of `cros buildresult` - # is not in the `builder_status_mapping`. + # Verify an exception is raised when the return value of `cros + # buildresult` is not in the `builder_status_mapping`. with self.assertRaises(ValueError) as err: auto_llvm_bisection.GetBuildResult(chroot_path, buildbucket_id) @@ -271,8 +273,6 @@ class AutoLLVMBisectionTest(unittest.TestCase): mock_chroot_command.assert_called_once_with( [ - "cros_sdk", - "--", "cros", "buildresult", "--buildbucket-id", diff --git a/llvm_tools/bisect_clang_crashes.py b/llvm_tools/bisect_clang_crashes.py deleted file mode 100755 index b2759051..00000000 --- a/llvm_tools/bisect_clang_crashes.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2020 The ChromiumOS Authors -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Fetches and submits the artifacts from ChromeOS toolchain's crash bucket. -""" - -import argparse -import glob -import json -import logging -import os -import os.path -import shutil -import subprocess -import sys - -import chroot - - -def get_artifacts(pattern): - results = subprocess.check_output( - ["gsutil.py", "ls", pattern], stderr=subprocess.STDOUT, encoding="utf-8" - ) - return sorted(l.strip() for l in results.splitlines()) - - -def get_crash_reproducers(working_dir): - results = [] - for src in [ - f - for f in glob.glob("%s/*.c*" % working_dir) - if f.split(".")[-1] in ["c", "cc", "cpp"] - ]: - script = ".".join(src.split(".")[:-1]) + ".sh" - if not os.path.exists(script): - logging.warning("could not find the matching script of %s", src) - else: - results.append((src, script)) - return results - - -def submit_crash_to_forcey( - forcey: str, temporary_directory: str, buildbucket_id: str, url: str -) -> None: - dest_dir = os.path.join(temporary_directory, buildbucket_id) - dest_file = os.path.join(dest_dir, os.path.basename(url)) - logging.info("Downloading and submitting %r...", url) - subprocess.check_output( - ["gsutil.py", "cp", url, dest_file], stderr=subprocess.STDOUT - ) - subprocess.check_output(["tar", "-xJf", dest_file], cwd=dest_dir) - for src, script in get_crash_reproducers(dest_dir): - subprocess.check_output( - [ - forcey, - "reduce", - "-wait=false", - "-note", - "%s:%s" % (url, src), - "-sh_file", - script, - "-src_file", - src, - ] - ) - - -def main(argv): - chroot.VerifyOutsideChroot() - logging.basicConfig( - format="%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s", - level=logging.INFO, - ) - cur_dir = os.path.dirname(os.path.abspath(__file__)) - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--4c", dest="forcey", required=True, help="Path to a 4c client binary" - ) - parser.add_argument( - "--state_file", - default=os.path.join(cur_dir, "chromeos-state.json"), - help="The path to the state file.", - ) - parser.add_argument( - "--nocleanup", - action="store_false", - dest="cleanup", - help="Keep temporary files created after the script finishes.", - ) - opts = parser.parse_args(argv) - - state_file = os.path.abspath(opts.state_file) - os.makedirs(os.path.dirname(state_file), exist_ok=True) - temporary_directory = "/tmp/bisect_clang_crashes" - os.makedirs(temporary_directory, exist_ok=True) - urls = get_artifacts( - "gs://chromeos-toolchain-artifacts/clang-crash-diagnoses" - "/**/*clang_crash_diagnoses.tar.xz" - ) - logging.info("%d crash URLs found", len(urls)) - - visited = {} - if os.path.exists(state_file): - buildbucket_ids = {url.split("/")[-2] for url in urls} - with open(state_file, encoding="utf-8") as f: - data = json.load(f) - visited = {k: v for k, v in data.items() if k in buildbucket_ids} - logging.info( - "Successfully loaded %d previously-submitted crashes", len(visited) - ) - - try: - for url in urls: - splits = url.split("/") - buildbucket_id = splits[-2] - # Skip the builds that has been processed - if buildbucket_id in visited: - continue - submit_crash_to_forcey( - forcey=opts.forcey, - temporary_directory=temporary_directory, - buildbucket_id=buildbucket_id, - url=url, - ) - visited[buildbucket_id] = url - - exception_in_flight = False - except: - exception_in_flight = True - raise - finally: - if exception_in_flight: - # This is best-effort. If the machine powers off or similar, we'll just - # resubmit the same crashes, which is suboptimal, but otherwise - # acceptable. - logging.error( - "Something went wrong; attempting to save our work..." - ) - else: - logging.info("Persisting state...") - - tmp_state_file = state_file + ".tmp" - with open(tmp_state_file, "w", encoding="utf-8") as f: - json.dump(visited, f, indent=2) - os.rename(tmp_state_file, state_file) - - logging.info("State successfully persisted") - - if opts.cleanup: - shutil.rmtree(temporary_directory) - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/llvm_tools/bisect_clang_crashes_unittest.py b/llvm_tools/bisect_clang_crashes_unittest.py deleted file mode 100755 index 22c9be19..00000000 --- a/llvm_tools/bisect_clang_crashes_unittest.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2020 The ChromiumOS Authors -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Tests for bisect_clang_crashes.""" - -import glob -import logging -import os.path -import subprocess -import unittest -import unittest.mock as mock - -import bisect_clang_crashes - - -class Test(unittest.TestCase): - """Tests for bisect_clang_crashes.""" - - class _SilencingFilter(object): - """Silences all log messages. - - Also collects info about log messages that would've been emitted. - """ - - def __init__(self): - self.messages = [] - - def filter(self, record): - self.messages.append(record.getMessage()) - return 0 - - @mock.patch.object(subprocess, "check_output") - def test_get_artifacts(self, mock_gsutil_ls): - pattern = ( - "gs://chromeos-toolchain-artifacts/clang-crash-diagnoses/" - "**/*clang_crash_diagnoses.tar.xz" - ) - mock_gsutil_ls.return_value = "artifact1\nartifact2\nartifact3" - results = bisect_clang_crashes.get_artifacts(pattern) - self.assertEqual(results, ["artifact1", "artifact2", "artifact3"]) - mock_gsutil_ls.assert_called_once_with( - ["gsutil.py", "ls", pattern], - stderr=subprocess.STDOUT, - encoding="utf-8", - ) - - @mock.patch.object(os.path, "exists") - @mock.patch.object(glob, "glob") - def test_get_crash_reproducers_succeed( - self, mock_file_search, mock_file_check - ): - working_dir = "SomeDirectory" - mock_file_search.return_value = ["a.c", "b.cpp", "c.cc"] - mock_file_check.side_effect = [True, True, True] - results = bisect_clang_crashes.get_crash_reproducers(working_dir) - mock_file_search.assert_called_once_with("%s/*.c*" % working_dir) - self.assertEqual(mock_file_check.call_count, 3) - self.assertEqual(mock_file_check.call_args_list[0], mock.call("a.sh")) - self.assertEqual(mock_file_check.call_args_list[1], mock.call("b.sh")) - self.assertEqual(mock_file_check.call_args_list[2], mock.call("c.sh")) - self.assertEqual( - results, [("a.c", "a.sh"), ("b.cpp", "b.sh"), ("c.cc", "c.sh")] - ) - - @mock.patch.object(os.path, "exists") - @mock.patch.object(glob, "glob") - def test_get_crash_reproducers_no_matching_script( - self, mock_file_search, mock_file_check - ): - def silence_logging(): - root = logging.getLogger() - filt = self._SilencingFilter() - root.addFilter(filt) - self.addCleanup(root.removeFilter, filt) - return filt - - log_filter = silence_logging() - working_dir = "SomeDirectory" - mock_file_search.return_value = ["a.c", "b.cpp", "c.cc"] - mock_file_check.side_effect = [True, False, True] - results = bisect_clang_crashes.get_crash_reproducers(working_dir) - mock_file_search.assert_called_once_with("%s/*.c*" % working_dir) - self.assertEqual(mock_file_check.call_count, 3) - self.assertEqual(mock_file_check.call_args_list[0], mock.call("a.sh")) - self.assertEqual(mock_file_check.call_args_list[1], mock.call("b.sh")) - self.assertEqual(mock_file_check.call_args_list[2], mock.call("c.sh")) - self.assertEqual(results, [("a.c", "a.sh"), ("c.cc", "c.sh")]) - self.assertTrue( - any( - "could not find the matching script of b.cpp" in x - for x in log_filter.messages - ), - log_filter.messages, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/llvm_tools/chroot.py b/llvm_tools/chroot.py index 46464feb..cb0ebc87 100755 --- a/llvm_tools/chroot.py +++ b/llvm_tools/chroot.py @@ -9,7 +9,9 @@ import collections import os +from pathlib import Path import subprocess +from typing import List CommitContents = collections.namedtuple("CommitContents", ["url", "cl_number"]) @@ -30,6 +32,19 @@ def VerifyOutsideChroot(): assert not InChroot(), "Script should be run outside the chroot." +def VerifyChromeOSRoot(chromeos_root): + """Checks whether the path actually points to ChromiumOS checkout root. + + Raises: + AssertionError: The path is not ChromiumOS checkout root. + """ + + subdir = "src/third_party/chromiumos-overlay" + path = Path(chromeos_root).expanduser() / subdir + msg = f"Wrong ChromeOS path. No {subdir} directory in {chromeos_root} ." + assert path.is_dir(), msg + + def GetChrootEbuildPaths(chromeos_root, packages): """Gets the chroot path(s) of the package(s). @@ -60,7 +75,10 @@ def GetChrootEbuildPaths(chromeos_root, packages): return chroot_paths -def ConvertChrootPathsToAbsolutePaths(chromeos_root, chroot_paths): +def ConvertChrootPathsToAbsolutePaths( + chromeos_root: str, + chroot_paths: List[str], +) -> List[str]: """Converts the chroot path(s) to absolute symlink path(s). Args: @@ -76,11 +94,8 @@ def ConvertChrootPathsToAbsolutePaths(chromeos_root, chroot_paths): """ abs_paths = [] - chroot_prefix = "/mnt/host/source/" - # Iterate through the chroot paths. - # # For each chroot file path, remove '/mnt/host/source/' prefix # and combine the chroot path with the result and add it to the list. for chroot_path in chroot_paths: @@ -88,12 +103,8 @@ def ConvertChrootPathsToAbsolutePaths(chromeos_root, chroot_paths): raise ValueError( "Invalid prefix for the chroot path: %s" % chroot_path ) - rel_path = chroot_path[len(chroot_prefix) :] - # combine the chromeos root path + '/src/...' abs_path = os.path.join(chromeos_root, rel_path) - abs_paths.append(abs_path) - return abs_paths diff --git a/llvm_tools/get_upstream_patch.py b/llvm_tools/get_upstream_patch.py index 72aa16b6..415faaa7 100755 --- a/llvm_tools/get_upstream_patch.py +++ b/llvm_tools/get_upstream_patch.py @@ -59,7 +59,7 @@ def validate_patch_application( if predecessor_apply_results.failed_patches: logging.error("Failed to apply patches from PATCHES.json:") for p in predecessor_apply_results.failed_patches: - logging.error(f"Patch title: {p.title()}") + logging.error("Patch title: %s", p.title()) raise PatchApplicationError("Failed to apply patch from PATCHES.json") patch_entry = patch_utils.PatchEntry.from_dict( @@ -126,7 +126,9 @@ def add_patch( ) with open(patches_json_path, encoding="utf-8") as f: - patches_json = json.load(f) + contents = f.read() + indent_len = patch_utils.predict_indent(contents.splitlines()) + patches_json = json.loads(contents) for p in patches_json: rel_path = p["rel_patch_path"] @@ -182,7 +184,11 @@ def add_patch( temp_file = patches_json_path + ".tmp" with open(temp_file, "w", encoding="utf-8") as f: json.dump( - patches_json, f, indent=4, separators=(",", ": "), sort_keys=True + patches_json, + f, + indent=indent_len, + separators=(",", ": "), + sort_keys=True, ) f.write("\n") os.rename(temp_file, patches_json_path) @@ -327,6 +333,7 @@ def find_patches_and_make_cl( start_rev: git_llvm_rev.Rev, llvm_config: git_llvm_rev.LLVMConfig, llvm_symlink_dir: str, + allow_failures: bool, create_cl: bool, skip_dependencies: bool, reviewers: t.Optional[t.List[str]], @@ -354,6 +361,8 @@ def find_patches_and_make_cl( if create_cl: git.CreateBranch(llvm_symlink_dir, branch) + successes = [] + failures = [] for parsed_patch in converted_patches: # Find out the llvm projects changed in this commit packages = get_package_names(parsed_patch.sha, llvm_config.dir) @@ -369,15 +378,25 @@ def find_patches_and_make_cl( chroot_path, symlinks ) # Create a local patch for all the affected llvm projects - create_patch_for_packages( - packages, - symlinks, - start_rev, - parsed_patch.rev, - parsed_patch.sha, - llvm_config.dir, - platforms=platforms, - ) + try: + create_patch_for_packages( + packages, + symlinks, + start_rev, + parsed_patch.rev, + parsed_patch.sha, + llvm_config.dir, + platforms=platforms, + ) + except PatchApplicationError as e: + if allow_failures: + logging.warning(e) + failures.append(parsed_patch.sha) + continue + else: + raise e + successes.append(parsed_patch.sha) + if create_cl: symlinks_to_uprev.extend(symlinks) @@ -397,7 +416,17 @@ def find_patches_and_make_cl( ["git", "reset", "--hard", "HEAD^"], cwd=llvm_config.dir ) - if create_cl: + if allow_failures: + success_list = (":\n\t" + "\n\t".join(successes)) if successes else "." + logging.info( + "Successfully applied %d patches%s", len(successes), success_list + ) + failure_list = (":\n\t" + "\n\t".join(failures)) if failures else "." + logging.info( + "Failed to apply %d patches%s", len(failures), failure_list + ) + + if successes and create_cl: make_cl( symlinks_to_uprev, llvm_symlink_dir, @@ -478,6 +507,7 @@ def get_from_upstream( start_sha: str, patches: t.List[str], platforms: t.List[str], + allow_failures: bool, skip_dependencies: bool = False, reviewers: t.List[str] = None, cc: t.List[str] = None, @@ -515,7 +545,9 @@ def get_from_upstream( skip_dependencies=skip_dependencies, reviewers=reviewers, cc=cc, + allow_failures=allow_failures, ) + logging.info("Complete.") @@ -565,6 +597,11 @@ def main(): "apply to multiple platforms", ) parser.add_argument( + "--allow_failures", + action="store_true", + help="Skip patches that fail to apply and continue.", + ) + parser.add_argument( "--create_cl", action="store_true", help="Automatically create a CL if specified", @@ -576,6 +613,7 @@ def main(): "when --differential appears exactly once.", ) args = parser.parse_args() + chroot.VerifyChromeOSRoot(args.chroot_path) if not (args.sha or args.differential): parser.error("--sha or --differential required") @@ -588,6 +626,7 @@ def main(): get_from_upstream( chroot_path=args.chroot_path, + allow_failures=args.allow_failures, create_cl=args.create_cl, start_sha=args.start_sha, patches=args.sha + args.differential, diff --git a/llvm_tools/git.py b/llvm_tools/git.py index 0f56aa0d..3bb702c9 100755 --- a/llvm_tools/git.py +++ b/llvm_tools/git.py @@ -65,14 +65,17 @@ def DeleteBranch(repo, branch): if not os.path.isdir(repo): raise ValueError("Invalid directory path provided: %s" % repo) - subprocess.check_output(["git", "-C", repo, "checkout", "cros/main"]) + def run_checked(cmd): + subprocess.run(["git", "-C", repo] + cmd, check=True) - subprocess.check_output(["git", "-C", repo, "reset", "HEAD", "--hard"]) - - subprocess.check_output(["git", "-C", repo, "branch", "-D", branch]) + run_checked(["checkout", "-q", "m/main"]) + run_checked(["reset", "-q", "HEAD", "--hard"]) + run_checked(["branch", "-q", "-D", branch]) -def UploadChanges(repo, branch, commit_messages, reviewers=None, cc=None): +def UploadChanges( + repo, branch, commit_messages, reviewers=None, cc=None, wip=False +): """Uploads the changes in the specifed branch of the given repo for review. Args: @@ -114,6 +117,8 @@ def UploadChanges(repo, branch, commit_messages, reviewers=None, cc=None): if cc: git_args.append(f'--cc={",".join(cc)}') + if wip: + git_args.append("--wip") out = subprocess.check_output( git_args, @@ -123,13 +128,11 @@ def UploadChanges(repo, branch, commit_messages, reviewers=None, cc=None): ) print(out) - + # Matches both internal and external CLs. found_url = re.search( - r"https://chromium-review.googlesource.com/c/" - r"chromiumos/overlays/chromiumos-overlay/\+/([0-9]+)", + r"https?://[\w-]*-review.googlesource.com/c/.*/([0-9]+)", out.rstrip(), ) - if not found_url: raise ValueError("Failed to find change list URL.") diff --git a/llvm_tools/git_llvm_rev.py b/llvm_tools/git_llvm_rev.py index 3dc34fce..1db94461 100755 --- a/llvm_tools/git_llvm_rev.py +++ b/llvm_tools/git_llvm_rev.py @@ -104,7 +104,7 @@ def translate_prebase_sha_to_rev_number( This function assumes that the given SHA is an ancestor of |base_llvm_sha|. """ commit_message = check_output( - ["git", "log", "-n1", "--format=%B", sha], + ["git", "log", "-n1", "--format=%B", sha, "--"], cwd=llvm_config.dir, ) last_line = commit_message.strip().splitlines()[-1] @@ -125,13 +125,13 @@ def translate_sha_to_rev(llvm_config: LLVMConfig, sha_or_ref: str) -> Rev: sha = sha_or_ref else: sha = check_output( - ["git", "rev-parse", sha_or_ref], + ["git", "rev-parse", "--revs-only", sha_or_ref, "--"], cwd=llvm_config.dir, ) sha = sha.strip() merge_base = check_output( - ["git", "merge-base", base_llvm_sha, sha], + ["git", "merge-base", base_llvm_sha, sha, "--"], cwd=llvm_config.dir, ) merge_base = merge_base.strip() @@ -144,6 +144,7 @@ def translate_sha_to_rev(llvm_config: LLVMConfig, sha_or_ref: str) -> Rev: "--count", "--first-parent", f"{base_llvm_sha}..{sha}", + "--", ], cwd=llvm_config.dir, ) @@ -167,6 +168,7 @@ def translate_sha_to_rev(llvm_config: LLVMConfig, sha_or_ref: str) -> Rev: "--count", "--first-parent", f"{merge_base}..{sha}", + "--", ], cwd=llvm_config.dir, ) @@ -263,23 +265,21 @@ def translate_prebase_rev_to_sha(llvm_config: LLVMConfig, rev: Rev) -> str: base_llvm_sha, ] - subp = subprocess.Popen( + with subprocess.Popen( git_command, cwd=llvm_config.dir, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, encoding="utf-8", - ) - - with subp: + ) as subp: for sha, message in parse_git_commit_messages(subp.stdout, separator): last_line = message.splitlines()[-1] if last_line.strip() == looking_for: subp.terminate() return sha + if subp.wait() != 0: + raise subprocess.CalledProcessError(subp.returncode, git_command) - if subp.returncode: - raise subprocess.CalledProcessError(subp.returncode, git_command) raise ValueError(f"No commit with revision {rev} found") @@ -317,7 +317,13 @@ def translate_rev_to_sha(llvm_config: LLVMConfig, rev: Rev) -> str: # about rev walking/counting locally compared to long |log|s, so we walk back # twice. head = check_output( - ["git", "rev-parse", f"{llvm_config.remote}/{branch}"], + [ + "git", + "rev-parse", + "--revs-only", + f"{llvm_config.remote}/{branch}", + "--", + ], cwd=llvm_config.dir, ) branch_head_sha = head.strip() @@ -330,6 +336,7 @@ def translate_rev_to_sha(llvm_config: LLVMConfig, rev: Rev) -> str: "--count", "--first-parent", f"{base_sha}..{branch_head_sha}", + "--", ], cwd=llvm_config.dir, ) diff --git a/llvm_tools/git_unittest.py b/llvm_tools/git_unittest.py index ce21e6c9..35ba2430 100755 --- a/llvm_tools/git_unittest.py +++ b/llvm_tools/git_unittest.py @@ -68,7 +68,7 @@ class HelperFunctionsTest(unittest.TestCase): mock_isdir.assert_called_once() @mock.patch.object(os.path, "isdir", return_value=True) - @mock.patch.object(subprocess, "check_output", return_value=None) + @mock.patch.object(subprocess, "run", return_value=None) def testSuccessfullyDeletedBranch(self, mock_command_output, mock_isdir): path_to_repo = "/valid/path/to/repo" branch = "branch-name" diff --git a/llvm_tools/llvm_bisection.py b/llvm_tools/llvm_bisection.py index 0b851ebe..6cc93aec 100755 --- a/llvm_tools/llvm_bisection.py +++ b/llvm_tools/llvm_bisection.py @@ -289,7 +289,6 @@ def Bisect( last_tested, update_packages, chroot_path, - patch_metadata_file, extra_change_lists, options, builder, @@ -304,7 +303,6 @@ def Bisect( git_hash, svn_revision, chroot_path, - patch_metadata_file, extra_change_lists, options, builder, @@ -346,7 +344,7 @@ def main(args_output): """ chroot.VerifyOutsideChroot() - patch_metadata_file = "PATCHES.json" + chroot.VerifyChromeOSRoot(args_output.chroot_path) start = args_output.start_rev end = args_output.end_rev @@ -453,7 +451,6 @@ def main(args_output): args_output.last_tested, update_chromeos_llvm_hash.DEFAULT_PACKAGES, args_output.chroot_path, - patch_metadata_file, args_output.extra_change_lists, args_output.options, args_output.builder, diff --git a/llvm_tools/llvm_bisection_unittest.py b/llvm_tools/llvm_bisection_unittest.py index 1e86a678..a2bcc90b 100755 --- a/llvm_tools/llvm_bisection_unittest.py +++ b/llvm_tools/llvm_bisection_unittest.py @@ -209,7 +209,6 @@ class LLVMBisectionTest(unittest.TestCase): _git_hash, _revision, _chroot_path, - _patch_file, _extra_cls, _options, _builder, @@ -238,7 +237,6 @@ class LLVMBisectionTest(unittest.TestCase): args_output = test_helpers.ArgsOutputTest() packages = ["sys-devel/llvm"] - patch_file = "/abs/path/to/PATCHES.json" # Create a temporary .JSON file to simulate a status file for bisection. with test_helpers.CreateTemporaryJsonFile() as temp_json_file: @@ -255,7 +253,6 @@ class LLVMBisectionTest(unittest.TestCase): temp_json_file, packages, args_output.chroot_path, - patch_file, args_output.extra_change_lists, args_output.options, args_output.builders, @@ -289,10 +286,12 @@ class LLVMBisectionTest(unittest.TestCase): @mock.patch.object(llvm_bisection, "GetCommitsBetween") @mock.patch.object(llvm_bisection, "GetRemainingRange") @mock.patch.object(llvm_bisection, "LoadStatusFile") + @mock.patch.object(chroot, "VerifyChromeOSRoot") @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) def testMainPassed( self, mock_outside_chroot, + mock_chromeos_root, mock_load_status_file, mock_get_range, mock_get_revision_and_hash_list, @@ -337,6 +336,8 @@ class LLVMBisectionTest(unittest.TestCase): llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value, ) + mock_chromeos_root.assert_called_once() + mock_outside_chroot.assert_called_once() mock_load_status_file.assert_called_once() @@ -362,9 +363,10 @@ class LLVMBisectionTest(unittest.TestCase): ) @mock.patch.object(llvm_bisection, "LoadStatusFile") + @mock.patch.object(chroot, "VerifyChromeOSRoot") @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) def testMainFailedWithInvalidRange( - self, mock_outside_chroot, mock_load_status_file + self, mock_chromeos_root, mock_outside_chroot, mock_load_status_file ): start = 500 @@ -394,6 +396,8 @@ class LLVMBisectionTest(unittest.TestCase): self.assertEqual(str(err.exception), error_message) + mock_chromeos_root.assert_called_once() + mock_outside_chroot.assert_called_once() mock_load_status_file.assert_called_once() @@ -401,9 +405,11 @@ class LLVMBisectionTest(unittest.TestCase): @mock.patch.object(llvm_bisection, "GetCommitsBetween") @mock.patch.object(llvm_bisection, "GetRemainingRange") @mock.patch.object(llvm_bisection, "LoadStatusFile") + @mock.patch.object(chroot, "VerifyChromeOSRoot") @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) def testMainFailedWithPendingBuilds( self, + mock_chromeos_root, mock_outside_chroot, mock_load_status_file, mock_get_range, @@ -451,6 +457,8 @@ class LLVMBisectionTest(unittest.TestCase): self.assertEqual(str(err.exception), error_message) + mock_chromeos_root.assert_called_once() + mock_outside_chroot.assert_called_once() mock_load_status_file.assert_called_once() @@ -462,10 +470,12 @@ class LLVMBisectionTest(unittest.TestCase): @mock.patch.object(llvm_bisection, "GetCommitsBetween") @mock.patch.object(llvm_bisection, "GetRemainingRange") @mock.patch.object(llvm_bisection, "LoadStatusFile") + @mock.patch.object(chroot, "VerifyChromeOSRoot") @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) def testMainFailedWithDuplicateBuilds( self, mock_outside_chroot, + mock_chromeos_root, mock_load_status_file, mock_get_range, mock_get_revision_and_hash_list, @@ -508,6 +518,8 @@ class LLVMBisectionTest(unittest.TestCase): error_message = 'Revision %d exists already in "jobs"' % rev self.assertEqual(str(err.exception), error_message) + mock_chromeos_root.assert_called_once() + mock_outside_chroot.assert_called_once() mock_load_status_file.assert_called_once() @@ -523,10 +535,12 @@ class LLVMBisectionTest(unittest.TestCase): @mock.patch.object(llvm_bisection, "GetCommitsBetween") @mock.patch.object(llvm_bisection, "GetRemainingRange") @mock.patch.object(llvm_bisection, "LoadStatusFile") + @mock.patch.object(chroot, "VerifyChromeOSRoot") @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) def testMainFailedToAbandonCL( self, mock_outside_chroot, + mock_chromeos_root, mock_load_status_file, mock_get_range, mock_get_revision_and_hash_list, @@ -574,6 +588,8 @@ class LLVMBisectionTest(unittest.TestCase): self.assertEqual(err.exception.output, error_message) + mock_chromeos_root.assert_called_once() + mock_outside_chroot.assert_called_once() mock_load_status_file.assert_called_once() diff --git a/llvm_tools/llvm_local_bisection.sh b/llvm_tools/llvm_local_bisection.sh index e319080c..0dde49d7 100755 --- a/llvm_tools/llvm_local_bisection.sh +++ b/llvm_tools/llvm_local_bisection.sh @@ -92,7 +92,7 @@ build_llvm () { build_pkg () { local pkg="${1}" - local logfile="/tmp/build-${pkg}.${CURRENT}.out" + local logfile="/tmp/build-${pkg//\//_}.${CURRENT}.out" log "Writing logs to ${logfile}" log "sudo emerge ${pkg}" logdo sudo emerge "${pkg}" \ diff --git a/llvm_tools/manifest_utils.py b/llvm_tools/manifest_utils.py new file mode 100644 index 00000000..67eae4f3 --- /dev/null +++ b/llvm_tools/manifest_utils.py @@ -0,0 +1,103 @@ +# Copyright 2023 The ChromiumOS Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Provides utilities to read and edit the ChromiumOS Manifest entries. + +While this code reads and edits the internal manifest, it should only operate +on toolchain projects (llvm-project, etc.) which are public. +""" + +from pathlib import Path +import shutil +import subprocess +from xml.etree import ElementTree + +import atomic_write_file + + +LLVM_PROJECT_PATH = "src/third_party/llvm-project" + + +class FormattingError(Exception): + """Error occurred when formatting the manifest.""" + + pass + + +class UpdateManifestError(Exception): + """Error occurred when updating the manifest.""" + + pass + + +def make_xmlparser(): + """Return a new xmlparser with custom TreeBuilder.""" + return ElementTree.XMLParser( + target=ElementTree.TreeBuilder(insert_comments=True) + ) + + +def update_chromeos_manifest(revision: str, src_tree: Path) -> Path: + """Replaces the manifest project revision with 'revision'. + + Notably, this function reformats the manifest file to preserve + the formatting as specified by 'cros format'. + + Args: + revision: Revision (git sha) to use in the manifest. + src_tree: Path to the root of the source tree checkout. + + Returns: + The manifest path. + + Post: + The llvm-project revision info in the chromeos repo manifest + is updated with 'revision'. + + Raises: + UpdateManifestError: The manifest could not be changed. + FormattingError: The manifest could not be reformatted. + """ + manifest_path = get_chromeos_manifest_path(src_tree) + parser = make_xmlparser() + xmltree = ElementTree.parse(manifest_path, parser) + update_chromeos_manifest_tree(revision, xmltree.getroot()) + with atomic_write_file.atomic_write(manifest_path, mode="wb") as f: + xmltree.write(f, encoding="UTF-8") + format_manifest(manifest_path) + return manifest_path + + +def get_chromeos_manifest_path(src_tree: Path) -> Path: + """Return the path to the toolchain manifest.""" + return src_tree / "manifest-internal" / "_toolchain.xml" + + +def update_chromeos_manifest_tree(revision: str, xmlroot: ElementTree.Element): + """Update the revision info for LLVM for a manifest XML root.""" + + # This exists mostly for testing. + def is_llvm_project(child): + return ( + child.tag == "project" and child.attrib["path"] == LLVM_PROJECT_PATH + ) + + finder = (child for child in xmlroot if is_llvm_project(child)) + llvm_project_elem = next(finder, None) + # Element objects can be falsy, so we need to explicitly check None. + if llvm_project_elem is not None: + # Update the llvm revision git sha + llvm_project_elem.attrib["revision"] = revision + else: + raise UpdateManifestError("xmltree did not have llvm-project") + + +def format_manifest(repo_manifest: Path): + """Use cros format to format the given manifest.""" + if not shutil.which("cros"): + raise FormattingError( + "unable to format manifest, 'cros'" " executable not in PATH" + ) + cmd = ["cros", "format", repo_manifest] + subprocess.run(cmd, check=True) diff --git a/llvm_tools/manifest_utils_unittest.py b/llvm_tools/manifest_utils_unittest.py new file mode 100644 index 00000000..9a21d990 --- /dev/null +++ b/llvm_tools/manifest_utils_unittest.py @@ -0,0 +1,84 @@ +# Copyright 2023 The ChromiumOS Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import io +from pathlib import Path +import re +import unittest +from xml.etree import ElementTree + +import manifest_utils + + +"""Provides utilities to read and edit the ChromiumOS Manifest entries. + +While this code reads and edits the internal manifest, it should only operate +on toolchain projects (llvm-project, etc.) which are public. +""" + +MANIFEST_FIXTURE = """<?xml version="1.0" encoding="UTF-8"?> +<manifest> + <!-- Comment that should not be removed. + Multiple lines. --> + <include name="_remotes.xml" /> + <default revision="refs/heads/main" + remote="cros" + sync-j="8" /> + + <include name="_kernel_upstream.xml" /> + + <!-- Common projects for developing CrOS. --> + <project path="src/repohooks" + name="chromiumos/repohooks" + groups="minilayout,paygen,firmware,buildtools,labtools,crosvm" /> + <repo-hooks in-project="chromiumos/repohooks" + enabled-list="pre-upload" /> + <project path="chromite" + name="chromiumos/chromite" + groups="minilayout,paygen,firmware,buildtools,chromeos-admin"> + <copyfile src="AUTHORS" dest="AUTHORS" /> + <copyfile src="LICENSE" dest="LICENSE" /> + </project> + <project path="src/third_party/llvm-project" + name="external/github.com/llvm/llvm-project" + groups="notdefault,bazel" + revision="abcd" /> + <project path="chromite/third_party/pyelftools" + name="chromiumos/third_party/pyelftools" + revision="refs/heads/chromeos-0.22" + groups="minilayout,paygen,firmware,buildtools" /> +</manifest> +""" + + +class TestManifestUtils(unittest.TestCase): + """Test manifest_utils.""" + + def test_update_chromeos_manifest(self): + root = ElementTree.fromstring( + MANIFEST_FIXTURE, + parser=manifest_utils.make_xmlparser(), + ) + manifest_utils.update_chromeos_manifest_tree("wxyz", root) + string_root1 = ElementTree.tostring(root) + self.assertRegex( + str(string_root1, encoding="utf-8"), + r'revision="wxyz"', + ) + self.assertRegex( + str(string_root1, encoding="utf-8"), + r"<!-- Comment that should not be removed.", + ) + self.assertNotRegex( + str(string_root1, encoding="utf-8"), + r'revision="abcd"', + ) + # Check idempotence. + manifest_utils.update_chromeos_manifest_tree("wxyz", root) + string_root2 = ElementTree.tostring(root) + self.assertEqual(string_root1, string_root2) + + +if __name__ == "__main__": + unittest.main() diff --git a/llvm_tools/modify_a_tryjob.py b/llvm_tools/modify_a_tryjob.py index 03de606d..2ecca800 100755 --- a/llvm_tools/modify_a_tryjob.py +++ b/llvm_tools/modify_a_tryjob.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -11,6 +10,7 @@ import argparse import enum import json import os +from pathlib import Path import sys import chroot @@ -45,11 +45,12 @@ def GetCommandLineArgs(): parser.add_argument( "--status_file", required=True, - help="The absolute path to the JSON file that contains the tryjobs used " - "for bisecting LLVM.", + help="The absolute path to the JSON file that contains the tryjobs " + "used for bisecting LLVM.", ) - # Add argument that determines what action to take on the revision specified. + # Add argument that determines what action to take on the revision + # specified. parser.add_argument( "--modify_tryjob", required=True, @@ -66,7 +67,8 @@ def GetCommandLineArgs(): help="The revision to either remove or relaunch.", ) - # Add argument for other change lists that want to run alongside the tryjob. + # Add argument for other change lists that want to run alongside the + # tryjob. parser.add_argument( "--extra_change_lists", type=int, @@ -134,7 +136,6 @@ def GetCLAfterUpdatingPackages( git_hash, svn_version, chroot_path, - patch_metadata_file, svn_option, ): """Updates the packages' LLVM_NEXT.""" @@ -145,7 +146,7 @@ def GetCLAfterUpdatingPackages( llvm_variant=update_chromeos_llvm_hash.LLVMVariant.next, git_hash=git_hash, svn_version=svn_version, - chroot_path=chroot_path, + chroot_path=Path(chroot_path), mode=failure_modes.FailureModes.DISABLE_PATCHES, git_hash_source=svn_option, extra_commit_msg=None, @@ -196,7 +197,6 @@ def AddTryjob( git_hash, revision, chroot_path, - patch_metadata_file, extra_cls, options, builder, @@ -212,7 +212,6 @@ def AddTryjob( git_hash, revision, chroot_path, - patch_metadata_file, svn_option, ) @@ -242,16 +241,17 @@ def PerformTryjobModification( """Removes, relaunches, or adds a tryjob. Args: - revision: The revision associated with the tryjob. - modify_tryjob: What action to take on the tryjob. - Ex: ModifyTryjob.REMOVE, ModifyTryjob.RELAUNCH, ModifyTryjob.ADD - status_file: The .JSON file that contains the tryjobs. - extra_cls: Extra change lists to be run alongside tryjob - options: Extra options to pass into 'cros tryjob'. - builder: The builder to use for 'cros tryjob'. - chroot_path: The absolute path to the chroot (used by 'cros tryjob' when - relaunching a tryjob). - verbose: Determines whether to print the contents of a command to `stdout`. + revision: The revision associated with the tryjob. + modify_tryjob: What action to take on the tryjob. + Ex: ModifyTryjob.REMOVE, ModifyTryjob.RELAUNCH, ModifyTryjob.ADD + status_file: The .JSON file that contains the tryjobs. + extra_cls: Extra change lists to be run alongside tryjob + options: Extra options to pass into 'cros tryjob'. + builder: The builder to use for 'cros tryjob'. + chroot_path: The absolute path to the chroot (used by 'cros tryjob' + when relaunching a tryjob). + verbose: Determines whether to print the contents of a command to + `stdout`. """ # Format of 'bisect_contents': @@ -265,7 +265,7 @@ def PerformTryjobModification( # {[TRYJOB_INFORMATION]} # ] # } - with open(status_file) as tryjobs: + with open(status_file, encoding="utf-8") as tryjobs: bisect_contents = json.load(tryjobs) if not bisect_contents["jobs"] and modify_tryjob != ModifyTryjob.ADD: @@ -318,12 +318,10 @@ def PerformTryjobModification( % (tryjob_index, status_file) ) - # Make sure the revision is within the bounds of the start and end of the - # bisection. + # Make sure the revision is within the bounds of the start and end of + # the bisection. elif bisect_contents["start"] < revision < bisect_contents["end"]: - patch_metadata_file = "PATCHES.json" - ( git_hash, revision, @@ -334,7 +332,6 @@ def PerformTryjobModification( git_hash, revision, chroot_path, - patch_metadata_file, extra_cls, options, builder, @@ -352,7 +349,7 @@ def PerformTryjobModification( 'Invalid "modify_tryjob" option provided: %s' % modify_tryjob ) - with open(status_file, "w") as update_tryjobs: + with open(status_file, "w", encoding="utf-8") as update_tryjobs: json.dump( bisect_contents, update_tryjobs, indent=4, separators=(",", ": ") ) @@ -365,6 +362,8 @@ def main(): args_output = GetCommandLineArgs() + chroot.VerifyChromeOSRoot(args_output.chroot_path) + PerformTryjobModification( args_output.revision, ModifyTryjob(args_output.modify_tryjob), diff --git a/llvm_tools/patch_manager_unittest.py b/llvm_tools/patch_manager_unittest.py index 42697d91..91573a82 100755 --- a/llvm_tools/patch_manager_unittest.py +++ b/llvm_tools/patch_manager_unittest.py @@ -12,6 +12,7 @@ from typing import Callable import unittest from unittest import mock +import atomic_write_file import patch_manager import patch_utils @@ -103,7 +104,9 @@ class PatchManagerTest(unittest.TestCase): ), ] patches_path = dirpath / "PATCHES.json" - with patch_utils.atomic_write(patches_path, encoding="utf-8") as f: + with atomic_write_file.atomic_write( + patches_path, encoding="utf-8" + ) as f: json.dump([pe.to_dict() for pe in patch_entries], f) def _harness1( diff --git a/llvm_tools/patch_sync/src/patch_parsing.rs b/llvm_tools/patch_sync/src/patch_parsing.rs index 00153834..7f545e5b 100644 --- a/llvm_tools/patch_sync/src/patch_parsing.rs +++ b/llvm_tools/patch_sync/src/patch_parsing.rs @@ -3,11 +3,11 @@ // found in the LICENSE file. use std::collections::{BTreeMap, BTreeSet}; -use std::fs::{copy, File}; +use std::fs::{copy, read_to_string, File}; use std::io::{BufRead, BufReader, Read, Write}; use std::path::{Path, PathBuf}; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -45,18 +45,22 @@ impl PatchDictSchema { pub struct PatchCollection { pub patches: Vec<PatchDictSchema>, pub workdir: PathBuf, + pub indent_len: usize, } impl PatchCollection { /// Create a `PatchCollection` from a PATCHES. pub fn parse_from_file(json_file: &Path) -> Result<Self> { - Ok(Self { - patches: serde_json::from_reader(File::open(json_file)?)?, - workdir: json_file + // We can't just use a file reader because we + // need to know what the original indent_len is. + let contents = read_to_string(json_file)?; + Self::parse_from_str( + json_file .parent() .ok_or_else(|| anyhow!("failed to get json_file parent"))? .to_path_buf(), - }) + &contents, + ) } /// Create a `PatchCollection` from a string literal and a workdir. @@ -64,6 +68,7 @@ impl PatchCollection { Ok(Self { patches: serde_json::from_str(contents).context("parsing from str")?, workdir, + indent_len: predict_indent(contents), }) } @@ -72,6 +77,7 @@ impl PatchCollection { Self { patches: self.patches.iter().cloned().filter(f).collect(), workdir: self.workdir.clone(), + indent_len: self.indent_len, } } @@ -80,6 +86,7 @@ impl PatchCollection { Self { patches: self.patches.iter().map(f).collect(), workdir: self.workdir.clone(), + indent_len: self.indent_len, } } @@ -89,7 +96,7 @@ impl PatchCollection { } /// Compute the set-set subtraction, returning a new `PatchCollection` which - /// keeps the minuend's workdir. + /// keeps the minuend's workdir and indent level. pub fn subtract(&self, subtrahend: &Self) -> Result<Self> { let mut new_patches = Vec::new(); // This is O(n^2) when it could be much faster, but n is always going to be less @@ -111,6 +118,7 @@ impl PatchCollection { Ok(Self { patches: new_patches, workdir: self.workdir.clone(), + indent_len: self.indent_len, }) } @@ -173,6 +181,7 @@ impl PatchCollection { .collect(); Self { workdir: self.workdir.clone(), + indent_len: self.indent_len, patches: cloned_patches, } } @@ -236,6 +245,7 @@ impl PatchCollection { Ok(Self { workdir: self.workdir.clone(), + indent_len: self.indent_len, patches: combined_patches, }) } @@ -262,11 +272,12 @@ impl PatchCollection { } pub fn serialize_patches(&self) -> Result<String> { + let indent_str = " ".repeat(self.indent_len); let mut serialization_buffer = Vec::<u8>::new(); // Four spaces to indent json serialization. let mut serializer = serde_json::Serializer::with_formatter( &mut serialization_buffer, - serde_json::ser::PrettyFormatter::with_indent(b" "), + serde_json::ser::PrettyFormatter::with_indent(indent_str.as_bytes()), ); self.patches .serialize(&mut serializer) @@ -329,8 +340,7 @@ pub fn new_patches( // Set up the current patch collection. let cur_collection = PatchCollection::parse_from_file(patches_path) .with_context(|| format!("parsing {} PATCHES.json", platform))?; - let cur_collection = filter_patches_by_platform(&cur_collection, platform); - let cur_collection = cur_collection.filter_patches(|p| cur_collection.patch_exists(p)); + validate_patches(&cur_collection, platform)?; // Set up the old patch collection. let old_collection = PatchCollection::parse_from_str( @@ -367,6 +377,25 @@ pub fn filter_patches_by_platform(collection: &PatchCollection, platform: &str) }) } +/// Verify the patches all exist and apply to the given platform. +/// +/// If all good, return Unit. Otherwise, return an Err. +pub fn validate_patches(collection: &PatchCollection, platform: &str) -> Result<()> { + for p in &collection.patches { + if !collection.patch_exists(p) { + bail!("Patch {} does not exist", p.rel_patch_path); + } + if !p.platforms.is_empty() && !p.platforms.contains(platform) { + bail!( + "Patch {} did not apply to platform {}", + p.rel_patch_path, + platform + ); + } + } + Ok(()) +} + /// Get the hash from the patch file contents. /// /// Not every patch file actually contains its own hash, @@ -413,6 +442,24 @@ fn copy_create_parents(from: &Path, to: &Path) -> Result<()> { Ok(()) } +/// Given a json string, predict and return the maximum indentation unit the json string uses. +pub fn predict_indent(json: &str) -> usize { + let indents = json + .split('\n') + .map(|line| line.len() - line.trim_start_matches(' ').len()) + .collect::<Vec<usize>>(); + if indents.iter().all(|x| x % 4 == 0) { + return 4; + } + if indents.iter().all(|x| x % 2 == 0) { + return 2; + } + if indents.iter().all(|x| *x == 0) { + return 0; + } + 1 +} + #[cfg(test)] mod test { @@ -444,6 +491,40 @@ mod test { } #[test] + fn test_keep_indent2() { + let example_json = "\ +[ + { + \"rel_patch_path\": \"some_patch.\", + \"metadata\": null, + \"platforms\": [] + } +] +"; + let collection1 = PatchCollection::parse_from_str(PathBuf::new(), example_json) + .expect("could not parse example_json"); + assert_eq!(collection1.indent_len, 2); + let collection2 = PatchCollection::parse_from_str( + PathBuf::new(), + &collection1 + .serialize_patches() + .expect("expected to serialize patches"), + ) + .expect("could not parse from serialization"); + assert_eq!(collection2.indent_len, 2); + let mut collection3 = collection1; + collection3.indent_len = 4; + let collection4 = PatchCollection::parse_from_str( + PathBuf::new(), + &collection3 + .serialize_patches() + .expect("expected to serialize patches"), + ) + .expect("could not parse from serialization"); + assert_eq!(collection4.indent_len, 4) + } + + #[test] fn test_union() { let patch1 = PatchDictSchema { rel_patch_path: "a".into(), @@ -466,10 +547,12 @@ mod test { let collection1 = PatchCollection { workdir: PathBuf::new(), patches: vec![patch1, patch2], + indent_len: 0, }; let collection2 = PatchCollection { workdir: PathBuf::new(), patches: vec![patch3], + indent_len: 0, }; let union = collection1 .union_helper( @@ -503,10 +586,12 @@ mod test { let collection1 = PatchCollection { workdir: PathBuf::new(), patches: vec![patch1.clone()], + indent_len: 4, }; let collection2 = PatchCollection { workdir: PathBuf::new(), patches: vec![patch1], + indent_len: 4, }; let union = collection1 .union_helper( @@ -577,14 +662,17 @@ mod test { let collection1 = PatchCollection { workdir: PathBuf::new(), patches: vec![patch1, patch2.clone()], + indent_len: 0, }; let collection2 = PatchCollection { workdir: PathBuf::new(), patches: vec![patch1_updated, patch2.clone()], + indent_len: 0, }; let collection3 = PatchCollection { workdir: PathBuf::new(), patches: vec![patch2], + indent_len: 0, }; vec![collection1, collection2, collection3] } diff --git a/llvm_tools/patch_utils.py b/llvm_tools/patch_utils.py index affb3d0d..0c8ad19c 100644 --- a/llvm_tools/patch_utils.py +++ b/llvm_tools/patch_utils.py @@ -12,7 +12,9 @@ from pathlib import Path import re import subprocess import sys -from typing import Any, Dict, IO, Iterable, List, Optional, Tuple, Union +from typing import Any, Dict, IO, Iterable, List, Optional, Tuple + +import atomic_write_file CHECKED_FILE_RE = re.compile(r"^checking file\s+(.*)$") @@ -22,38 +24,6 @@ HUNK_END_RE = re.compile(r"^--\s*$") PATCH_SUBFILE_HEADER_RE = re.compile(r"^\+\+\+ [ab]/(.*)$") -@contextlib.contextmanager -def atomic_write(fp: Union[Path, str], mode="w", *args, **kwargs): - """Write to a filepath atomically. - - This works by a temp file swap, created with a .tmp suffix in - the same directory briefly until being renamed to the desired - filepath. - - Args: - fp: Filepath to open. - mode: File mode; can be 'w', 'wb'. Default 'w'. - *args: Passed to Path.open as nargs. - **kwargs: Passed to Path.open as kwargs. - - Raises: - ValueError when the mode is invalid. - """ - if isinstance(fp, str): - fp = Path(fp) - if mode not in ("w", "wb"): - raise ValueError(f"mode {mode} not accepted") - temp_fp = fp.with_suffix(fp.suffix + ".tmp") - try: - with temp_fp.open(mode, *args, **kwargs) as f: - yield f - except: - if temp_fp.is_file(): - temp_fp.unlink() - raise - temp_fp.rename(fp) - - @dataclasses.dataclass class Hunk: """Represents a patch Hunk.""" @@ -260,6 +230,7 @@ class PatchEntry: "-d", root_dir.absolute(), "-f", + "-E", "-p1", "--no-backup-if-mismatch", "-i", @@ -324,6 +295,16 @@ def json_to_patch_entries(workdir: Path, json_fd: IO[str]) -> List[PatchEntry]: return [PatchEntry.from_dict(workdir, d) for d in json.load(json_fd)] +def json_str_to_patch_entries(workdir: Path, json_str: str) -> List[PatchEntry]: + """Convert a json IO object to List[PatchEntry]. + + Examples: + >>> f = open('PATCHES.json').read() + >>> patch_entries = json_str_to_patch_entries(Path(), f) + """ + return [PatchEntry.from_dict(workdir, d) for d in json.loads(json_str)] + + def _print_failed_patch(pe: PatchEntry, failed_hunks: Dict[str, List[Hunk]]): """Print information about a single failing PatchEntry. @@ -462,13 +443,27 @@ def git_clean_context(git_root_dir: Path): clean_src_tree(git_root_dir) -def _write_json_changes(patches: List[Dict[str, Any]], file_io: IO[str]): +def _write_json_changes( + patches: List[Dict[str, Any]], file_io: IO[str], indent_len=2 +): """Write JSON changes to file, does not acquire new file lock.""" - json.dump(patches, file_io, indent=4, separators=(",", ": ")) + json.dump(patches, file_io, indent=indent_len, separators=(",", ": ")) # Need to add a newline as json.dump omits it. file_io.write("\n") +def predict_indent(patches_lines: List[str]) -> int: + """Given file lines, predict and return the max indentation unit.""" + indents = [len(x) - len(x.lstrip(" ")) for x in patches_lines] + if all(x % 4 == 0 for x in indents): + return 4 + if all(x % 2 == 0 for x in indents): + return 2 + if all(x == 0 for x in indents): + return 0 + return 1 + + def update_version_ranges( svn_version: int, llvm_src_dir: Path, patches_json_fp: Path ) -> PatchInfo: @@ -488,15 +483,19 @@ def update_version_ranges( PatchInfo for applied and disabled patches. """ with patches_json_fp.open(encoding="utf-8") as f: - patch_entries = json_to_patch_entries( - patches_json_fp.parent, - f, - ) + contents = f.read() + indent_len = predict_indent(contents.splitlines()) + patch_entries = json_str_to_patch_entries( + patches_json_fp.parent, + contents, + ) modified_entries, applied_patches = update_version_ranges_with_entries( svn_version, llvm_src_dir, patch_entries ) - with atomic_write(patches_json_fp, encoding="utf-8") as f: - _write_json_changes([p.to_dict() for p in patch_entries], f) + with atomic_write_file.atomic_write(patches_json_fp, encoding="utf-8") as f: + _write_json_changes( + [p.to_dict() for p in patch_entries], f, indent_len=indent_len + ) for entry in modified_entries: print( f"Stopped applying {entry.rel_patch_path} ({entry.title()}) " @@ -532,7 +531,9 @@ def update_version_ranges_with_entries( """ modified_entries: List[PatchEntry] = [] applied_patches: List[PatchEntry] = [] - active_patches = (pe for pe in patch_entries if not pe.is_old(svn_version)) + active_patches = ( + pe for pe in patch_entries if pe.can_patch_version(svn_version) + ) with git_clean_context(llvm_src_dir): for pe in active_patches: test_result = pe.test_apply(llvm_src_dir) @@ -570,14 +571,16 @@ def remove_old_patches( PatchInfo for modified patches. """ with patches_json_fp.open(encoding="utf-8") as f: - patches_list = json.load(f) - patch_entries = ( - PatchEntry.from_dict(llvm_src_dir, elem) for elem in patches_list + contents = f.read() + indent_len = predict_indent(contents.splitlines()) + patch_entries = json_str_to_patch_entries( + llvm_src_dir, + contents, ) oldness = [(entry, entry.is_old(svn_version)) for entry in patch_entries] filtered_entries = [entry.to_dict() for entry, old in oldness if not old] - with atomic_write(patches_json_fp, encoding="utf-8") as f: - _write_json_changes(filtered_entries, f) + with atomic_write_file.atomic_write(patches_json_fp, encoding="utf-8") as f: + _write_json_changes(filtered_entries, f, indent_len=indent_len) removed_entries = [entry for entry, old in oldness if old] plural_patches = "patch" if len(removed_entries) == 1 else "patches" print(f"Removed {len(removed_entries)} old {plural_patches}:") diff --git a/llvm_tools/patch_utils_unittest.py b/llvm_tools/patch_utils_unittest.py index b8c21390..dfee55e3 100755 --- a/llvm_tools/patch_utils_unittest.py +++ b/llvm_tools/patch_utils_unittest.py @@ -20,32 +20,23 @@ import patch_utils as pu class TestPatchUtils(unittest.TestCase): """Test the patch_utils.""" - def test_atomic_write(self): - """Test that atomic write safely writes.""" - prior_contents = "This is a test written by patch_utils_unittest.py\n" - new_contents = "I am a test written by patch_utils_unittest.py\n" - with tempfile.TemporaryDirectory( - prefix="patch_utils_unittest" - ) as dirname: - dirpath = Path(dirname) - filepath = dirpath / "test_atomic_write.txt" - with filepath.open("w", encoding="utf-8") as f: - f.write(prior_contents) - - def _t(): - with pu.atomic_write(filepath, encoding="utf-8") as f: - f.write(new_contents) - raise Exception("Expected failure") - - self.assertRaises(Exception, _t) - with filepath.open(encoding="utf-8") as f: - lines = f.readlines() - self.assertEqual(lines[0], prior_contents) - with pu.atomic_write(filepath, encoding="utf-8") as f: - f.write(new_contents) - with filepath.open(encoding="utf-8") as f: - lines = f.readlines() - self.assertEqual(lines[0], new_contents) + def test_predict_indent(self): + test_str1 = """ +a + a + a + a +a +""" + self.assertEqual(pu.predict_indent(test_str1.splitlines()), 2) + test_str2 = """ +a + a + a + a +a +""" + self.assertEqual(pu.predict_indent(test_str2.splitlines()), 4) def test_from_to_dict(self): """Test to and from dict conversion.""" @@ -123,6 +114,9 @@ class TestPatchUtils(unittest.TestCase): } ] """ + result = pu.json_str_to_patch_entries(Path(), patches_json) + self.assertEqual(len(result), 4) + result = pu.json_to_patch_entries(Path(), io.StringIO(patches_json)) self.assertEqual(len(result), 4) @@ -245,6 +239,16 @@ Hunk #1 SUCCEEDED at 96 with fuzz 1. "until": 2, }, ), + pu.PatchEntry( + workdir=dirpath, + rel_patch_path="z.patch", + metadata=None, + platforms=None, + version_range={ + "from": 4, + "until": 5, + }, + ), ] patches[0].apply = mock.MagicMock( return_value=pu.PatchResult( @@ -254,6 +258,9 @@ Hunk #1 SUCCEEDED at 96 with fuzz 1. patches[1].apply = mock.MagicMock( return_value=pu.PatchResult(succeeded=True) ) + patches[2].apply = mock.MagicMock( + return_value=pu.PatchResult(succeeded=False) + ) results, _ = pu.update_version_ranges_with_entries( 1, dirpath, patches ) @@ -263,6 +270,7 @@ Hunk #1 SUCCEEDED at 96 with fuzz 1. self.assertEqual(results[0].version_range, {"from": 0, "until": 1}) self.assertEqual(patches[0].version_range, {"from": 0, "until": 1}) self.assertEqual(patches[1].version_range, {"from": 0, "until": 2}) + self.assertEqual(patches[2].version_range, {"from": 4, "until": 5}) @mock.patch("builtins.print") def test_remove_old_patches(self, _): diff --git a/llvm_tools/update_chromeos_llvm_hash.py b/llvm_tools/update_chromeos_llvm_hash.py index 75c6ce6c..9a8754d4 100755 --- a/llvm_tools/update_chromeos_llvm_hash.py +++ b/llvm_tools/update_chromeos_llvm_hash.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -17,12 +16,15 @@ import os from pathlib import Path import re import subprocess -from typing import Dict, Iterable +import textwrap +from typing import Dict, Iterable, List, Optional +import atomic_write_file import chroot import failure_modes import get_llvm_hash import git +import manifest_utils import patch_utils import subprocess_helpers @@ -33,6 +35,7 @@ DEFAULT_PACKAGES = [ "sys-libs/compiler-rt", "sys-libs/libcxx", "sys-libs/llvm-libunwind", + "sys-libs/scudo", ] DEFAULT_MANIFEST_PACKAGES = ["sys-devel/llvm"] @@ -46,9 +49,87 @@ class LLVMVariant(enum.Enum): next = "LLVM_NEXT_HASH" -# If set to `True`, then the contents of `stdout` after executing a command will -# be displayed to the terminal. -verbose = False +class PortagePackage: + """Represents a portage package with location info.""" + + def __init__(self, chromeos_root: Path, package: str): + """Create a new PortagePackage. + + Args: + chromeos_root: Path to the root of the code checkout. + package: "category/package" string. + """ + self.package = package + potential_ebuild_path = PortagePackage.find_package_ebuild( + chromeos_root, package + ) + if potential_ebuild_path.is_symlink(): + self.uprev_target = potential_ebuild_path.absolute() + self.ebuild_path = potential_ebuild_path.resolve() + else: + # Should have a 9999 ebuild, no uprevs needed. + self.uprev_target = None + self.ebuild_path = potential_ebuild_path.absolute() + + @staticmethod + def find_package_ebuild(chromeos_root: Path, package: str) -> Path: + """Look up the package's ebuild location.""" + chromeos_root_str = str(chromeos_root) + ebuild_paths = chroot.GetChrootEbuildPaths( + chromeos_root_str, + [package], + ) + converted = chroot.ConvertChrootPathsToAbsolutePaths( + chromeos_root_str, ebuild_paths + )[0] + return Path(converted) + + def package_dir(self) -> Path: + """Return the package directory.""" + return self.ebuild_path.parent + + def update( + self, llvm_variant: LLVMVariant, git_hash: str, svn_version: int + ): + """Update the package with the new LLVM git sha and revision. + + Args: + llvm_variant: Which LLVM hash to update. + Either LLVM_HASH or LLVM_NEXT_HASH. + git_hash: Upstream LLVM git hash to update to. + svn_version: Matching LLVM revision string for the git_hash. + """ + live_ebuild = self.live_ebuild() + if live_ebuild: + # Working with a -9999 ebuild package here, no + # upreving. + UpdateEbuildLLVMHash( + live_ebuild, llvm_variant, git_hash, svn_version + ) + return + if not self.uprev_target: + # We can exit early if we're not working with a live ebuild, + # and we don't have something to uprev. + raise RuntimeError( + f"Cannot update: no live ebuild or symlink found for {self.package}" + ) + + UpdateEbuildLLVMHash( + self.ebuild_path, llvm_variant, git_hash, svn_version + ) + if llvm_variant == LLVMVariant.current: + UprevEbuildToVersion(str(self.uprev_target), svn_version, git_hash) + else: + UprevEbuildSymlink(str(self.uprev_target)) + + def live_ebuild(self) -> Optional[Path]: + """Path to the live ebuild if it exists. + + Returns: + The patch to the live ebuild if it exists. None otherwise. + """ + matches = self.package_dir().glob("*-9999.ebuild") + return next(matches, None) def defaultCrosRoot() -> Path: @@ -104,14 +185,6 @@ def GetCommandLineArgs(): "(default: %(default)s)", ) - # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - "--verbose", - action="store_true", - help="display contents of a command to the terminal " - "(default: %(default)s)", - ) - # Add argument for the LLVM hash to update parser.add_argument( "--is_llvm_next", @@ -150,59 +223,23 @@ def GetCommandLineArgs(): help="the .json file that has all the patches and their " "metadata if applicable (default: PATCHES.json inside $FILESDIR)", ) + parser.add_argument( + "--repo_manifest", + action="store_true", + help="Updates the llvm-project revision attribute" + " in the internal manifest.", + ) # Parse the command line. - args_output = parser.parse_args() - - # FIXME: We shouldn't be using globals here, but until we fix it, make pylint - # stop complaining about it. - # pylint: disable=global-statement - global verbose + return parser.parse_args() - verbose = args_output.verbose - - return args_output - - -def GetEbuildPathsFromSymLinkPaths(symlinks): - """Reads the symlink(s) to get the ebuild path(s) to the package(s). - - Args: - symlinks: A list of absolute path symlink/symlinks that point - to the package's ebuild. - - Returns: - A dictionary where the key is the absolute path of the symlink and the value - is the absolute path to the ebuild that was read from the symlink. - Raises: - ValueError: Invalid symlink(s) were provided. - """ - - # A dictionary that holds: - # key: absolute symlink path - # value: absolute ebuild path - resolved_paths = {} - - # Iterate through each symlink. - # - # For each symlink, check that it is a valid symlink, - # and then construct the ebuild path, and - # then add the ebuild path to the dict. - for cur_symlink in symlinks: - if not os.path.islink(cur_symlink): - raise ValueError(f"Invalid symlink provided: {cur_symlink}") - - # Construct the absolute path to the ebuild. - ebuild_path = os.path.realpath(cur_symlink) - - if cur_symlink not in resolved_paths: - resolved_paths[cur_symlink] = ebuild_path - - return resolved_paths - - -def UpdateEbuildLLVMHash(ebuild_path, llvm_variant, git_hash, svn_version): +def UpdateEbuildLLVMHash( + ebuild_path: Path, + llvm_variant: LLVMVariant, + git_hash: str, + svn_version: int, +): """Updates the LLVM hash in the ebuild. The build changes are staged for commit in the temporary repo. @@ -218,33 +255,26 @@ def UpdateEbuildLLVMHash(ebuild_path, llvm_variant, git_hash, svn_version): of the changes or failed to update the LLVM hash. """ - # Iterate through each ebuild. - # # For each ebuild, read the file in # advance and then create a temporary file # that gets updated with the new LLVM hash # and revision number and then the ebuild file # gets updated to the temporary file. - if not os.path.isfile(ebuild_path): raise ValueError(f"Invalid ebuild path provided: {ebuild_path}") - temp_ebuild_file = f"{ebuild_path}.temp" - - with open(ebuild_path) as ebuild_file: - # write updates to a temporary file in case of interrupts - with open(temp_ebuild_file, "w") as temp_file: - for cur_line in ReplaceLLVMHash( - ebuild_file, llvm_variant, git_hash, svn_version - ): - temp_file.write(cur_line) - os.rename(temp_ebuild_file, ebuild_path) - - # Get the path to the parent directory. - parent_dir = os.path.dirname(ebuild_path) - + with open(ebuild_path, encoding="utf-8") as ebuild_file: + new_lines = list( + ReplaceLLVMHash(ebuild_file, llvm_variant, git_hash, svn_version) + ) + with atomic_write_file.atomic_write( + ebuild_path, "w", encoding="utf-8" + ) as ebuild_file: + ebuild_file.writelines(new_lines) # Stage the changes. - subprocess.check_output(["git", "-C", parent_dir, "add", ebuild_path]) + subprocess.check_output( + ["git", "-C", ebuild_path.parent, "add", ebuild_path] + ) def ReplaceLLVMHash(ebuild_lines, llvm_variant, git_hash, svn_version): @@ -367,46 +397,11 @@ def UprevEbuildToVersion(symlink, svn_version, git_hash): # Create a symlink of the renamed ebuild new_symlink = new_ebuild[: -len(".ebuild")] + "-r1.ebuild" subprocess.check_output(["ln", "-s", "-r", new_ebuild, new_symlink]) - - if not os.path.islink(new_symlink): - raise ValueError( - f'Invalid symlink name: {new_ebuild[:-len(".ebuild")]}' - ) - subprocess.check_output(["git", "-C", symlink_dir, "add", new_symlink]) - # Remove the old symlink subprocess.check_output(["git", "-C", symlink_dir, "rm", symlink]) -def CreatePathDictionaryFromPackages(chroot_path, update_packages): - """Creates a symlink and ebuild path pair dictionary from the packages. - - Args: - chroot_path: The absolute path to the chroot. - update_packages: The filtered packages to be updated. - - Returns: - A dictionary where the key is the absolute path to the symlink - of the package and the value is the absolute path to the ebuild of - the package. - """ - - # Construct a list containing the chroot file paths of the package(s). - chroot_file_paths = chroot.GetChrootEbuildPaths( - chroot_path, update_packages - ) - - # Construct a list containing the symlink(s) of the package(s). - symlink_file_paths = chroot.ConvertChrootPathsToAbsolutePaths( - chroot_path, chroot_file_paths - ) - - # Create a dictionary where the key is the absolute path of the symlink to - # the package and the value is the absolute path to the ebuild of the package. - return GetEbuildPathsFromSymLinkPaths(symlink_file_paths) - - def RemovePatchesFromFilesDir(patches): """Removes the patches from $FILESDIR of a package. @@ -506,8 +501,8 @@ def StagePackagesPatchResultsForCommit(package_info_dict, commit_messages): return commit_messages -def UpdateManifests(packages: Iterable[str], chroot_path: Path): - """Updates manifest files for packages. +def UpdatePortageManifests(packages: Iterable[str], chroot_path: Path): + """Updates portage manifest files for packages. Args: packages: A list of packages to update manifests for. @@ -518,17 +513,21 @@ def UpdateManifests(packages: Iterable[str], chroot_path: Path): """ manifest_ebuilds = chroot.GetChrootEbuildPaths(chroot_path, packages) for ebuild_path in manifest_ebuilds: + ebuild_dir = os.path.dirname(ebuild_path) subprocess_helpers.ChrootRunCommand( chroot_path, ["ebuild", ebuild_path, "manifest"] ) + subprocess_helpers.ChrootRunCommand( + chroot_path, ["git", "-C", ebuild_dir, "add", "Manifest"] + ) def UpdatePackages( packages: Iterable[str], manifest_packages: Iterable[str], - llvm_variant, - git_hash, - svn_version, + llvm_variant: LLVMVariant, + git_hash: str, + svn_version: int, chroot_path: Path, mode, git_hash_source, @@ -558,86 +557,55 @@ def UpdatePackages( Gerrit commit URL and the second pair is the change list number. """ - # Construct a dictionary where the key is the absolute path of the symlink to - # the package and the value is the absolute path to the ebuild of the package. - paths_dict = CreatePathDictionaryFromPackages(chroot_path, packages) - - repo_path = os.path.dirname(next(iter(paths_dict.values()))) - - branch = "update-" + llvm_variant.value + "-" + git_hash + portage_packages = (PortagePackage(chroot_path, pkg) for pkg in packages) + chromiumos_overlay_path = ( + chroot_path / "src" / "third_party" / "chromiumos-overlay" + ) + branch_name = "update-" + llvm_variant.value + "-" + git_hash + + commit_message_header = "llvm" + if llvm_variant == LLVMVariant.next: + commit_message_header = "llvm-next" + if git_hash_source in get_llvm_hash.KNOWN_HASH_SOURCES: + commit_message_header += ( + f"/{git_hash_source}: upgrade to {git_hash} (r{svn_version})" + ) + else: + commit_message_header += f": upgrade to {git_hash} (r{svn_version})" - git.CreateBranch(repo_path, branch) + commit_lines = [ + commit_message_header + "\n", + "The following packages have been updated:", + ] + # Holds the list of packages that are updating. + updated_packages: List[str] = [] + git.CreateBranch(chromiumos_overlay_path, branch_name) try: - commit_message_header = "llvm" - if llvm_variant == LLVMVariant.next: - commit_message_header = "llvm-next" - if git_hash_source in get_llvm_hash.KNOWN_HASH_SOURCES: - commit_message_header += ( - f"/{git_hash_source}: upgrade to {git_hash} (r{svn_version})" - ) - else: - commit_message_header += f": upgrade to {git_hash} (r{svn_version})" - - commit_lines = [ - commit_message_header + "\n", - "The following packages have been updated:", - ] - - # Holds the list of packages that are updating. - packages = [] - - # Iterate through the dictionary. - # - # For each iteration: - # 1) Update the ebuild's LLVM hash. - # 2) Uprev the ebuild (symlink). - # 3) Add the modified package to the commit message. - for symlink_path, ebuild_path in paths_dict.items(): - path_to_ebuild_dir = os.path.dirname(ebuild_path) - - UpdateEbuildLLVMHash( - ebuild_path, llvm_variant, git_hash, svn_version - ) - - if llvm_variant == LLVMVariant.current: - UprevEbuildToVersion(symlink_path, svn_version, git_hash) - else: - UprevEbuildSymlink(symlink_path) - - cur_dir_name = os.path.basename(path_to_ebuild_dir) - parent_dir_name = os.path.basename( - os.path.dirname(path_to_ebuild_dir) - ) - - packages.append(f"{parent_dir_name}/{cur_dir_name}") - commit_lines.append(f"{parent_dir_name}/{cur_dir_name}") - + for pkg in portage_packages: + pkg.update(llvm_variant, git_hash, svn_version) + updated_packages.append(pkg.package) + commit_lines.append(pkg.package) if manifest_packages: - UpdateManifests(manifest_packages, chroot_path) + UpdatePortageManifests(manifest_packages, chroot_path) commit_lines.append("Updated manifest for:") commit_lines.extend(manifest_packages) - EnsurePackageMaskContains(chroot_path, git_hash) - # Handle the patches for each package. package_info_dict = UpdatePackagesPatchMetadataFile( - chroot_path, svn_version, packages, mode + chroot_path, svn_version, updated_packages, mode ) - # Update the commit message if changes were made to a package's patches. commit_lines = StagePackagesPatchResultsForCommit( package_info_dict, commit_lines ) - if extra_commit_msg: commit_lines.append(extra_commit_msg) - - change_list = git.UploadChanges(repo_path, branch, commit_lines) - + change_list = git.UploadChanges( + chromiumos_overlay_path, branch_name, commit_lines + ) finally: - git.DeleteBranch(repo_path, branch) - + git.DeleteBranch(chromiumos_overlay_path, branch_name) return change_list @@ -660,7 +628,7 @@ def EnsurePackageMaskContains(chroot_path, git_hash): mask_path = os.path.join( overlay_dir, "profiles/targets/chromeos/package.mask" ) - with open(mask_path, "r+") as mask_file: + with open(mask_path, "r+", encoding="utf-8") as mask_file: mask_contents = mask_file.read() expected_line = f"=sys-devel/llvm-{llvm_major_version}.0_pre*\n" if expected_line not in mask_contents: @@ -715,7 +683,7 @@ def UpdatePackagesPatchMetadataFile( ) chroot_ebuild_path = Path( chroot.ConvertChrootPathsToAbsolutePaths( - chroot_path, [chroot_ebuild_str] + str(chroot_path), [chroot_ebuild_str] )[0] ) patches_json_fp = ( @@ -747,12 +715,61 @@ def UpdatePackagesPatchMetadataFile( patches_info = patch_utils.update_version_ranges( svn_version, src_path, patches_json_fp ) + else: + raise RuntimeError(f"unsupported failure mode: {mode}") package_info[cur_package] = patches_info._asdict() return package_info +def ChangeRepoManifest(git_hash: str, src_tree: Path): + """Change the repo internal manifest for llvm-project. + + Args: + git_hash: The LLVM git hash to change to. + src_tree: ChromiumOS source tree checkout. + + Returns: + The uploaded changelist CommitContents. + """ + manifest_dir = manifest_utils.get_chromeos_manifest_path(src_tree).parent + branch_name = "update-llvm-project-" + git_hash + commit_lines = ( + textwrap.dedent( + f""" + manifest: Update llvm-project to {git_hash} + + Upgrade the local LLVM reversion to match the new llvm ebuild + hash. This must be merged along with any chromiumos-overlay + changes to LLVM. Automatic uprevs rely on the manifest hash + to match what is specified by LLVM_HASH. + + This CL is generated by the update_chromeos_llvm_hash.py script. + + BUG=None + TEST=CQ + """ + ) + .lstrip() + .splitlines() + ) + + git.CreateBranch(manifest_dir, branch_name) + try: + manifest_path = manifest_utils.update_chromeos_manifest( + git_hash, + src_tree, + ) + subprocess.run( + ["git", "-C", manifest_dir, "add", manifest_path.name], check=True + ) + change_list = git.UploadChanges(manifest_dir, branch_name, commit_lines) + finally: + git.DeleteBranch(manifest_dir, branch_name) + return change_list + + def main(): """Updates the LLVM next hash for each package. @@ -764,6 +781,8 @@ def main(): args_output = GetCommandLineArgs() + chroot.VerifyChromeOSRoot(args_output.chroot_path) + llvm_variant = LLVMVariant.current if args_output.is_llvm_next: llvm_variant = LLVMVariant.next @@ -773,7 +792,6 @@ def main(): git_hash, svn_version = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption( git_hash_source ) - # Filter out empty strings. For example "".split{",") returns [""]. packages = set(p for p in args_output.update_packages.split(",") if p) manifest_packages = set( @@ -798,6 +816,17 @@ def main(): print(f"Gerrit URL: {change_list.url}") print(f"Change list number: {change_list.cl_number}") + if args_output.repo_manifest: + print( + f"Updating internal manifest to {git_hash} ({svn_version})...", + end="", + ) + change_list = ChangeRepoManifest(git_hash, args_output.chroot_path) + print(" Done!") + print("New repo manifest CL:") + print(f" URL: {change_list.url}") + print(f" CL Number: {change_list.cl_number}") + if __name__ == "__main__": main() diff --git a/llvm_tools/update_chromeos_llvm_hash_unittest.py b/llvm_tools/update_chromeos_llvm_hash_unittest.py index b758538c..568242aa 100755 --- a/llvm_tools/update_chromeos_llvm_hash_unittest.py +++ b/llvm_tools/update_chromeos_llvm_hash_unittest.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -13,6 +12,7 @@ import os from pathlib import Path import subprocess import sys +import tempfile import unittest import unittest.mock as mock @@ -53,7 +53,7 @@ class UpdateLLVMHashTest(unittest.TestCase): # does not exist. @mock.patch.object(os.path, "isfile", return_value=False) def testFailedToUpdateLLVMHashForInvalidEbuildPath(self, mock_isfile): - ebuild_path = "/some/path/to/package.ebuild" + ebuild_path = Path("/some/path/to/package.ebuild") llvm_variant = update_chromeos_llvm_hash.LLVMVariant.current git_hash = "a123testhash1" svn_version = 1000 @@ -94,7 +94,7 @@ class UpdateLLVMHashTest(unittest.TestCase): # 'LLVM_HASH'. with self.assertRaises(ValueError) as err: update_chromeos_llvm_hash.UpdateEbuildLLVMHash( - ebuild_file, llvm_variant, git_hash, svn_version + Path(ebuild_file), llvm_variant, git_hash, svn_version ) self.assertEqual(str(err.exception), "Failed to update LLVM_HASH") @@ -127,7 +127,7 @@ class UpdateLLVMHashTest(unittest.TestCase): # 'LLVM_NEXT_HASH'. with self.assertRaises(ValueError) as err: update_chromeos_llvm_hash.UpdateEbuildLLVMHash( - ebuild_file, llvm_variant, git_hash, svn_version + Path(ebuild_file), llvm_variant, git_hash, svn_version ) self.assertEqual( @@ -141,7 +141,6 @@ class UpdateLLVMHashTest(unittest.TestCase): def testSuccessfullyStageTheEbuildForCommitForLLVMHashUpdate( self, mock_stage_commit_command, mock_isfile ): - # Create a temporary file to simulate an ebuild file of a package. with test_helpers.CreateTemporaryJsonFile() as ebuild_file: # Updates LLVM_HASH to 'git_hash' and revision to @@ -163,7 +162,7 @@ class UpdateLLVMHashTest(unittest.TestCase): ) update_chromeos_llvm_hash.UpdateEbuildLLVMHash( - ebuild_file, llvm_variant, git_hash, svn_version + Path(ebuild_file), llvm_variant, git_hash, svn_version ) expected_file_contents = [ @@ -190,7 +189,6 @@ class UpdateLLVMHashTest(unittest.TestCase): def testSuccessfullyStageTheEbuildForCommitForLLVMNextHashUpdate( self, mock_stage_commit_command, mock_isfile ): - # Create a temporary file to simulate an ebuild file of a package. with test_helpers.CreateTemporaryJsonFile() as ebuild_file: # Updates LLVM_NEXT_HASH to 'git_hash' and revision to @@ -212,7 +210,7 @@ class UpdateLLVMHashTest(unittest.TestCase): ) update_chromeos_llvm_hash.UpdateEbuildLLVMHash( - ebuild_file, llvm_variant, git_hash, svn_version + Path(ebuild_file), llvm_variant, git_hash, svn_version ) expected_file_contents = [ @@ -374,19 +372,35 @@ class UpdateLLVMHashTest(unittest.TestCase): def testManifestUpdate(self, mock_subprocess, mock_ebuild_paths): manifest_packages = ["sys-devel/llvm"] chroot_path = "/path/to/chroot" - update_chromeos_llvm_hash.UpdateManifests( - manifest_packages, chroot_path - ) - - args = mock_subprocess.call_args[0][-1] - manifest_cmd = [ - "cros_sdk", - "--", - "ebuild", - "/chroot/path/test.ebuild", - "manifest", - ] - self.assertEqual(args, manifest_cmd) + update_chromeos_llvm_hash.UpdatePortageManifests( + manifest_packages, Path(chroot_path) + ) + + args = mock_subprocess.call_args_list[0] + manifest_cmd = ( + [ + "cros_sdk", + "--", + "ebuild", + "/chroot/path/test.ebuild", + "manifest", + ], + ) + self.assertEqual(args[0], manifest_cmd) + + args = mock_subprocess.call_args_list[1] + git_add_cmd = ( + [ + "cros_sdk", + "--", + "git", + "-C", + "/chroot/path", + "add", + "Manifest", + ], + ) + self.assertEqual(args[0], git_add_cmd) mock_ebuild_paths.assert_called_once() @mock.patch.object(get_llvm_hash, "GetLLVMMajorVersion") @@ -457,122 +471,6 @@ class UpdateLLVMHashTest(unittest.TestCase): mock_command_output.assert_called_once() - # Simulate behavior of 'os.path.isdir()' when the path to the repo is not a - - # directory. - - @mock.patch.object(chroot, "GetChrootEbuildPaths") - @mock.patch.object(chroot, "ConvertChrootPathsToAbsolutePaths") - def testExceptionRaisedWhenCreatingPathDictionaryFromPackages( - self, mock_chroot_paths_to_symlinks, mock_get_chroot_paths - ): - - chroot_path = "/some/path/to/chroot" - - package_name = "test-pckg/package" - package_chroot_path = "/some/chroot/path/to/package-r1.ebuild" - - # Test function to simulate 'ConvertChrootPathsToAbsolutePaths' when a - # symlink does not start with the prefix '/mnt/host/source'. - def BadPrefixChrootPath(*args): - assert len(args) == 2 - raise ValueError( - "Invalid prefix for the chroot path: " - "%s" % package_chroot_path - ) - - # Simulate 'GetChrootEbuildPaths' when valid packages are passed in. - # - # Returns a list of chroot paths. - mock_get_chroot_paths.return_value = [package_chroot_path] - - # Use test function to simulate 'ConvertChrootPathsToAbsolutePaths' - # behavior. - mock_chroot_paths_to_symlinks.side_effect = BadPrefixChrootPath - - # Verify exception is raised when for an invalid prefix in the symlink. - with self.assertRaises(ValueError) as err: - update_chromeos_llvm_hash.CreatePathDictionaryFromPackages( - chroot_path, [package_name] - ) - - self.assertEqual( - str(err.exception), - "Invalid prefix for the chroot path: " "%s" % package_chroot_path, - ) - - mock_get_chroot_paths.assert_called_once_with( - chroot_path, [package_name] - ) - - mock_chroot_paths_to_symlinks.assert_called_once_with( - chroot_path, [package_chroot_path] - ) - - @mock.patch.object(chroot, "GetChrootEbuildPaths") - @mock.patch.object(chroot, "ConvertChrootPathsToAbsolutePaths") - @mock.patch.object( - update_chromeos_llvm_hash, "GetEbuildPathsFromSymLinkPaths" - ) - def testSuccessfullyCreatedPathDictionaryFromPackages( - self, - mock_ebuild_paths_from_symlink_paths, - mock_chroot_paths_to_symlinks, - mock_get_chroot_paths, - ): - - package_chroot_path = "/mnt/host/source/src/path/to/package-r1.ebuild" - - # Simulate 'GetChrootEbuildPaths' when returning a chroot path for a valid - # package. - # - # Returns a list of chroot paths. - mock_get_chroot_paths.return_value = [package_chroot_path] - - package_symlink_path = ( - "/some/path/to/chroot/src/path/to/package-r1.ebuild" - ) - - # Simulate 'ConvertChrootPathsToAbsolutePaths' when returning a symlink to - # a chroot path that points to a package. - # - # Returns a list of symlink file paths. - mock_chroot_paths_to_symlinks.return_value = [package_symlink_path] - - chroot_package_path = "/some/path/to/chroot/src/path/to/package.ebuild" - - # Simulate 'GetEbuildPathsFromSymlinkPaths' when returning a dictionary of - # a symlink that points to an ebuild. - # - # Returns a dictionary of a symlink and ebuild file path pair - # where the key is the absolute path to the symlink of the ebuild file - # and the value is the absolute path to the ebuild file of the package. - mock_ebuild_paths_from_symlink_paths.return_value = { - package_symlink_path: chroot_package_path - } - - chroot_path = "/some/path/to/chroot" - package_name = "test-pckg/package" - - self.assertEqual( - update_chromeos_llvm_hash.CreatePathDictionaryFromPackages( - chroot_path, [package_name] - ), - {package_symlink_path: chroot_package_path}, - ) - - mock_get_chroot_paths.assert_called_once_with( - chroot_path, [package_name] - ) - - mock_chroot_paths_to_symlinks.assert_called_once_with( - chroot_path, [package_chroot_path] - ) - - mock_ebuild_paths_from_symlink_paths.assert_called_once_with( - [package_symlink_path] - ) - @mock.patch.object(subprocess, "check_output", return_value=None) def testSuccessfullyRemovedPatchesFromFilesDir(self, mock_run_cmd): patches_to_remove_list = [ @@ -607,7 +505,6 @@ class UpdateLLVMHashTest(unittest.TestCase): @mock.patch.object(os.path, "isfile", return_value=True) @mock.patch.object(subprocess, "check_output", return_value=None) def testSuccessfullyStagedPatchMetadataFileForCommit(self, mock_run_cmd, _): - patch_metadata_path = "/abs/path/to/filesdir/PATCHES.json" update_chromeos_llvm_hash.StagePatchMetadataFileForCommit( @@ -656,7 +553,6 @@ class UpdateLLVMHashTest(unittest.TestCase): def testAddedPatchResultsForCommit( self, mock_remove_patches, mock_stage_patches_for_commit ): - package_1_patch_info_dict = { "applied_patches": [], "failed_patches": [], @@ -707,321 +603,101 @@ class UpdateLLVMHashTest(unittest.TestCase): self.assertEqual(mock_stage_patches_for_commit.call_count, 2) - @mock.patch.object(get_llvm_hash, "GetLLVMMajorVersion") - @mock.patch.object( - update_chromeos_llvm_hash, "CreatePathDictionaryFromPackages" - ) - @mock.patch.object(git, "CreateBranch") - @mock.patch.object(update_chromeos_llvm_hash, "UpdateEbuildLLVMHash") - @mock.patch.object(update_chromeos_llvm_hash, "UprevEbuildSymlink") - @mock.patch.object(git, "UploadChanges") - @mock.patch.object(git, "DeleteBranch") - @mock.patch.object(os.path, "realpath") - def testExceptionRaisedWhenUpdatingPackages( - self, - mock_realpath, - mock_delete_repo, - mock_upload_changes, - mock_uprev_symlink, - mock_update_llvm_next, - mock_create_repo, - mock_create_path_dict, - mock_llvm_major_version, - ): - - path_to_package_dir = "/some/path/to/chroot/src/path/to" - abs_path_to_package = os.path.join( - path_to_package_dir, "package.ebuild" - ) - symlink_path_to_package = os.path.join( - path_to_package_dir, "package-r1.ebuild" + def setup_mock_src_tree(self, src_tree: Path): + package_dir = ( + src_tree / "src/third_party/chromiumos-overlay/sys-devel/llvm" ) - - mock_llvm_major_version.return_value = "1234" - - # Test function to simulate 'CreateBranch' when successfully created the - # branch on a valid repo path. - def SuccessfullyCreateBranchForChanges(_, branch): - self.assertEqual(branch, "update-LLVM_NEXT_HASH-a123testhash4") - - # Test function to simulate 'UpdateEbuildLLVMHash' when successfully - # updated the ebuild's 'LLVM_NEXT_HASH'. - def SuccessfullyUpdatedLLVMHash(ebuild_path, _, git_hash, svn_version): - self.assertEqual(ebuild_path, abs_path_to_package) - self.assertEqual(git_hash, "a123testhash4") - self.assertEqual(svn_version, 1000) - - # Test function to simulate 'UprevEbuildSymlink' when the symlink to the - # ebuild does not have a revision number. - def FailedToUprevEbuildSymlink(_): - # Raises a 'ValueError' exception because the symlink did not have have a - # revision number. - raise ValueError("Failed to uprev the ebuild.") - - # Test function to fail on 'UploadChanges' if the function gets called - # when an exception is raised. - def ShouldNotExecuteUploadChanges(*args): - # Test function should not be called (i.e. execution should resume in the - # 'finally' block) because 'UprevEbuildSymlink' raised an - # exception. - assert len(args) == 3 - assert False, ( - 'Failed to go to "finally" block ' - "after the exception was raised." + package_dir.mkdir(parents=True) + ebuild_path = package_dir / "llvm-00.00_pre0_p0.ebuild" + with ebuild_path.open("w", encoding="utf-8") as f: + f.writelines( + [ + 'LLVM_HASH="abcdef123456" # r123456', + 'LLVM_NEXT_HASH="987654321fedcba" # r99453', + ] ) - - test_package_path_dict = {symlink_path_to_package: abs_path_to_package} - - # Simulate behavior of 'CreatePathDictionaryFromPackages()' when - # successfully created a dictionary where the key is the absolute path to - # the symlink of the package and value is the absolute path to the ebuild of - # the package. - mock_create_path_dict.return_value = test_package_path_dict - - # Use test function to simulate behavior. - mock_create_repo.side_effect = SuccessfullyCreateBranchForChanges - mock_update_llvm_next.side_effect = SuccessfullyUpdatedLLVMHash - mock_uprev_symlink.side_effect = FailedToUprevEbuildSymlink - mock_upload_changes.side_effect = ShouldNotExecuteUploadChanges - mock_realpath.return_value = ( - "/abs/path/to/test-packages/package1.ebuild" - ) - - packages_to_update = ["test-packages/package1"] - llvm_variant = update_chromeos_llvm_hash.LLVMVariant.next - git_hash = "a123testhash4" - svn_version = 1000 - chroot_path = Path("/some/path/to/chroot") - git_hash_source = "google3" - branch = "update-LLVM_NEXT_HASH-a123testhash4" - extra_commit_msg = None - - # Verify exception is raised when an exception is thrown within - # the 'try' block by UprevEbuildSymlink function. - with self.assertRaises(ValueError) as err: - update_chromeos_llvm_hash.UpdatePackages( - packages=packages_to_update, - manifest_packages=[], - llvm_variant=llvm_variant, - git_hash=git_hash, - svn_version=svn_version, - chroot_path=chroot_path, - mode=failure_modes.FailureModes.FAIL, - git_hash_source=git_hash_source, - extra_commit_msg=extra_commit_msg, + symlink_path = package_dir / "llvm-00.00_pre0_p0-r1234.ebuild" + symlink_path.symlink_to(ebuild_path) + return package_dir, ebuild_path, symlink_path + + def testPortagePackageConstruction(self): + with tempfile.TemporaryDirectory( + "update_chromeos_llvm_hash.tmp" + ) as workdir_str: + src_tree = Path(workdir_str) + package_dir, ebuild_path, symlink_path = self.setup_mock_src_tree( + src_tree ) - self.assertEqual(str(err.exception), "Failed to uprev the ebuild.") - - mock_create_path_dict.assert_called_once_with( - chroot_path, packages_to_update - ) - - mock_create_repo.assert_called_once_with(path_to_package_dir, branch) - - mock_update_llvm_next.assert_called_once_with( - abs_path_to_package, llvm_variant, git_hash, svn_version - ) - - mock_uprev_symlink.assert_called_once_with(symlink_path_to_package) + # Test that we're upreving if there's a symlink. + def mock_find_package_ebuild(_, package_name): + self.assertEqual( + package_name, + f"{package_dir.parent.name}/{package_dir.name}", + ) + return symlink_path + + with mock.patch( + "update_chromeos_llvm_hash.PortagePackage.find_package_ebuild", + mock_find_package_ebuild, + ): + pkg = update_chromeos_llvm_hash.PortagePackage( + src_tree, "sys-devel/llvm" + ) + self.assertEqual(pkg.uprev_target, symlink_path.absolute()) + self.assertEqual(pkg.ebuild_path, ebuild_path.absolute()) + self.assertEqual(pkg.live_ebuild(), None) - mock_upload_changes.assert_not_called() + # Make sure if the live ebuild is there, we find it. + live_ebuild_path = package_dir / "llvm-9999.ebuild" + live_ebuild_path.touch() - mock_delete_repo.assert_called_once_with(path_to_package_dir, branch) + pkg = update_chromeos_llvm_hash.PortagePackage( + src_tree, "sys-devel/llvm" + ) + self.assertEqual(pkg.live_ebuild(), live_ebuild_path) - @mock.patch.object(update_chromeos_llvm_hash, "EnsurePackageMaskContains") + @mock.patch("subprocess.run") + @mock.patch("subprocess.check_output") @mock.patch.object(get_llvm_hash, "GetLLVMMajorVersion") - @mock.patch.object( - update_chromeos_llvm_hash, "CreatePathDictionaryFromPackages" - ) - @mock.patch.object(git, "CreateBranch") - @mock.patch.object(update_chromeos_llvm_hash, "UpdateEbuildLLVMHash") - @mock.patch.object(update_chromeos_llvm_hash, "UprevEbuildSymlink") - @mock.patch.object(git, "UploadChanges") - @mock.patch.object(git, "DeleteBranch") - @mock.patch.object( - update_chromeos_llvm_hash, "UpdatePackagesPatchMetadataFile" - ) - @mock.patch.object( - update_chromeos_llvm_hash, "StagePatchMetadataFileForCommit" - ) - def testSuccessfullyUpdatedPackages( - self, - mock_stage_patch_file, - mock_update_package_metadata_file, - mock_delete_repo, - mock_upload_changes, - mock_uprev_symlink, - mock_update_llvm_next, - mock_create_repo, - mock_create_path_dict, - mock_llvm_version, - mock_mask_contains, + def testUpdatePackages( + self, mock_llvm_major_version, mock_check_output, mock_run ): - - path_to_package_dir = "/some/path/to/chroot/src/path/to" - abs_path_to_package = os.path.join( - path_to_package_dir, "package.ebuild" - ) - symlink_path_to_package = os.path.join( - path_to_package_dir, "package-r1.ebuild" - ) - - # Test function to simulate 'CreateBranch' when successfully created the - # branch for the changes to be made to the ebuild files. - def SuccessfullyCreateBranchForChanges(_, branch): - self.assertEqual(branch, "update-LLVM_NEXT_HASH-a123testhash5") - - # Test function to simulate 'UploadChanges' after a successfull update of - # 'LLVM_NEXT_HASH" of the ebuild file. - def SuccessfullyUpdatedLLVMHash(ebuild_path, _, git_hash, svn_version): - self.assertEqual( - ebuild_path, "/some/path/to/chroot/src/path/to/package.ebuild" - ) - self.assertEqual(git_hash, "a123testhash5") - self.assertEqual(svn_version, 1000) - - # Test function to simulate 'UprevEbuildSymlink' when successfully - # incremented the revision number by 1. - def SuccessfullyUprevedEbuildSymlink(symlink_path): - self.assertEqual( - symlink_path, - "/some/path/to/chroot/src/path/to/package-r1.ebuild", + mock_llvm_major_version.return_value = "17" + with tempfile.TemporaryDirectory( + "update_chromeos_llvm_hash.tmp" + ) as workdir_str: + src_tree = Path(workdir_str) + package_dir, ebuild_path, symlink_path = self.setup_mock_src_tree( + src_tree ) - # Test function to simulate 'UpdatePackagesPatchMetadataFile()' when the - # patch results contains a disabled patch in 'disable_patches' mode. - def RetrievedPatchResults(chroot_path, svn_version, packages, mode): - - self.assertEqual(chroot_path, Path("/some/path/to/chroot")) - self.assertEqual(svn_version, 1000) - self.assertListEqual(packages, ["path/to"]) - self.assertEqual(mode, failure_modes.FailureModes.DISABLE_PATCHES) - - patch_metadata_file = "PATCHES.json" - PatchInfo = collections.namedtuple( - "PatchInfo", - [ - "applied_patches", - "failed_patches", - "non_applicable_patches", - "disabled_patches", - "removed_patches", - "modified_metadata", - ], - ) - - package_patch_info = PatchInfo( - applied_patches=["fix_display.patch"], - failed_patches=["fix_stdout.patch"], - non_applicable_patches=[], - disabled_patches=["fix_stdout.patch"], - removed_patches=[], - modified_metadata="/abs/path/to/filesdir/%s" - % patch_metadata_file, - ) - - package_info_dict = {"path/to": package_patch_info._asdict()} - - # Returns a dictionary where the key is the package and the value is a - # dictionary that contains information about the package's patch results - # produced by the patch manager. - return package_info_dict - - # Test function to simulate 'UploadChanges()' when successfully created a - # commit for the changes made to the packages and their patches and - # retrieved the change list of the commit. - def SuccessfullyUploadedChanges(*args): - assert len(args) == 3 - commit_url = "https://some_name/path/to/commit/+/12345" - return git.CommitContents(url=commit_url, cl_number=12345) - - test_package_path_dict = {symlink_path_to_package: abs_path_to_package} - - # Simulate behavior of 'CreatePathDictionaryFromPackages()' when - # successfully created a dictionary where the key is the absolute path to - # the symlink of the package and value is the absolute path to the ebuild of - # the package. - mock_create_path_dict.return_value = test_package_path_dict - - # Use test function to simulate behavior. - mock_create_repo.side_effect = SuccessfullyCreateBranchForChanges - mock_update_llvm_next.side_effect = SuccessfullyUpdatedLLVMHash - mock_uprev_symlink.side_effect = SuccessfullyUprevedEbuildSymlink - mock_update_package_metadata_file.side_effect = RetrievedPatchResults - mock_upload_changes.side_effect = SuccessfullyUploadedChanges - mock_llvm_version.return_value = "1234" - mock_mask_contains.reurn_value = None + def mock_find_package_ebuild(*_): + return symlink_path - packages_to_update = ["test-packages/package1"] - llvm_variant = update_chromeos_llvm_hash.LLVMVariant.next - git_hash = "a123testhash5" - svn_version = 1000 - chroot_path = Path("/some/path/to/chroot") - git_hash_source = "tot" - branch = "update-LLVM_NEXT_HASH-a123testhash5" - extra_commit_msg = "\ncommit-message-end" - - change_list = update_chromeos_llvm_hash.UpdatePackages( - packages=packages_to_update, - manifest_packages=[], - llvm_variant=llvm_variant, - git_hash=git_hash, - svn_version=svn_version, - chroot_path=chroot_path, - mode=failure_modes.FailureModes.DISABLE_PATCHES, - git_hash_source=git_hash_source, - extra_commit_msg=extra_commit_msg, - ) - - self.assertEqual( - change_list.url, "https://some_name/path/to/commit/+/12345" - ) - - self.assertEqual(change_list.cl_number, 12345) - - mock_create_path_dict.assert_called_once_with( - chroot_path, packages_to_update - ) - - mock_create_repo.assert_called_once_with(path_to_package_dir, branch) - - mock_update_llvm_next.assert_called_once_with( - abs_path_to_package, llvm_variant, git_hash, svn_version - ) - - mock_uprev_symlink.assert_called_once_with(symlink_path_to_package) - - mock_mask_contains.assert_called_once_with(chroot_path, git_hash) - - expected_commit_messages = [ - "llvm-next/tot: upgrade to a123testhash5 (r1000)\n", - "The following packages have been updated:", - "path/to", - "\nFor the package path/to:", - "The patch metadata file PATCHES.json was modified", - "The following patches were disabled:", - "fix_stdout.patch", - "\ncommit-message-end", - ] - - mock_update_package_metadata_file.assert_called_once() - - mock_stage_patch_file.assert_called_once_with( - "/abs/path/to/filesdir/PATCHES.json" - ) - - mock_upload_changes.assert_called_once_with( - path_to_package_dir, branch, expected_commit_messages - ) - - mock_delete_repo.assert_called_once_with(path_to_package_dir, branch) + with mock.patch( + "update_chromeos_llvm_hash.PortagePackage.find_package_ebuild", + mock_find_package_ebuild, + ): + pkg = update_chromeos_llvm_hash.PortagePackage( + src_tree, "sys-devel/llvm" + ) + pkg.update( + update_chromeos_llvm_hash.LLVMVariant.current, + "beef3333", + 3333, + ) + @mock.patch.object(chroot, "VerifyChromeOSRoot") @mock.patch.object(chroot, "VerifyOutsideChroot") @mock.patch.object(get_llvm_hash, "GetLLVMHashAndVersionFromSVNOption") @mock.patch.object(update_chromeos_llvm_hash, "UpdatePackages") def testMainDefaults( - self, mock_update_packages, mock_gethash, mock_outside_chroot + self, + mock_update_packages, + mock_gethash, + mock_outside_chroot, + mock_chromeos_root, ): git_hash = "1234abcd" svn_version = 5678 @@ -1053,12 +729,18 @@ class UpdateLLVMHashTest(unittest.TestCase): extra_commit_msg=None, ) mock_outside_chroot.assert_called() + mock_chromeos_root.assert_called() + @mock.patch.object(chroot, "VerifyChromeOSRoot") @mock.patch.object(chroot, "VerifyOutsideChroot") @mock.patch.object(get_llvm_hash, "GetLLVMHashAndVersionFromSVNOption") @mock.patch.object(update_chromeos_llvm_hash, "UpdatePackages") def testMainLlvmNext( - self, mock_update_packages, mock_gethash, mock_outside_chroot + self, + mock_update_packages, + mock_gethash, + mock_outside_chroot, + mock_chromeos_root, ): git_hash = "1234abcd" svn_version = 5678 @@ -1089,12 +771,18 @@ class UpdateLLVMHashTest(unittest.TestCase): extra_commit_msg=None, ) mock_outside_chroot.assert_called() + mock_chromeos_root.assert_called() + @mock.patch.object(chroot, "VerifyChromeOSRoot") @mock.patch.object(chroot, "VerifyOutsideChroot") @mock.patch.object(get_llvm_hash, "GetLLVMHashAndVersionFromSVNOption") @mock.patch.object(update_chromeos_llvm_hash, "UpdatePackages") def testMainAllArgs( - self, mock_update_packages, mock_gethash, mock_outside_chroot + self, + mock_update_packages, + mock_gethash, + mock_outside_chroot, + mock_chromeos_root, ): packages_to_update = "test-packages/package1,test-libs/lib1" manifest_packages = "test-libs/lib1,test-libs/lib2" @@ -1140,6 +828,7 @@ class UpdateLLVMHashTest(unittest.TestCase): extra_commit_msg=None, ) mock_outside_chroot.assert_called() + mock_chromeos_root.assert_called() @mock.patch.object(subprocess, "check_output", return_value=None) @mock.patch.object(get_llvm_hash, "GetLLVMMajorVersion") diff --git a/llvm_tools/update_packages_and_run_tests.py b/llvm_tools/update_packages_and_run_tests.py index dc14b6de..598d9099 100755 --- a/llvm_tools/update_packages_and_run_tests.py +++ b/llvm_tools/update_packages_and_run_tests.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -11,6 +10,7 @@ import argparse import datetime import json import os +from pathlib import Path import subprocess import chroot @@ -26,10 +26,10 @@ def GetCommandLineArgs(): """Parses the command line for the command line arguments. Returns: - The log level to use when retrieving the LLVM hash or google3 LLVM version, - the chroot path to use for executing chroot commands, - a list of a package or packages to update their LLVM next hash, - and the LLVM version to use when retrieving the LLVM hash. + The log level to use when retrieving the LLVM hash or google3 LLVM + version, the chroot path to use for executing chroot commands, + a list of a package or packages to update their LLVM next hash, + and the LLVM version to use when retrieving the LLVM hash. """ # Default path to the chroot if a path is not specified. @@ -67,8 +67,8 @@ def GetCommandLineArgs(): "Otherwise, update LLVM_HASH", ) - # Add argument for the absolute path to the file that contains information on - # the previous tested svn version. + # Add argument for the absolute path to the file that contains information + # on the previous tested svn version. parser.add_argument( "--last_tested", help="the absolute path to the file that contains the last tested " @@ -94,14 +94,6 @@ def GetCommandLineArgs(): help="The reviewers for the package update changelist", ) - # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - "--verbose", - action="store_true", - help="display contents of a command to the terminal " - "(default: %(default)s)", - ) - subparsers = parser.add_subparsers(dest="subparser_name") subparser_names = [] # Testing with the tryjobs. @@ -168,13 +160,13 @@ def UnchangedSinceLastRun(last_tested_file, arg_dict): """Gets the arguments used for last run Args: - last_tested_file: The absolute path to the file that contains the - arguments for the last run. - arg_dict: The arguments used for this run. + last_tested_file: The absolute path to the file that contains the + arguments for the last run. + arg_dict: The arguments used for this run. Returns: - Return true if the arguments used for last run exist and are the - same as the arguments used for this run. Otherwise return false. + Return true if the arguments used for last run exist and are the + same as the arguments used for this run. Otherwise return false. """ if not last_tested_file: @@ -183,7 +175,7 @@ def UnchangedSinceLastRun(last_tested_file, arg_dict): # Get the last tested svn version if the file exists. last_arg_dict = None try: - with open(last_tested_file) as f: + with open(last_tested_file, encoding="utf-8") as f: last_arg_dict = json.load(f) except (IOError, ValueError): @@ -205,12 +197,12 @@ def AddReviewers(cl, reviewers, chroot_path): def AddLinksToCL(tests, cl, chroot_path): """Adds the test link(s) to the CL as a comment.""" - # NOTE: Invoking `cros_sdk` does not make each tryjob link appear on its own - # line, so invoking the `gerrit` command directly instead of using `cros_sdk` - # to do it for us. + # NOTE: Invoking `cros_sdk` does not make each tryjob link appear on its + # own line, so invoking the `gerrit` command directly instead of using + # `cros_sdk` to do it for us. # - # FIXME: Need to figure out why `cros_sdk` does not add each tryjob link as a - # newline. + # FIXME: Need to figure out why `cros_sdk` does not add each tryjob link as + # a newline. gerrit_abs_path = os.path.join(chroot_path, "chromite/bin/gerrit") links = ["Started the following tests:"] @@ -231,15 +223,15 @@ def GetTryJobCommand(change_list, extra_change_lists, options, builder): """Constructs the 'tryjob' command. Args: - change_list: The CL obtained from updating the packages. - extra_change_lists: Extra change lists that would like to be run alongside - the change list of updating the packages. - options: Options to be passed into the tryjob command. - builder: The builder to be passed into the tryjob command. + change_list: The CL obtained from updating the packages. + extra_change_lists: Extra change lists that would like to be run + alongside the change list of updating the packages. + options: Options to be passed into the tryjob command. + builder: The builder to be passed into the tryjob command. Returns: - The 'tryjob' command with the change list of updating the packages and - any extra information that was passed into the command line. + The 'tryjob' command with the change list of updating the packages and + any extra information that was passed into the command line. """ tryjob_cmd = ["cros", "tryjob", "--yes", "--json", "-g", "%d" % change_list] @@ -260,20 +252,20 @@ def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path): """Runs a tryjob/tryjobs. Args: - cl_number: The CL created by updating the packages. - extra_change_lists: Any extra change lists that would run alongside the CL - that was created by updating the packages ('cl_number'). - options: Any options to be passed into the 'tryjob' command. - builders: All the builders to run the 'tryjob' with. - chroot_path: The absolute path to the chroot. + cl_number: The CL created by updating the packages. + extra_change_lists: Any extra change lists that would run alongside the + CL that was created by updating the packages ('cl_number'). + options: Any options to be passed into the 'tryjob' command. + builders: All the builders to run the 'tryjob' with. + chroot_path: The absolute path to the chroot. Returns: - A list that contains stdout contents of each tryjob, where stdout is - information (a hashmap) about the tryjob. The hashmap also contains stderr - if there was an error when running a tryjob. + A list that contains stdout contents of each tryjob, where stdout is + information (a hashmap) about the tryjob. The hashmap also contains + stderr if there was an error when running a tryjob. Raises: - ValueError: Failed to submit a tryjob. + ValueError: Failed to submit a tryjob. """ # Contains the results of each builder. @@ -313,20 +305,20 @@ def StartRecipeBuilders( """Launch recipe builders. Args: - cl_number: The CL created by updating the packages. - extra_change_lists: Any extra change lists that would run alongside the CL - that was created by updating the packages ('cl_number'). - options: Any options to be passed into the 'tryjob' command. - builders: All the builders to run the 'tryjob' with. - chroot_path: The absolute path to the chroot. + cl_number: The CL created by updating the packages. + extra_change_lists: Any extra change lists that would run alongside the + CL that was created by updating the packages ('cl_number'). + options: Any options to be passed into the 'tryjob' command. + builders: All the builders to run the 'tryjob' with. + chroot_path: The absolute path to the chroot. Returns: - A list that contains stdout contents of each builder, where stdout is - information (a hashmap) about the tryjob. The hashmap also contains stderr - if there was an error when running a tryjob. + A list that contains stdout contents of each builder, where stdout is + information (a hashmap) about the tryjob. The hashmap also contains + stderr if there was an error when running a tryjob. Raises: - ValueError: Failed to start a builder. + ValueError: Failed to start a builder. """ # Contains the results of each builder. @@ -415,21 +407,23 @@ def main(): """Updates the packages' LLVM hash and run tests. Raises: - AssertionError: The script was run inside the chroot. + AssertionError: The script was run inside the chroot. """ chroot.VerifyOutsideChroot() args_output = GetCommandLineArgs() + chroot.VerifyChromeOSRoot(args_output.chroot_path) + svn_option = args_output.llvm_version git_hash, svn_version = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption( svn_option ) - # There is no need to run tryjobs when all the key parameters remain unchanged - # from last time. + # There is no need to run tryjobs when all the key parameters remain + # unchanged from last time. # If --last_tested is specified, check if the current run has the same # arguments last time --last_tested is used. @@ -455,7 +449,6 @@ def main(): llvm_variant = update_chromeos_llvm_hash.LLVMVariant.current if args_output.is_llvm_next: llvm_variant = update_chromeos_llvm_hash.LLVMVariant.next - update_chromeos_llvm_hash.verbose = args_output.verbose extra_commit_msg = None if args_output.subparser_name == "cq": cq_depend_msg = GetCQDependString(args_output.extra_change_lists) @@ -471,7 +464,7 @@ def main(): llvm_variant=llvm_variant, git_hash=git_hash, svn_version=svn_version, - chroot_path=args_output.chroot_path, + chroot_path=Path(args_output.chroot_path), mode=failure_modes.FailureModes.DISABLE_PATCHES, git_hash_source=svn_option, extra_commit_msg=extra_commit_msg, @@ -517,7 +510,7 @@ def main(): # If --last_tested is specified, record the arguments used if args_output.last_tested: - with open(args_output.last_tested, "w") as f: + with open(args_output.last_tested, "w", encoding="utf-8") as f: json.dump(arg_dict, f, indent=2) diff --git a/llvm_tools/update_packages_and_run_tests_unittest.py b/llvm_tools/update_packages_and_run_tests_unittest.py index fc65749f..e62ed04f 100755 --- a/llvm_tools/update_packages_and_run_tests_unittest.py +++ b/llvm_tools/update_packages_and_run_tests_unittest.py @@ -263,12 +263,14 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): @mock.patch.object(update_chromeos_llvm_hash, "UpdatePackages") @mock.patch.object(update_packages_and_run_tests, "GetCommandLineArgs") @mock.patch.object(get_llvm_hash, "GetLLVMHashAndVersionFromSVNOption") + @mock.patch.object(chroot, "VerifyChromeOSRoot") @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) @mock.patch.object(chroot, "GetChrootEbuildPaths") def testUpdatedLastTestedFileWithNewTestedRevision( self, mock_get_chroot_build_paths, mock_outside_chroot, + mock_chromeos_root, mock_get_hash_and_version, mock_get_commandline_args, mock_update_packages, @@ -336,6 +338,8 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): mock_outside_chroot.assert_called_once() + mock_chromeos_root.assert_called_once() + mock_get_commandline_args.assert_called_once() mock_get_hash_and_version.assert_called_once() diff --git a/llvm_tools/upload_lexan_crashes_to_forcey.py b/llvm_tools/upload_lexan_crashes_to_forcey.py deleted file mode 100755 index 885a88f6..00000000 --- a/llvm_tools/upload_lexan_crashes_to_forcey.py +++ /dev/null @@ -1,285 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2020 The ChromiumOS Authors -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Fetches and submits the latest test-cases from Lexan's crash bucket.""" - -import argparse -import contextlib -import datetime -import json -import logging -import os -import shutil -import subprocess -import sys -import tempfile -from typing import Generator, Iterable, List - - -gsurl_base = "gs://chrome-clang-crash-reports/v1" - - -def gsutil_ls(loc: str) -> List[str]: - results = subprocess.run( - ["gsutil.py", "ls", loc], - stdout=subprocess.PIPE, - check=True, - encoding="utf-8", - ) - return [l.strip() for l in results.stdout.splitlines()] - - -def gsurl_ls_last_numbers(url: str) -> List[int]: - return sorted(int(x.rstrip("/").split("/")[-1]) for x in gsutil_ls(url)) - - -def get_available_year_numbers() -> List[int]: - return gsurl_ls_last_numbers(gsurl_base) - - -def get_available_month_numbers(year: int) -> List[int]: - return gsurl_ls_last_numbers(f"{gsurl_base}/{year}") - - -def get_available_day_numbers(year: int, month: int) -> List[int]: - return gsurl_ls_last_numbers(f"{gsurl_base}/{year}/{month:02d}") - - -def get_available_test_case_urls(year: int, month: int, day: int) -> List[str]: - return gsutil_ls(f"{gsurl_base}/{year}/{month:02d}/{day:02d}") - - -def test_cases_on_or_after( - date: datetime.datetime, -) -> Generator[str, None, None]: - """Yields all test-cases submitted on or after the given date.""" - for year in get_available_year_numbers(): - if year < date.year: - continue - - for month in get_available_month_numbers(year): - if year == date.year and month < date.month: - continue - - for day in get_available_day_numbers(year, month): - when = datetime.date(year, month, day) - if when < date: - continue - - yield when, get_available_test_case_urls(year, month, day) - - -def to_ymd(date: datetime.date) -> str: - return date.strftime("%Y-%m-%d") - - -def from_ymd(date_str: str) -> datetime.date: - return datetime.datetime.strptime(date_str, "%Y-%m-%d").date() - - -def persist_state( - seen_urls: Iterable[str], state_file: str, current_date: datetime.date -): - tmp_state_file = state_file + ".tmp" - with open(tmp_state_file, "w", encoding="utf-8") as f: - json.dump( - { - "already_seen": sorted(seen_urls), - "most_recent_date": to_ymd(current_date), - }, - f, - ) - os.rename(tmp_state_file, state_file) - - -@contextlib.contextmanager -def temp_dir() -> Generator[str, None, None]: - loc = tempfile.mkdtemp("lexan-autosubmit") - try: - yield loc - finally: - shutil.rmtree(loc) - - -def download_and_unpack_test_case(gs_url: str, tempdir: str) -> None: - suffix = os.path.splitext(gs_url)[1] - target_name = "test_case" + suffix - target = os.path.join(tempdir, target_name) - subprocess.run(["gsutil.py", "cp", gs_url, target], check=True) - subprocess.run(["tar", "xaf", target_name], check=True, cwd=tempdir) - os.unlink(target) - - -def submit_test_case(gs_url: str, cr_tool: str) -> None: - logging.info("Submitting %s", gs_url) - with temp_dir() as tempdir: - download_and_unpack_test_case(gs_url, tempdir) - - # Sometimes (e.g., in - # gs://chrome-clang-crash-reports/v1/2020/03/27/ - # chromium.clang-ToTiOS-12754-GTXToolKit-2bfcde.tgz) - # we'll get `.crash` files. Unclear why, but let's filter them out anyway. - repro_files = [ - os.path.join(tempdir, x) - for x in os.listdir(tempdir) - if not x.endswith(".crash") - ] - assert len(repro_files) == 2, repro_files - if repro_files[0].endswith(".sh"): - sh_file, src_file = repro_files - assert not src_file.endswith(".sh"), repro_files - else: - src_file, sh_file = repro_files - assert sh_file.endswith(".sh"), repro_files - - # Peephole: lexan got a crash upload with a way old clang. Ignore it. - with open(sh_file, encoding="utf-8") as f: - if "Crash reproducer for clang version 9.0.0" in f.read(): - logging.warning( - "Skipping upload for %s; seems to be with an old clang", - gs_url, - ) - return - - subprocess.run( - [ - cr_tool, - "reduce", - "-stream=false", - "-wait=false", - "-note", - gs_url, - "-sh_file", - os.path.join(tempdir, sh_file), - "-src_file", - os.path.join(tempdir, src_file), - ], - check=True, - ) - - -def submit_new_test_cases( - last_seen_test_cases: Iterable[str], - earliest_date_to_check: datetime.date, - forcey: str, - state_file_path: str, -) -> None: - """Submits new test-cases to forcey. - - This will persist state after each test-case is submitted. - - Args: - last_seen_test_cases: test-cases which have been submitted already, and - should be skipped if seen again. - earliest_date_to_check: the earliest date we should consider test-cases - from. - forcey: path to the forcey binary. - state_file_path: path to our state file. - """ - # `all_test_cases_seen` is the union of all test-cases seen on this and prior - # invocations. It guarantees, in all cases we care about, that we won't - # submit the same test-case twice. `test_cases_seen_this_invocation` is - # persisted as "all of the test-cases we've seen on this and prior - # invocations" if we successfully submit _all_ test-cases. - # - # Since you can visualize the test-cases this script considers as a sliding - # window that only moves forward, if we saw a test-case on a prior iteration - # but no longer see it, we'll never see it again (since it fell out of our - # sliding window by being too old). Hence, keeping it around is - # pointless. - # - # We only persist this minimized set of test-cases if _everything_ succeeds, - # since if something fails below, there's a chance that we haven't revisited - # test-cases that we've already seen. - all_test_cases_seen = set(last_seen_test_cases) - test_cases_seen_this_invocation = [] - most_recent_date = earliest_date_to_check - for date, candidates in test_cases_on_or_after(earliest_date_to_check): - most_recent_date = max(most_recent_date, date) - - for url in candidates: - test_cases_seen_this_invocation.append(url) - if url in all_test_cases_seen: - continue - - all_test_cases_seen.add(url) - submit_test_case(url, forcey) - - # Persisting on each iteration of this loop isn't free, but it's the - # easiest way to not resubmit test-cases, and it's good to keep in mind - # that: - # - the state file will be small (<12KB, since it only keeps a few days - # worth of test-cases after the first run) - # - in addition to this, we're downloading+unzipping+reuploading multiple - # MB of test-case bytes. - # - # So comparatively, the overhead here probably isn't an issue. - persist_state( - all_test_cases_seen, state_file_path, most_recent_date - ) - - persist_state( - test_cases_seen_this_invocation, state_file_path, most_recent_date - ) - - -def main(argv: List[str]): - logging.basicConfig( - format=">> %(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: " - "%(message)s", - level=logging.INFO, - ) - - my_dir = os.path.dirname(os.path.abspath(__file__)) - - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--state_file", default=os.path.join(my_dir, "lexan-state.json") - ) - parser.add_argument( - "--last_date", - help="The earliest date that we care about. All test cases from here " - "on will be picked up. Format is YYYY-MM-DD.", - ) - parser.add_argument( - "--4c", dest="forcey", required=True, help="Path to a 4c client binary" - ) - opts = parser.parse_args(argv) - - forcey = opts.forcey - state_file = opts.state_file - last_date_str = opts.last_date - - os.makedirs(os.path.dirname(state_file), 0o755, exist_ok=True) - - if last_date_str is None: - with open(state_file, encoding="utf-8") as f: - data = json.load(f) - most_recent_date = from_ymd(data["most_recent_date"]) - submit_new_test_cases( - last_seen_test_cases=data["already_seen"], - # Note that we always subtract one day from this to avoid a race: - # uploads may appear slightly out-of-order (or builders may lag, or - # ...), so the last test-case uploaded for 2020/01/01 might appear - # _after_ the first test-case for 2020/01/02. Assuming that builders - # won't lag behind for over a day, the easiest way to handle this is to - # always check the previous and current days. - earliest_date_to_check=most_recent_date - - datetime.timedelta(days=1), - forcey=forcey, - state_file_path=state_file, - ) - else: - submit_new_test_cases( - last_seen_test_cases=(), - earliest_date_to_check=from_ymd(last_date_str), - forcey=forcey, - state_file_path=state_file, - ) - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/llvm_tools/upload_lexan_crashes_to_forcey_test.py b/llvm_tools/upload_lexan_crashes_to_forcey_test.py deleted file mode 100755 index 7238281a..00000000 --- a/llvm_tools/upload_lexan_crashes_to_forcey_test.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2020 The ChromiumOS Authors -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Tests for upload_lexan_crashes_to_forcey.""" - -import datetime -import os -import unittest -import unittest.mock - -import upload_lexan_crashes_to_forcey - - -class Test(unittest.TestCase): - """Tests for upload_lexan_crashes_to_forcey.""" - - def test_date_parsing_functions(self): - self.assertEqual( - datetime.date(2020, 2, 1), - upload_lexan_crashes_to_forcey.from_ymd("2020-02-01"), - ) - - @unittest.mock.patch( - "upload_lexan_crashes_to_forcey.test_cases_on_or_after", - return_value=( - ( - datetime.date(2020, 1, 1), - ("gs://test-case-1", "gs://test-case-1.1"), - ), - (datetime.date(2020, 1, 2), ("gs://test-case-2",)), - (datetime.date(2020, 1, 1), ("gs://test-case-3",)), - (datetime.date(2020, 1, 4), ("gs://test-case-4",)), - ), - ) - @unittest.mock.patch("upload_lexan_crashes_to_forcey.submit_test_case") - @unittest.mock.patch("upload_lexan_crashes_to_forcey.persist_state") - def test_new_test_case_submission_functions( - self, - persist_state_mock, - submit_test_case_mock, - test_cases_on_or_after_mock, - ): - forcey_path = "/path/to/4c" - real_state_file_path = "/path/to/state/file" - earliest_date = datetime.date(2020, 1, 1) - - persist_state_calls = [] - - # Since the set this gets is mutated, we need to copy it somehow. - def persist_state_side_effect( - test_cases_to_persist, state_file_path, most_recent_date - ): - self.assertEqual(state_file_path, real_state_file_path) - persist_state_calls.append( - (sorted(test_cases_to_persist), most_recent_date) - ) - - persist_state_mock.side_effect = persist_state_side_effect - - upload_lexan_crashes_to_forcey.submit_new_test_cases( - last_seen_test_cases=( - "gs://test-case-0", - "gs://test-case-1", - ), - earliest_date_to_check=earliest_date, - forcey=forcey_path, - state_file_path=real_state_file_path, - ) - - test_cases_on_or_after_mock.assert_called_once_with(earliest_date) - self.assertEqual( - submit_test_case_mock.call_args_list, - [ - unittest.mock.call("gs://test-case-1.1", forcey_path), - unittest.mock.call("gs://test-case-2", forcey_path), - unittest.mock.call("gs://test-case-3", forcey_path), - unittest.mock.call("gs://test-case-4", forcey_path), - ], - ) - - self.assertEqual( - persist_state_calls, - [ - ( - [ - "gs://test-case-0", - "gs://test-case-1", - "gs://test-case-1.1", - ], - datetime.date(2020, 1, 1), - ), - ( - [ - "gs://test-case-0", - "gs://test-case-1", - "gs://test-case-1.1", - "gs://test-case-2", - ], - datetime.date(2020, 1, 2), - ), - ( - [ - "gs://test-case-0", - "gs://test-case-1", - "gs://test-case-1.1", - "gs://test-case-2", - "gs://test-case-3", - ], - datetime.date(2020, 1, 2), - ), - ( - [ - "gs://test-case-0", - "gs://test-case-1", - "gs://test-case-1.1", - "gs://test-case-2", - "gs://test-case-3", - "gs://test-case-4", - ], - datetime.date(2020, 1, 4), - ), - ( - [ - "gs://test-case-1", - "gs://test-case-1.1", - "gs://test-case-2", - "gs://test-case-3", - "gs://test-case-4", - ], - datetime.date(2020, 1, 4), - ), - ], - ) - - @unittest.mock.patch( - "upload_lexan_crashes_to_forcey.download_and_unpack_test_case" - ) - @unittest.mock.patch("subprocess.run") - def test_test_case_submission_functions( - self, subprocess_run_mock, download_and_unpack_mock - ): - mock_gs_url = "gs://foo/bar/baz" - - def side_effect(gs_url: str, tempdir: str) -> None: - self.assertEqual(gs_url, mock_gs_url) - - with open(os.path.join(tempdir, "test_case.c"), "w") as f: - # All we need is an empty file here. - pass - - with open( - os.path.join(tempdir, "test_case.sh"), "w", encoding="utf-8" - ) as f: - f.write("# Crash reproducer for clang version 9.0.0 (...)\n") - f.write("clang something or other\n") - - download_and_unpack_mock.side_effect = side_effect - upload_lexan_crashes_to_forcey.submit_test_case(mock_gs_url, "4c") - subprocess_run_mock.assert_not_called() - - -if __name__ == "__main__": - unittest.main() diff --git a/orderfile/post_process_orderfile.py b/orderfile/post_process_orderfile.py index 5f52aa6f..d90c1af7 100755 --- a/orderfile/post_process_orderfile.py +++ b/orderfile/post_process_orderfile.py @@ -59,7 +59,13 @@ def run(c3_ordered_stream, chrome_nm_stream, output_stream): head_marker = "chrome_begin_ordered_code" tail_marker = "chrome_end_ordered_code" - c3_ordered_syms = [x.strip() for x in c3_ordered_stream.readlines()] + # Filter out $symbols which can come up in the orderfile as well. + # Linker ignores the whole orderfile if it finds $symb. + c3_ordered_syms = [ + x.strip() + for x in c3_ordered_stream.readlines() + if not x.strip().startswith("$") + ] all_chrome_syms = set(_parse_nm_output(chrome_nm_stream)) # Sort by name, so it's predictable. Otherwise, these should all land in the # same hugepage anyway, so order doesn't matter as much. diff --git a/orderfile/post_process_orderfile_test.py b/orderfile/post_process_orderfile_test.py index 60716d87..9dd574a8 100755 --- a/orderfile/post_process_orderfile_test.py +++ b/orderfile/post_process_orderfile_test.py @@ -29,6 +29,7 @@ def _write_nm_file(name): def _write_orderfile(name): with open(name, "w") as out: out.write("SymbolOrdered1\n") + out.write("$toignore.123\n") out.write("SymbolOrdered2\n") diff --git a/pgo_tools_rust/pgo_rust.py b/pgo_tools_rust/pgo_rust.py index 298c343f..fc693169 100755 --- a/pgo_tools_rust/pgo_rust.py +++ b/pgo_tools_rust/pgo_rust.py @@ -4,6 +4,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +# pylint: disable=line-too-long """Handle most aspects of creating and benchmarking PGO profiles for Rust. This is meant to be done at Rust uprev time. Ultimately profdata files need @@ -111,7 +112,7 @@ import re import shutil import subprocess import sys -from typing import Dict, List, Optional +from typing import cast, List, Mapping, Optional TARGET_TRIPLES = [ @@ -146,14 +147,14 @@ def run( args: List, *, indent: int = 4, - env: Optional[Dict[str, str]] = None, + env: Optional[Mapping[str, str]] = None, capture_stdout: bool = False, message: bool = True, ) -> Optional[str]: args = [str(arg) for arg in args] if env is None: - new_env = os.environ + new_env: Mapping[str, str] = os.environ else: new_env = os.environ.copy() new_env.update(env) @@ -199,8 +200,13 @@ def run( return ret +def get_command_output(args: List, **kwargs) -> str: + """Runs a command and returns its stdout and stderr as a string.""" + return cast(str, run(args, capture_stdout=True, **kwargs)) + + def get_rust_version() -> str: - s = run(["rustc", "--version"], capture_stdout=True) + s = get_command_output(["rustc", "--version"]) m = re.search(r"\d+\.\d+\.\d+", s) assert m is not None, repr(s) return m.group(0) @@ -214,9 +220,8 @@ def download_unpack_crate(*, crate_name: str, crate_version: str): local_path / f"{crate_name}-{crate_version}", ignore_errors=True ) with chdir(local_path): - run(["gsutil.py", "cp", f"gs:/{gs_path}", "."]) - run(["xz", "-d", f"{filename_no_extension}.tar.xz"]) - run(["tar", "xvf", f"{filename_no_extension}.tar"]) + run(["gsutil", "cp", f"gs:/{gs_path}", "."]) + run(["tar", "xaf", f"{filename_no_extension}.tar.xz"]) def build_crate( @@ -229,7 +234,7 @@ def build_crate( local_path = LOCAL_BASE / "crates" / f"{crate_name}-{crate_version}" with chdir(local_path): Path(".cargo").mkdir(exist_ok=True) - with open(".cargo/config.toml", "w") as f: + with open(".cargo/config.toml", "w", encoding="utf-8") as f: f.write( "\n".join( ( @@ -275,9 +280,10 @@ def build_rust( ): if use_frontend_profile or use_llvm_profile: - assert ( - not generate_frontend_profile and not generate_llvm_profile - ), "Can't build a compiler to both use profile information and generate it" + assert not generate_frontend_profile and not generate_llvm_profile, ( + "Can't build a compiler to both use profile information " + "and generate it" + ) assert ( not generate_frontend_profile or not generate_llvm_profile @@ -293,7 +299,9 @@ def build_rust( if use_llvm_profile: use += "rust_profile_llvm_use_local " - # -E to preserve our USE environment variable. + env_use = os.getenv("USE", "").rstrip() + use = (env_use + " " + use).strip() + # -E to preserve environment variables like USE, FEATURES, etc. run( ["sudo", "-E", "emerge", "dev-lang/rust", "dev-lang/rust-host"], env={"USE": use}, @@ -303,9 +311,9 @@ def build_rust( def merge_profdata(llvm_or_frontend, *, source_directory: Path, dest: Path): assert llvm_or_frontend in ("llvm", "frontend") - # The two `llvm-profdata` programs come from different LLVM versions, and may - # support different versions of the profdata format, so make sure to use the - # right one. + # The two `llvm-profdata` programs come from different LLVM versions, and + # may support different versions of the profdata format, so make sure to + # use the right one. llvm_profdata = ( "/usr/bin/llvm-profdata" if llvm_or_frontend == "llvm" @@ -328,12 +336,13 @@ def upload_file( *, source: Path, dest: PurePosixPath, public_read: bool = False ): if public_read: - run(["gsutil.py", "cp", "-a", "public-read", source, f"gs:/{dest}"]) + run(["gsutil", "cp", "-a", "public-read", source, f"gs:/{dest}"]) else: - run(["gsutil.py", "cp", source, f"gs:/{dest}"]) + run(["gsutil", "cp", source, f"gs:/{dest}"]) def maybe_download_crate(*, crate_name: str, crate_version: str): + """Downloads a crate if its download directory does not already exist.""" directory = LOCAL_BASE / "crates" / f"{crate_name}-{crate_version}" if directory.is_dir(): logging.info("Crate already downloaded") @@ -409,6 +418,10 @@ def generate(args): def benchmark_nopgo(args): + maybe_download_crate( + crate_name=args.bench_crate_name, crate_version=args.bench_crate_version + ) + logging.info("Building Rust, no PGO") build_rust() @@ -531,13 +544,13 @@ def upload_profdata(args): ) -def main(): +def main(argv: List[str]) -> int: logging.basicConfig( stream=sys.stdout, level=logging.NOTSET, format="%(message)s" ) parser = argparse.ArgumentParser( - prog=sys.argv[0], + prog=argv[0], description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -580,7 +593,8 @@ def main(): parser_benchmark_nopgo.add_argument( "--suffix", default="", - help="Suffix to distinguish benchmarks and profdata with identical rustc versions", + help="Suffix to distinguish benchmarks and profdata with identical " + "rustc versions", ) parser_benchmark_pgo = subparsers.add_parser( @@ -613,7 +627,8 @@ def main(): parser_benchmark_pgo.add_argument( "--suffix", default="", - help="Suffix to distinguish benchmarks and profdata with identical rustc versions", + help="Suffix to distinguish benchmarks and profdata with identical " + "rustc versions", ) parser_upload_profdata = subparsers.add_parser( @@ -633,10 +648,11 @@ def main(): parser_upload_profdata.add_argument( "--suffix", default="", - help="Suffix to distinguish benchmarks and profdata with identical rustc versions", + help="Suffix to distinguish benchmarks and profdata with identical " + "rustc versions", ) - args = parser.parse_args() + args = parser.parse_args(argv[1:]) (LOCAL_BASE / "crates").mkdir(parents=True, exist_ok=True) (LOCAL_BASE / "llvm-profraw").mkdir(parents=True, exist_ok=True) @@ -649,4 +665,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + sys.exit(main(sys.argv)) diff --git a/rust_tools/copy_rust_bootstrap.py b/rust_tools/copy_rust_bootstrap.py index 5da8007f..c32d1fd9 100755 --- a/rust_tools/copy_rust_bootstrap.py +++ b/rust_tools/copy_rust_bootstrap.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -# Copyright 2022 The ChromiumOS Authors. +# Copyright 2022 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Copies rust-bootstrap artifacts from an SDK build to localmirror. We use localmirror to host these artifacts, but they've changed a bit over -time, so simply `gsutil.py cp $FROM $TO` doesn't work. This script allows the +time, so simply `gsutil cp $FROM $TO` doesn't work. This script allows the convenience of the old `cp` command. """ @@ -21,172 +21,183 @@ import tempfile from typing import List -_LOCALMIRROR_ROOT = 'gs://chromeos-localmirror/distfiles/' +_LOCALMIRROR_ROOT = "gs://chromeos-localmirror/distfiles/" def _is_in_chroot() -> bool: - return Path('/etc/cros_chroot_version').exists() + return Path("/etc/cros_chroot_version").exists() def _ensure_pbzip2_is_installed(): - if shutil.which('pbzip2'): - return + if shutil.which("pbzip2"): + return - logging.info('Auto-installing pbzip2...') - subprocess.run(['sudo', 'emerge', '-G', 'pbzip2'], check=True) + logging.info("Auto-installing pbzip2...") + subprocess.run(["sudo", "emerge", "-g", "pbzip2"], check=True) def _determine_target_path(sdk_path: str) -> str: - """Determine where `sdk_path` should sit in localmirror.""" - gs_prefix = 'gs://' - if not sdk_path.startswith(gs_prefix): - raise ValueError(f'Invalid GS path: {sdk_path!r}') + """Determine where `sdk_path` should sit in localmirror.""" + gs_prefix = "gs://" + if not sdk_path.startswith(gs_prefix): + raise ValueError(f"Invalid GS path: {sdk_path!r}") - file_name = Path(sdk_path[len(gs_prefix):]).name - return _LOCALMIRROR_ROOT + file_name + file_name = Path(sdk_path[len(gs_prefix) :]).name + return _LOCALMIRROR_ROOT + file_name def _download(remote_path: str, local_file: Path): - """Downloads the given gs:// path to the given local file.""" - logging.info('Downloading %s -> %s', remote_path, local_file) - subprocess.run( - ['gsutil.py', 'cp', remote_path, - str(local_file)], - check=True, - ) - - -def _debinpkgify(binpkg_file: Path) -> Path: - """Converts a binpkg into the files it installs. - - Note that this function makes temporary files in the same directory as - `binpkg_file`. It makes no attempt to clean them up. - """ - logging.info('Converting %s from a binpkg...', binpkg_file) - - # The SDK builder produces binary packages: - # https://wiki.gentoo.org/wiki/Binary_package_guide - # - # Which means that `binpkg_file` is in the XPAK format. We want to split - # that out, and recompress it from zstd (which is the compression format - # that CrOS uses) to bzip2 (which is what we've historically used, and - # which is what our ebuild expects). - tmpdir = binpkg_file.parent - - def _mkstemp(suffix=None) -> str: - fd, file_path = tempfile.mkstemp(dir=tmpdir, suffix=suffix) - os.close(fd) - return Path(file_path) - - # First, split the actual artifacts that land in the chroot out to - # `temp_file`. - artifacts_file = _mkstemp() - logging.info('Extracting artifacts from %s into %s...', binpkg_file, - artifacts_file) - with artifacts_file.open('wb') as f: + """Downloads the given gs:// path to the given local file.""" + logging.info("Downloading %s -> %s", remote_path, local_file) subprocess.run( - [ - 'qtbz2', - '-s', - '-t', - '-O', - str(binpkg_file), - ], + ["gsutil", "cp", remote_path, str(local_file)], check=True, - stdout=f, ) - decompressed_artifacts_file = _mkstemp() - decompressed_artifacts_file.unlink() - logging.info('Decompressing artifacts from %s to %s...', artifacts_file, - decompressed_artifacts_file) - subprocess.run( - [ - 'zstd', - '-d', - str(artifacts_file), - '-o', - str(decompressed_artifacts_file), - ], - check=True, - ) - - # Finally, recompress it as a tbz2. - tbz2_file = _mkstemp('.tbz2') - logging.info( - 'Recompressing artifacts from %s to %s (this may take a while)...', - decompressed_artifacts_file, tbz2_file) - with tbz2_file.open('wb') as f: + +def _debinpkgify(binpkg_file: Path) -> Path: + """Converts a binpkg into the files it installs. + + Note that this function makes temporary files in the same directory as + `binpkg_file`. It makes no attempt to clean them up. + """ + logging.info("Converting %s from a binpkg...", binpkg_file) + + # The SDK builder produces binary packages: + # https://wiki.gentoo.org/wiki/Binary_package_guide + # + # Which means that `binpkg_file` is in the XPAK format. We want to split + # that out, and recompress it from zstd (which is the compression format + # that CrOS uses) to bzip2 (which is what we've historically used, and + # which is what our ebuild expects). + tmpdir = binpkg_file.parent + + def _mkstemp(suffix=None) -> str: + fd, file_path = tempfile.mkstemp(dir=tmpdir, suffix=suffix) + os.close(fd) + return Path(file_path) + + # First, split the actual artifacts that land in the chroot out to + # `temp_file`. + artifacts_file = _mkstemp() + logging.info( + "Extracting artifacts from %s into %s...", binpkg_file, artifacts_file + ) + with artifacts_file.open("wb") as f: + subprocess.run( + [ + "qtbz2", + "-s", + "-t", + "-O", + str(binpkg_file), + ], + check=True, + stdout=f, + ) + + decompressed_artifacts_file = _mkstemp() + decompressed_artifacts_file.unlink() + logging.info( + "Decompressing artifacts from %s to %s...", + artifacts_file, + decompressed_artifacts_file, + ) subprocess.run( [ - 'pbzip2', - '-9', - '-c', + "zstd", + "-d", + str(artifacts_file), + "-o", str(decompressed_artifacts_file), ], check=True, - stdout=f, ) - return tbz2_file + + # Finally, recompress it as a tbz2. + tbz2_file = _mkstemp(".tbz2") + logging.info( + "Recompressing artifacts from %s to %s (this may take a while)...", + decompressed_artifacts_file, + tbz2_file, + ) + with tbz2_file.open("wb") as f: + subprocess.run( + [ + "pbzip2", + "-9", + "-c", + str(decompressed_artifacts_file), + ], + check=True, + stdout=f, + ) + return tbz2_file def _upload(local_file: Path, remote_path: str, force: bool): - """Uploads the local file to the given gs:// path.""" - logging.info('Uploading %s -> %s', local_file, remote_path) - cmd_base = ['gsutil.py', 'cp', '-a', 'public-read'] - if not force: - cmd_base.append('-n') - subprocess.run( - cmd_base + [str(local_file), remote_path], - check=True, - stdin=subprocess.DEVNULL, - ) + """Uploads the local file to the given gs:// path.""" + logging.info("Uploading %s -> %s", local_file, remote_path) + cmd_base = ["gsutil", "cp", "-a", "public-read"] + if not force: + cmd_base.append("-n") + subprocess.run( + cmd_base + [str(local_file), remote_path], + check=True, + stdin=subprocess.DEVNULL, + ) def main(argv: List[str]): - logging.basicConfig( - format='>> %(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: ' - '%(message)s', - level=logging.INFO, - ) - - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - - parser.add_argument( - 'sdk_artifact', - help='Path to the SDK rust-bootstrap artifact to copy. e.g., ' - 'gs://chromeos-prebuilt/host/amd64/amd64-host/' - 'chroot-2022.07.12.134334/packages/dev-lang/' - 'rust-bootstrap-1.59.0.tbz2.') - parser.add_argument( - '-n', - '--dry-run', - action='store_true', - help='Do everything except actually uploading the artifact.') - parser.add_argument( - '--force', - action='store_true', - help='Upload the artifact even if one exists in localmirror already.') - opts = parser.parse_args(argv) - - if not _is_in_chroot(): - parser.error('Run me from within the chroot.') - _ensure_pbzip2_is_installed() - - target_path = _determine_target_path(opts.sdk_artifact) - with tempfile.TemporaryDirectory() as tempdir: - download_path = Path(tempdir) / 'sdk_artifact' - _download(opts.sdk_artifact, download_path) - file_to_upload = _debinpkgify(download_path) - if opts.dry_run: - logging.info('--dry-run specified; skipping upload of %s to %s', - file_to_upload, target_path) - else: - _upload(file_to_upload, target_path, opts.force) - - -if __name__ == '__main__': - main(sys.argv[1:]) + logging.basicConfig( + format=">> %(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: " + "%(message)s", + level=logging.INFO, + ) + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "sdk_artifact", + help="Path to the SDK rust-bootstrap artifact to copy. e.g., " + "gs://chromeos-prebuilt/host/amd64/amd64-host/" + "chroot-2022.07.12.134334/packages/dev-lang/" + "rust-bootstrap-1.59.0.tbz2.", + ) + parser.add_argument( + "-n", + "--dry-run", + action="store_true", + help="Do everything except actually uploading the artifact.", + ) + parser.add_argument( + "--force", + action="store_true", + help="Upload the artifact even if one exists in localmirror already.", + ) + opts = parser.parse_args(argv) + + if not _is_in_chroot(): + parser.error("Run me from within the chroot.") + _ensure_pbzip2_is_installed() + + target_path = _determine_target_path(opts.sdk_artifact) + with tempfile.TemporaryDirectory() as tempdir: + download_path = Path(tempdir) / "sdk_artifact" + _download(opts.sdk_artifact, download_path) + file_to_upload = _debinpkgify(download_path) + if opts.dry_run: + logging.info( + "--dry-run specified; skipping upload of %s to %s", + file_to_upload, + target_path, + ) + else: + _upload(file_to_upload, target_path, opts.force) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py index 382d991a..e9113ea7 100755 --- a/rust_tools/rust_uprev.py +++ b/rust_tools/rust_uprev.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -36,34 +35,98 @@ import re import shlex import shutil import subprocess -import sys -from typing import Any, Callable, Dict, List, NamedTuple, Optional, T, Tuple +from typing import ( + Any, + Callable, + Dict, + List, + NamedTuple, + Optional, + Protocol, + Tuple, + TypeVar, + Union, +) +import urllib.request from llvm_tools import chroot from llvm_tools import git +from pgo_tools_rust import pgo_rust + + +T = TypeVar("T") +Command = List[Union[str, os.PathLike]] +PathOrStr = Union[str, os.PathLike] + + +class RunStepFn(Protocol): + """Protocol that corresponds to run_step's type. + + This can be used as the type of a function parameter that accepts + run_step as its value. + """ + + def __call__( + self, + step_name: str, + step_fn: Callable[[], T], + result_from_json: Optional[Callable[[Any], T]] = None, + result_to_json: Optional[Callable[[T], Any]] = None, + ) -> T: + ... EQUERY = "equery" -GSUTIL = "gsutil.py" +GPG = "gpg" +GSUTIL = "gsutil" MIRROR_PATH = "gs://chromeos-localmirror/distfiles" EBUILD_PREFIX = Path("/mnt/host/source/src/third_party/chromiumos-overlay") +CROS_RUSTC_ECLASS = EBUILD_PREFIX / "eclass/cros-rustc.eclass" +# Keyserver to use with GPG. Not all keyservers have Rust's signing key; +# this must be set to a keyserver that does. +GPG_KEYSERVER = "keyserver.ubuntu.com" RUST_PATH = Path(EBUILD_PREFIX, "dev-lang", "rust") +# This is the signing key used by upstream Rust as of 2023-08-09. +# If the project switches to a different key, this will have to be updated. +# We require the key to be updated manually so that we have an opportunity +# to verify that the key change is legitimate. +RUST_SIGNING_KEY = "85AB96E6FA1BE5FE" +RUST_SRC_BASE_URI = "https://static.rust-lang.org/dist/" + +class SignatureVerificationError(Exception): + """Error that indicates verification of a downloaded file failed. -def get_command_output(command: List[str], *args, **kwargs) -> str: + Attributes: + message: explanation of why the verification failed. + path: the path to the file whose integrity was being verified. + """ + + def __init__(self, message: str, path: Path): + super(SignatureVerificationError, self).__init__() + self.message = message + self.path = path + + +def get_command_output(command: Command, *args, **kwargs) -> str: return subprocess.check_output( command, encoding="utf-8", *args, **kwargs ).strip() -def get_command_output_unchecked(command: List[str], *args, **kwargs) -> str: +def get_command_output_unchecked(command: Command, *args, **kwargs) -> str: + # pylint: disable=subprocess-run-check return subprocess.run( command, - check=False, - stdout=subprocess.PIPE, - encoding="utf-8", *args, - **kwargs, + **dict( + { + "check": False, + "stdout": subprocess.PIPE, + "encoding": "utf-8", + }, + **kwargs, + ), ).stdout.strip() @@ -109,6 +172,14 @@ class RustVersion(NamedTuple): ) +class PreparedUprev(NamedTuple): + """Container for the information returned by prepare_uprev.""" + + template_version: RustVersion + ebuild_path: Path + bootstrap_version: RustVersion + + def compute_rustc_src_name(version: RustVersion) -> str: return f"rustc-{version}-src.tar.gz" @@ -117,7 +188,7 @@ def compute_rust_bootstrap_prebuilt_name(version: RustVersion) -> str: return f"rust-bootstrap-{version}.tbz2" -def find_ebuild_for_package(name: str) -> os.PathLike: +def find_ebuild_for_package(name: str) -> str: """Returns the path to the ebuild for the named package.""" return get_command_output([EQUERY, "w", name]) @@ -303,7 +374,7 @@ def parse_commandline_args() -> argparse.Namespace: def prepare_uprev( rust_version: RustVersion, template: Optional[RustVersion] -) -> Optional[Tuple[RustVersion, str, RustVersion]]: +) -> Optional[PreparedUprev]: if template is None: ebuild_path = find_ebuild_for_package("rust") ebuild_name = os.path.basename(ebuild_path) @@ -329,32 +400,11 @@ def prepare_uprev( ) logging.info("rust-bootstrap version is %s", bootstrap_version) - return template_version, ebuild_path, bootstrap_version - - -def copy_patches( - directory: Path, template_version: RustVersion, new_version: RustVersion -) -> None: - patch_path = directory / "files" - prefix = "%s-%s-" % (directory.name, template_version) - new_prefix = "%s-%s-" % (directory.name, new_version) - for f in os.listdir(patch_path): - if not f.startswith(prefix): - continue - logging.info("Copy patch %s to new version", f) - new_name = f.replace(str(template_version), str(new_version)) - shutil.copyfile( - os.path.join(patch_path, f), - os.path.join(patch_path, new_name), - ) - - subprocess.check_call( - ["git", "add", f"{new_prefix}*.patch"], cwd=patch_path - ) + return PreparedUprev(template_version, Path(ebuild_path), bootstrap_version) def create_ebuild( - template_ebuild: str, pkgatom: str, new_version: RustVersion + template_ebuild: PathOrStr, pkgatom: str, new_version: RustVersion ) -> str: filename = f"{Path(pkgatom).name}-{new_version}.ebuild" ebuild = EBUILD_PREFIX.joinpath(f"{pkgatom}/{filename}") @@ -363,11 +413,38 @@ def create_ebuild( return str(ebuild) +def set_include_profdata_src(ebuild_path: os.PathLike, include: bool) -> None: + """Changes an ebuild file to include or omit profile data from SRC_URI. + + If include is True, the ebuild file will be rewritten to include + profile data in SRC_URI. + + If include is False, the ebuild file will be rewritten to omit profile + data from SRC_URI. + """ + if include: + old = "" + new = "yes" + else: + old = "yes" + new = "" + contents = Path(ebuild_path).read_text(encoding="utf-8") + contents, subs = re.subn( + f"^INCLUDE_PROFDATA_IN_SRC_URI={old}$", + f"INCLUDE_PROFDATA_IN_SRC_URI={new}", + contents, + flags=re.MULTILINE, + ) + # We expect exactly one substitution. + assert subs == 1, "Failed to update INCLUDE_PROFDATA_IN_SRC_URI" + Path(ebuild_path).write_text(contents, encoding="utf-8") + + def update_bootstrap_ebuild(new_bootstrap_version: RustVersion) -> None: old_ebuild = find_ebuild_path(rust_bootstrap_path(), "rust-bootstrap") - m = re.match(r"^rust-bootstrap-(\d+).(\d+).(\d+)", old_ebuild.name) + m = re.match(r"^rust-bootstrap-(\d+.\d+.\d+)", old_ebuild.name) assert m, old_ebuild.name - old_version = RustVersion(m.group(1), m.group(2), m.group(3)) + old_version = RustVersion.parse(m.group(1)) new_ebuild = old_ebuild.parent.joinpath( f"rust-bootstrap-{new_bootstrap_version}.ebuild" ) @@ -383,9 +460,10 @@ def update_bootstrap_ebuild(new_bootstrap_version: RustVersion) -> None: def update_bootstrap_version( - path: str, new_bootstrap_version: RustVersion + path: PathOrStr, new_bootstrap_version: RustVersion ) -> None: - contents = open(path, encoding="utf-8").read() + path = Path(path) + contents = path.read_text(encoding="utf-8") contents, subs = re.subn( r"^BOOTSTRAP_VERSION=.*$", 'BOOTSTRAP_VERSION="%s"' % (new_bootstrap_version,), @@ -394,7 +472,7 @@ def update_bootstrap_version( ) if not subs: raise RuntimeError(f"BOOTSTRAP_VERSION not found in {path}") - open(path, "w", encoding="utf-8").write(contents) + path.write_text(contents, encoding="utf-8") logging.info("Rust BOOTSTRAP_VERSION updated to %s", new_bootstrap_version) @@ -426,7 +504,7 @@ def fetch_distfile_from_mirror(name: str) -> None: """ mirror_file = MIRROR_PATH + "/" + name local_file = Path(get_distdir(), name) - cmd = [GSUTIL, "cp", mirror_file, local_file] + cmd: Command = [GSUTIL, "cp", mirror_file, local_file] logging.info("Running %r", cmd) rc = subprocess.call(cmd) if rc != 0: @@ -499,12 +577,116 @@ def fetch_rust_distfiles(version: RustVersion) -> None: fetch_distfile_from_mirror(compute_rustc_src_name(version)) -def get_distdir() -> os.PathLike: +def fetch_rust_src_from_upstream(uri: str, local_path: Path) -> None: + """Fetches Rust sources from upstream. + + This downloads the source distribution and the .asc file + containing the signatures. It then verifies that the sources + have the expected signature and have been signed by + the expected key. + """ + subprocess.run( + [GPG, "--keyserver", GPG_KEYSERVER, "--recv-keys", RUST_SIGNING_KEY], + check=True, + ) + subprocess.run( + [GPG, "--keyserver", GPG_KEYSERVER, "--refresh-keys", RUST_SIGNING_KEY], + check=True, + ) + asc_uri = uri + ".asc" + local_asc_path = Path(local_path.parent, local_path.name + ".asc") + logging.info("Fetching %s", uri) + urllib.request.urlretrieve(uri, local_path) + logging.info("%s fetched", uri) + + # Raise SignatureVerificationError if we cannot get the signature. + try: + logging.info("Fetching %s", asc_uri) + urllib.request.urlretrieve(asc_uri, local_asc_path) + logging.info("%s fetched", asc_uri) + except Exception as e: + raise SignatureVerificationError( + f"error fetching signature file {asc_uri}", + local_path, + ) from e + + # Raise SignatureVerificationError if verifying the signature + # failed. + try: + output = get_command_output( + [GPG, "--verify", "--status-fd", "1", local_asc_path] + ) + except subprocess.CalledProcessError as e: + raise SignatureVerificationError( + f"error verifying signature. GPG output:\n{e.stdout}", + local_path, + ) from e + + # Raise SignatureVerificationError if the file was not signed + # with the expected key. + if f"GOODSIG {RUST_SIGNING_KEY}" not in output: + message = f"GOODSIG {RUST_SIGNING_KEY} not found in output" + if f"REVKEYSIG {RUST_SIGNING_KEY}" in output: + message = "signing key has been revoked" + elif f"EXPKEYSIG {RUST_SIGNING_KEY}" in output: + message = "signing key has expired" + elif f"EXPSIG {RUST_SIGNING_KEY}" in output: + message = "signature has expired" + raise SignatureVerificationError( + f"{message}. GPG output:\n{output}", + local_path, + ) + + +def get_distdir() -> str: """Returns portage's distdir.""" return get_command_output(["portageq", "distdir"]) -def update_manifest(ebuild_file: os.PathLike) -> None: +def mirror_has_file(name: str) -> bool: + """Checks if the mirror has the named file.""" + mirror_file = MIRROR_PATH + "/" + name + cmd: Command = [GSUTIL, "ls", mirror_file] + proc = subprocess.run( + cmd, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + ) + if "URLs matched no objects" in proc.stdout: + return False + elif proc.returncode == 0: + return True + + raise Exception( + "Unexpected result from gsutil ls:" + f" rc {proc.returncode} output:\n{proc.stdout}" + ) + + +def mirror_rust_source(version: RustVersion) -> None: + """Ensures source code for a Rust version is on the local mirror. + + If the source code is not found on the mirror, it is fetched + from upstream, its integrity is verified, and it is uploaded + to the mirror. + """ + filename = compute_rustc_src_name(version) + if mirror_has_file(filename): + logging.info("%s is present on the mirror", filename) + return + uri = f"{RUST_SRC_BASE_URI}{filename}" + local_path = Path(get_distdir()) / filename + mirror_path = f"{MIRROR_PATH}/{filename}" + fetch_rust_src_from_upstream(uri, local_path) + subprocess.run( + [GSUTIL, "cp", "-a", "public-read", local_path, mirror_path], + check=True, + ) + + +def update_manifest(ebuild_file: PathOrStr) -> None: """Updates the MANIFEST for the ebuild at the given path.""" ebuild = Path(ebuild_file) ebuild_actions(ebuild.parent.name, ["manifest"]) @@ -596,28 +778,57 @@ def perform_step( return val -def prepare_uprev_from_json( - obj: Any, -) -> Optional[Tuple[RustVersion, str, RustVersion]]: +def prepare_uprev_from_json(obj: Any) -> Optional[PreparedUprev]: if not obj: return None version, ebuild_path, bootstrap_version = obj - return RustVersion(*version), ebuild_path, RustVersion(*bootstrap_version) + return PreparedUprev( + RustVersion(*version), + Path(ebuild_path), + RustVersion(*bootstrap_version), + ) + + +def prepare_uprev_to_json( + prepared_uprev: Optional[PreparedUprev], +) -> Optional[Tuple[RustVersion, str, RustVersion]]: + if prepared_uprev is None: + return None + return ( + prepared_uprev.template_version, + str(prepared_uprev.ebuild_path), + prepared_uprev.bootstrap_version, + ) def create_rust_uprev( rust_version: RustVersion, maybe_template_version: Optional[RustVersion], skip_compile: bool, - run_step: Callable[[], T], + run_step: RunStepFn, ) -> None: - template_version, template_ebuild, old_bootstrap_version = run_step( + prepared = run_step( "prepare uprev", lambda: prepare_uprev(rust_version, maybe_template_version), result_from_json=prepare_uprev_from_json, + result_to_json=prepare_uprev_to_json, ) - if template_ebuild is None: + if prepared is None: return + template_version, template_ebuild, old_bootstrap_version = prepared + + run_step( + "mirror bootstrap sources", + lambda: mirror_rust_source( + template_version, + ), + ) + run_step( + "mirror rust sources", + lambda: mirror_rust_source( + rust_version, + ), + ) # The fetch steps will fail (on purpose) if the files they check for # are not available on the mirror. To make them pass, fetch the @@ -644,13 +855,11 @@ def create_rust_uprev( ) run_step( "update bootstrap version", - lambda: update_bootstrap_version( - EBUILD_PREFIX.joinpath("eclass/cros-rustc.eclass"), template_version - ), + lambda: update_bootstrap_version(CROS_RUSTC_ECLASS, template_version), ) run_step( - "copy patches", - lambda: copy_patches(RUST_PATH, template_version, rust_version), + "turn off profile data sources in cros-rustc.eclass", + lambda: set_include_profdata_src(CROS_RUSTC_ECLASS, include=False), ) template_host_ebuild = EBUILD_PREFIX.joinpath( f"dev-lang/rust-host/rust-host-{template_version}.ebuild" @@ -673,6 +882,26 @@ def create_rust_uprev( "update target manifest to add new version", lambda: update_manifest(Path(target_file)), ) + run_step( + "generate profile data for rustc", + lambda: pgo_rust.main(["pgo_rust", "generate"]), + ) + run_step( + "upload profile data for rustc", + lambda: pgo_rust.main(["pgo_rust", "upload-profdata"]), + ) + run_step( + "turn on profile data sources in cros-rustc.eclass", + lambda: set_include_profdata_src(CROS_RUSTC_ECLASS, include=True), + ) + run_step( + "update host manifest to add profile data", + lambda: update_manifest(Path(host_file)), + ) + run_step( + "update target manifest to add profile data", + lambda: update_manifest(Path(target_file)), + ) if not skip_compile: run_step("build packages", lambda: rebuild_packages(rust_version)) run_step( @@ -755,16 +984,16 @@ def rebuild_packages(version: RustVersion): raise -def remove_ebuild_version(path: os.PathLike, name: str, version: RustVersion): +def remove_ebuild_version(path: PathOrStr, name: str, version: RustVersion): """Remove the specified version of an ebuild. Removes {path}/{name}-{version}.ebuild and {path}/{name}-{version}-*.ebuild using git rm. Args: - path: The directory in which the ebuild files are. - name: The name of the package (e.g. 'rust'). - version: The version of the ebuild to remove. + path: The directory in which the ebuild files are. + name: The name of the package (e.g. 'rust'). + version: The version of the ebuild to remove. """ path = Path(path) pattern = f"{name}-{version}-*.ebuild" @@ -780,12 +1009,13 @@ def remove_ebuild_version(path: os.PathLike, name: str, version: RustVersion): remove_files(m.name, path) -def remove_files(filename: str, path: str) -> None: +def remove_files(filename: PathOrStr, path: PathOrStr) -> None: subprocess.check_call(["git", "rm", filename], cwd=path) def remove_rust_bootstrap_version( - version: RustVersion, run_step: Callable[[], T] + version: RustVersion, + run_step: RunStepFn, ) -> None: run_step( "remove old bootstrap ebuild", @@ -801,7 +1031,8 @@ def remove_rust_bootstrap_version( def remove_rust_uprev( - rust_version: Optional[RustVersion], run_step: Callable[[], T] + rust_version: Optional[RustVersion], + run_step: RunStepFn, ) -> None: def find_desired_rust_version() -> RustVersion: if rust_version: @@ -817,10 +1048,6 @@ def remove_rust_uprev( result_from_json=find_desired_rust_version_from_json, ) run_step( - "remove patches", - lambda: remove_files(f"files/rust-{delete_version}-*.patch", RUST_PATH), - ) - run_step( "remove target ebuild", lambda: remove_ebuild_version(RUST_PATH, "rust", delete_version), ) @@ -963,7 +1190,8 @@ def main() -> None: elif args.subparser_name == "remove-bootstrap": remove_rust_bootstrap_version(args.version, run_step) else: - # If you have added more subparser_name, please also add the handlers above + # If you have added more subparser_name, please also add the handlers + # above assert args.subparser_name == "roll" run_step("create new repo", lambda: create_new_repo(args.uprev)) if not args.skip_cross_compiler: @@ -972,10 +1200,9 @@ def main() -> None: args.uprev, args.template, args.skip_compile, run_step ) remove_rust_uprev(args.remove, run_step) - bootstrap_version = prepare_uprev_from_json( - completed_steps["prepare uprev"] - )[2] - remove_rust_bootstrap_version(bootstrap_version, run_step) + prepared = prepare_uprev_from_json(completed_steps["prepare uprev"]) + assert prepared is not None, "no prepared uprev decoded from JSON" + remove_rust_bootstrap_version(prepared.bootstrap_version, run_step) if not args.no_upload: run_step( "create rust uprev CL", lambda: create_new_commit(args.uprev) @@ -983,4 +1210,4 @@ def main() -> None: if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py index 0c4c91ed..3f8c829c 100755 --- a/rust_tools/rust_uprev_test.py +++ b/rust_tools/rust_uprev_test.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -25,6 +24,23 @@ def _fail_command(cmd, *_args, **_kwargs): raise err +def start_mock(obj, *args, **kwargs): + """Creates a patcher, starts it, and registers a cleanup to stop it. + + Args: + obj: + the object to attach the cleanup to + *args: + passed to mock.patch() + **kwargs: + passsed to mock.patch() + """ + patcher = mock.patch(*args, **kwargs) + val = patcher.start() + obj.addCleanup(patcher.stop) + return val + + class FetchDistfileTest(unittest.TestCase): """Tests rust_uprev.fetch_distfile_from_mirror()""" @@ -79,6 +95,151 @@ class FetchDistfileTest(unittest.TestCase): ) +class FetchRustSrcFromUpstreamTest(unittest.TestCase): + """Tests for rust_uprev.fetch_rust_src_from_upstream.""" + + def setUp(self) -> None: + self._mock_get_distdir = start_mock( + self, + "rust_uprev.get_distdir", + return_value="/fake/distfiles", + ) + + self._mock_gpg = start_mock( + self, + "subprocess.run", + side_effect=self.fake_gpg, + ) + + self._mock_urlretrieve = start_mock( + self, + "urllib.request.urlretrieve", + side_effect=self.fake_urlretrieve, + ) + + self._mock_rust_signing_key = start_mock( + self, + "rust_uprev.RUST_SIGNING_KEY", + "1234567", + ) + + @staticmethod + def fake_urlretrieve(src: str, dest: Path) -> None: + pass + + @staticmethod + def fake_gpg(cmd, **_kwargs): + val = mock.Mock() + val.returncode = 0 + val.stdout = "" + if "--verify" in cmd: + val.stdout = "GOODSIG 1234567" + return val + + def test_success(self): + with mock.patch("rust_uprev.GPG", "gnupg"): + rust_uprev.fetch_rust_src_from_upstream( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ) + self._mock_urlretrieve.has_calls( + [ + ( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ), + ( + "fakehttps://rustc-1.60.3-src.tar.gz.asc", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz.asc"), + ), + ] + ) + self._mock_gpg.has_calls( + [ + (["gnupg", "--refresh-keys", "1234567"], {"check": True}), + ] + ) + + def test_no_signature_file(self): + def _urlretrieve(src, dest): + if src.endswith(".asc"): + raise Exception("404 not found") + return self.fake_urlretrieve(src, dest) + + self._mock_urlretrieve.side_effect = _urlretrieve + + with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx: + rust_uprev.fetch_rust_src_from_upstream( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ) + self.assertIn("error fetching signature file", ctx.exception.message) + + def test_key_expired(self): + def _gpg_verify(cmd, *args, **kwargs): + val = self.fake_gpg(cmd, *args, **kwargs) + if "--verify" in cmd: + val.stdout = "EXPKEYSIG 1234567" + return val + + self._mock_gpg.side_effect = _gpg_verify + + with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx: + rust_uprev.fetch_rust_src_from_upstream( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ) + self.assertIn("key has expired", ctx.exception.message) + + def test_key_revoked(self): + def _gpg_verify(cmd, *args, **kwargs): + val = self.fake_gpg(cmd, *args, **kwargs) + if "--verify" in cmd: + val.stdout = "REVKEYSIG 1234567" + return val + + self._mock_gpg.side_effect = _gpg_verify + + with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx: + rust_uprev.fetch_rust_src_from_upstream( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ) + self.assertIn("key has been revoked", ctx.exception.message) + + def test_signature_expired(self): + def _gpg_verify(cmd, *args, **kwargs): + val = self.fake_gpg(cmd, *args, **kwargs) + if "--verify" in cmd: + val.stdout = "EXPSIG 1234567" + return val + + self._mock_gpg.side_effect = _gpg_verify + + with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx: + rust_uprev.fetch_rust_src_from_upstream( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ) + self.assertIn("signature has expired", ctx.exception.message) + + def test_wrong_key(self): + def _gpg_verify(cmd, *args, **kwargs): + val = self.fake_gpg(cmd, *args, **kwargs) + if "--verify" in cmd: + val.stdout = "GOODSIG 0000000" + return val + + self._mock_gpg.side_effect = _gpg_verify + + with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx: + rust_uprev.fetch_rust_src_from_upstream( + "fakehttps://rustc-1.60.3-src.tar.gz", + Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"), + ) + self.assertIn("1234567 not found", ctx.exception.message) + + class FindEbuildPathTest(unittest.TestCase): """Tests for rust_uprev.find_ebuild_path()""" @@ -143,6 +304,79 @@ class FindEbuildPathTest(unittest.TestCase): self.assertEqual(result, ebuild) +class MirrorHasFileTest(unittest.TestCase): + """Tests for rust_uprev.mirror_has_file.""" + + @mock.patch.object(subprocess, "run") + def test_no(self, mock_run): + mock_run.return_value = mock.Mock( + returncode=1, + stdout="CommandException: One or more URLs matched no objects.", + ) + self.assertFalse(rust_uprev.mirror_has_file("rustc-1.69.0-src.tar.gz")) + + @mock.patch.object(subprocess, "run") + def test_yes(self, mock_run): + mock_run.return_value = mock.Mock( + returncode=0, + # pylint: disable=line-too-long + stdout="gs://chromeos-localmirror/distfiles/rustc-1.69.0-src.tar.gz", + ) + self.assertTrue(rust_uprev.mirror_has_file("rustc-1.69.0-src.tar.gz")) + + +class MirrorRustSourceTest(unittest.TestCase): + """Tests for rust_uprev.mirror_rust_source.""" + + def setUp(self) -> None: + start_mock(self, "rust_uprev.GSUTIL", "gsutil") + start_mock(self, "rust_uprev.MIRROR_PATH", "fakegs://fakemirror/") + start_mock( + self, "rust_uprev.get_distdir", return_value="/fake/distfiles" + ) + self.mock_mirror_has_file = start_mock( + self, + "rust_uprev.mirror_has_file", + ) + self.mock_fetch_rust_src_from_upstream = start_mock( + self, + "rust_uprev.fetch_rust_src_from_upstream", + ) + self.mock_subprocess_run = start_mock( + self, + "subprocess.run", + ) + + def test_already_present(self): + self.mock_mirror_has_file.return_value = True + rust_uprev.mirror_rust_source( + rust_uprev.RustVersion.parse("1.67.3"), + ) + self.mock_fetch_rust_src_from_upstream.assert_not_called() + self.mock_subprocess_run.assert_not_called() + + def test_fetch_and_upload(self): + self.mock_mirror_has_file.return_value = False + rust_uprev.mirror_rust_source( + rust_uprev.RustVersion.parse("1.67.3"), + ) + self.mock_fetch_rust_src_from_upstream.called_once() + self.mock_subprocess_run.has_calls( + [ + ( + [ + "gsutil", + "cp", + "-a", + "public-read", + "/fake/distdir/rustc-1.67.3-src.tar.gz", + "fakegs://fakemirror/rustc-1.67.3-src.tar.gz", + ] + ), + ] + ) + + class RemoveEbuildVersionTest(unittest.TestCase): """Tests for rust_uprev.remove_ebuild_version()""" @@ -240,7 +474,9 @@ class PrepareUprevTest(unittest.TestCase): f"rust-bootstrap-{self.bootstrap_version}.ebuild", ) mock_find_ebuild.return_value = bootstrap_ebuild_path - expected = (self.version_old, "/path/to/ebuild", self.bootstrap_version) + expected = rust_uprev.PreparedUprev( + self.version_old, Path("/path/to/ebuild"), self.bootstrap_version + ) actual = rust_uprev.prepare_uprev( rust_version=self.version_new, template=self.version_old ) @@ -280,7 +516,11 @@ class PrepareUprevTest(unittest.TestCase): f"rust-bootstrap-{self.bootstrap_version}.ebuild", ) mock_find_ebuild.return_value = bootstrap_ebuild_path - expected = (self.version_old, rust_ebuild_path, self.bootstrap_version) + expected = rust_uprev.PreparedUprev( + self.version_old, + Path(rust_ebuild_path), + self.bootstrap_version, + ) actual = rust_uprev.prepare_uprev( rust_version=self.version_new, template=None ) @@ -308,17 +548,76 @@ class PrepareUprevTest(unittest.TestCase): mock_exists.assert_not_called() def test_prepare_uprev_from_json(self): - ebuild_path = "/path/to/the/ebuild" + ebuild_path = Path("/path/to/the/ebuild") json_result = ( list(self.version_new), ebuild_path, list(self.bootstrap_version), ) - expected = (self.version_new, ebuild_path, self.bootstrap_version) + expected = rust_uprev.PreparedUprev( + self.version_new, + Path(ebuild_path), + self.bootstrap_version, + ) actual = rust_uprev.prepare_uprev_from_json(json_result) self.assertEqual(expected, actual) +class ToggleProfileData(unittest.TestCase): + """Tests functionality to include or exclude profile data from SRC_URI.""" + + ebuild_with_profdata = """ +# Some text here. +INCLUDE_PROFDATA_IN_SRC_URI=yes +some code here +""" + + ebuild_without_profdata = """ +# Some text here. +INCLUDE_PROFDATA_IN_SRC_URI= +some code here +""" + + ebuild_unexpected_content = """ +# Does not contain OMIT_PROFDATA_FROM_SRC_URI assignment +""" + + def setUp(self): + self.mock_read_text = start_mock(self, "pathlib.Path.read_text") + + def test_turn_off_profdata(self): + # Test that a file with profdata on is rewritten to a file with + # profdata off. + self.mock_read_text.return_value = self.ebuild_with_profdata + ebuild_file = "/path/to/eclass/cros-rustc.eclass" + with mock.patch("pathlib.Path.write_text") as mock_write_text: + rust_uprev.set_include_profdata_src(ebuild_file, include=False) + mock_write_text.assert_called_once_with( + self.ebuild_without_profdata, encoding="utf-8" + ) + + def test_turn_on_profdata(self): + # Test that a file with profdata off is rewritten to a file with + # profdata on. + self.mock_read_text.return_value = self.ebuild_without_profdata + ebuild_file = "/path/to/eclass/cros-rustc.eclass" + with mock.patch("pathlib.Path.write_text") as mock_write_text: + rust_uprev.set_include_profdata_src(ebuild_file, include=True) + mock_write_text.assert_called_once_with( + self.ebuild_with_profdata, encoding="utf-8" + ) + + def test_turn_on_profdata_fails_if_no_assignment(self): + # Test that if the string the code expects to find is not found, + # this causes an exception and the file is not overwritten. + self.mock_read_text.return_value = self.ebuild_unexpected_content + ebuild_file = "/path/to/eclass/cros-rustc.eclass" + with mock.patch("pathlib.Path.write_text") as mock_write_text: + with self.assertRaises(Exception): + rust_uprev.set_include_profdata_src(ebuild_file, include=False) + mock_write_text.assert_not_called() + + class UpdateBootstrapVersionTest(unittest.TestCase): """Tests for update_bootstrap_version step in rust_uprev""" @@ -329,26 +628,28 @@ BOOTSTRAP_VERSION="1.2.0" BOOTSTRAP_VERSION="1.3.6" """ + def setUp(self): + self.mock_read_text = start_mock(self, "pathlib.Path.read_text") + def test_success(self): - mock_open = mock.mock_open(read_data=self.ebuild_file_before) + self.mock_read_text.return_value = self.ebuild_file_before # ebuild_file and new bootstrap version are deliberately different ebuild_file = "/path/to/rust/cros-rustc.eclass" - with mock.patch("builtins.open", mock_open): + with mock.patch("pathlib.Path.write_text") as mock_write_text: rust_uprev.update_bootstrap_version( ebuild_file, rust_uprev.RustVersion.parse("1.3.6") ) - mock_open.return_value.__enter__().write.assert_called_once_with( - self.ebuild_file_after - ) + mock_write_text.assert_called_once_with( + self.ebuild_file_after, encoding="utf-8" + ) def test_fail_when_ebuild_misses_a_variable(self): - mock_open = mock.mock_open(read_data="") + self.mock_read_text.return_value = "" ebuild_file = "/path/to/rust/rust-1.3.5.ebuild" - with mock.patch("builtins.open", mock_open): - with self.assertRaises(RuntimeError) as context: - rust_uprev.update_bootstrap_version( - ebuild_file, rust_uprev.RustVersion.parse("1.2.0") - ) + with self.assertRaises(RuntimeError) as context: + rust_uprev.update_bootstrap_version( + ebuild_file, rust_uprev.RustVersion.parse("1.2.0") + ) self.assertEqual( "BOOTSTRAP_VERSION not found in /path/to/rust/rust-1.3.5.ebuild", str(context.exception), @@ -370,7 +671,8 @@ class UpdateBootstrapEbuildTest(unittest.TestCase): def test_update_bootstrap_ebuild(self): # The update should do two things: - # 1. Create a copy of rust-bootstrap's ebuild with the new version number. + # 1. Create a copy of rust-bootstrap's ebuild with the + # new version number. # 2. Add the old PV to RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE. with tempfile.TemporaryDirectory() as tmpdir_str, mock.patch.object( rust_uprev, "find_ebuild_path" @@ -394,7 +696,7 @@ some more text rust_uprev.update_bootstrap_ebuild(rust_uprev.RustVersion(1, 46, 0)) new_ebuild = bootstrapdir.joinpath("rust-bootstrap-1.46.0.ebuild") self.assertTrue(new_ebuild.exists()) - text = new_ebuild.read_text() + text = new_ebuild.read_text(encoding="utf-8") self.assertEqual( text, """ @@ -471,52 +773,6 @@ class RustUprevOtherStagesTests(unittest.TestCase): ) @mock.patch.object(shutil, "copyfile") - @mock.patch.object(os, "listdir") - @mock.patch.object(subprocess, "check_call") - def test_copy_patches(self, mock_call, mock_ls, mock_copy): - mock_ls.return_value = [ - f"rust-{self.old_version}-patch-1.patch", - f"rust-{self.old_version}-patch-2-old.patch", - f"rust-{self.current_version}-patch-1.patch", - f"rust-{self.current_version}-patch-2-new.patch", - ] - rust_uprev.copy_patches( - rust_uprev.RUST_PATH, self.current_version, self.new_version - ) - mock_copy.assert_has_calls( - [ - mock.call( - os.path.join( - rust_uprev.RUST_PATH, - "files", - f"rust-{self.current_version}-patch-1.patch", - ), - os.path.join( - rust_uprev.RUST_PATH, - "files", - f"rust-{self.new_version}-patch-1.patch", - ), - ), - mock.call( - os.path.join( - rust_uprev.RUST_PATH, - "files", - f"rust-{self.current_version}-patch-2-new.patch", - ), - os.path.join( - rust_uprev.RUST_PATH, - "files", - f"rust-{self.new_version}-patch-2-new.patch", - ), - ), - ] - ) - mock_call.assert_called_once_with( - ["git", "add", f"rust-{self.new_version}-*.patch"], - cwd=rust_uprev.RUST_PATH.joinpath("files"), - ) - - @mock.patch.object(shutil, "copyfile") @mock.patch.object(subprocess, "check_call") def test_create_rust_ebuild(self, mock_call, mock_copy): template_ebuild = f"/path/to/rust-{self.current_version}-r2.ebuild" diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py index dff239f3..743396f1 100755 --- a/rust_tools/rust_watch.py +++ b/rust_tools/rust_watch.py @@ -236,14 +236,18 @@ def atomically_write_state(state_file: pathlib.Path, state: State) -> None: def file_bug(title: str, body: str) -> None: - """Files a bug against gbiv@ with the given title/body.""" - bugs.CreateNewBug( - bugs.WellKnownComponents.CrOSToolchainPublic, - title, - body, - # To either take or reassign depending on the rotation. - assignee="gbiv@google.com", + """Files update bugs with the given title/body.""" + # (component, optional_assignee) + targets = ( + (bugs.WellKnownComponents.CrOSToolchainPublic, "gbiv@google.com"), + # b/269170429: Some Android folks said they wanted this before, and + # figuring out the correct way to apply permissions has been a pain. No + # one seems to be missing these notifications & the Android Rust folks + # are keeping on top of their toolchain, so ignore this for now. + # (bugs.WellKnownComponents.AndroidRustToolchain, None), ) + for component, assignee in targets: + bugs.CreateNewBug(component, title, body, assignee) def maybe_compose_bug( @@ -256,8 +260,20 @@ def maybe_compose_bug( title = f"[Rust] Update to {newest_release}" body = ( - "A new release has been detected; we should probably roll to it. " - "Please see go/crostc-rust-rotation for who's turn it is." + "A new Rust stable release has been detected; we should probably roll " + "to it.\n" + "\n" + "The regression-from-stable-to-stable tag might be interesting to " + "keep an eye on: https://github.com/rust-lang/rust/labels/" + "regression-from-stable-to-stable\n" + "\n" + "If you notice any bugs or issues you'd like to share, please " + "also note them on go/shared-rust-update-notes.\n" + "\n" + "See go/crostc-rust-rotation for the current rotation schedule.\n" + "\n" + "For questions about this bot, please contact chromeos-toolchain@ and " + "CC gbiv@." ) return title, body diff --git a/rust_tools/rust_watch_test.py b/rust_tools/rust_watch_test.py index 1e6aec51..79006f35 100755 --- a/rust_tools/rust_watch_test.py +++ b/rust_tools/rust_watch_test.py @@ -139,6 +139,7 @@ class Test(unittest.TestCase): self.assertIsNone(rust_watch.maybe_compose_email(new_gentoo_commits=())) def test_compose_bug_creates_bugs_on_new_versions(self): + bug_body_start = "A new Rust stable release has been detected;" title, body = rust_watch.maybe_compose_bug( old_state=rust_watch.State( last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), @@ -147,7 +148,7 @@ class Test(unittest.TestCase): newest_release=rust_watch.RustReleaseVersion(1, 0, 1), ) self.assertEqual(title, "[Rust] Update to 1.0.1") - self.assertTrue(body.startswith("A new release has been detected;")) + self.assertTrue(body.startswith(bug_body_start)) title, body = rust_watch.maybe_compose_bug( old_state=rust_watch.State( @@ -157,7 +158,7 @@ class Test(unittest.TestCase): newest_release=rust_watch.RustReleaseVersion(1, 1, 0), ) self.assertEqual(title, "[Rust] Update to 1.1.0") - self.assertTrue(body.startswith("A new release has been detected;")) + self.assertTrue(body.startswith(bug_body_start)) title, body = rust_watch.maybe_compose_bug( old_state=rust_watch.State( @@ -167,7 +168,7 @@ class Test(unittest.TestCase): newest_release=rust_watch.RustReleaseVersion(2, 0, 0), ) self.assertEqual(title, "[Rust] Update to 2.0.0") - self.assertTrue(body.startswith("A new release has been detected;")) + self.assertTrue(body.startswith(bug_body_start)) def test_compose_bug_does_nothing_when_no_new_updates_exist(self): self.assertIsNone( diff --git a/toolchain_utils_githooks/check-presubmit.py b/toolchain_utils_githooks/check-presubmit.py index 485737d5..0d2f3b31 100755 --- a/toolchain_utils_githooks/check-presubmit.py +++ b/toolchain_utils_githooks/check-presubmit.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be @@ -23,6 +22,12 @@ import traceback import typing as t +# This was originally had many packages in it (notably scipy) +# but due to changes in how scipy is built, we can no longer install +# it in the chroot. See b/284489250 +PIP_DEPENDENCIES = ("numpy",) + + def run_command_unchecked( command: t.List[str], cwd: str, env: t.Dict[str, str] = None ) -> t.Tuple[int, str]: @@ -49,7 +54,8 @@ def has_executable_on_path(exe: str) -> bool: def escape_command(command: t.Iterable[str]) -> str: """Returns a human-readable and copy-pastable shell command. - Only intended for use in output to users. shell=True is strongly discouraged. + Only intended for use in output to users. shell=True is strongly + discouraged. """ return " ".join(shlex.quote(x) for x in command) @@ -172,9 +178,9 @@ def check_black( toolchain_utils_root: str, black: Path, python_files: t.Iterable[str] ) -> CheckResult: """Subchecker of check_py_format. Checks python file formats with black""" - # Folks have been bitten by accidentally using multiple formatter versions in - # the past. This is an issue, since newer versions of black may format things - # differently. Make the version obvious. + # Folks have been bitten by accidentally using multiple formatter + # versions in the past. This is an issue, since newer versions of + # black may format things differently. Make the version obvious. command = [black, "--version"] exit_code, stdout_and_stderr = run_command_unchecked( command, cwd=toolchain_utils_root @@ -182,7 +188,8 @@ def check_black( if exit_code: return CheckResult( ok=False, - output=f"Failed getting black version; stdstreams: {stdout_and_stderr}", + output="Failed getting black version; " + f"stdstreams: {stdout_and_stderr}", autofix_commands=[], ) @@ -235,15 +242,16 @@ def check_black( err_str = "\n".join(errors) return CheckResult( ok=False, - output=f"Using {black_version!r} had the following errors:\n{err_str}", + output=f"Using {black_version!r} had the following errors:\n" + f"{err_str}", autofix_commands=[], ) autofix = black_invocation + bad_files return CheckResult( ok=False, - output=f"Using {black_version!r}, these file(s) have formatting errors: " - f"{bad_files}", + output=f"Using {black_version!r}, these file(s) have formatting " + f"errors: {bad_files}", autofix_commands=[autofix], ) @@ -297,7 +305,7 @@ def check_py_format( thread_pool: multiprocessing.pool.ThreadPool, files: t.Iterable[str], ) -> t.List[CheckResult]: - """Runs yapf on files to check for style bugs. Also checks for #!s.""" + """Runs black on files to check for style bugs. Also checks for #!s.""" black = "black" if not has_executable_on_path(black): return CheckResult( @@ -412,11 +420,10 @@ def check_cros_lint( ["golint", "-set_exit_status"] + go_files ) - complaint = "\n".join( - ( - "WARNING: go linting disabled. golint is not on your $PATH.", - "Please either enter a chroot, or install go locally. Continuing.", - ) + complaint = ( + "WARNING: go linting disabled. golint is not on your $PATH.\n" + "Please either enter a chroot, or install go locally. " + "Continuing." ) return CheckResult( ok=True, @@ -426,13 +433,11 @@ def check_cros_lint( tasks.append(("golint", thread_pool.apply_async(run_golint))) - complaint = "\n".join( - ( - "WARNING: No ChromeOS checkout detected, and no viable CrOS tree", - "found; falling back to linting only python and go. If you have a", - "ChromeOS checkout, please either develop from inside of the source", - "tree, or set $CHROMEOS_ROOT_DIRECTORY to the root of it.", - ) + complaint = ( + "WARNING: No ChromeOS checkout detected, and no viable CrOS tree\n" + "found; falling back to linting only python and go. If you have a\n" + "ChromeOS checkout, please either develop from inside of the source\n" + "tree, or set $CHROMEOS_ROOT_DIRECTORY to the root of it." ) results = [(name, get_check_result_or_catch(task)) for name, task in tasks] @@ -689,11 +694,15 @@ def maybe_reexec_inside_chroot(autofix: bool, files: t.List[str]) -> None: def ensure_pip_deps_installed() -> None: + if not PIP_DEPENDENCIES: + # No need to install pip if we don't have any deps. + return + if not has_executable_on_path("pip"): print("Autoinstalling `pip`...") subprocess.check_call(["sudo", "emerge", "dev-python/pip"]) - for package in ("scipy", "yapf"): + for package in PIP_DEPENDENCIES: exit_code = subprocess.call( ["python3", "-c", f"import {package}"], stdout=subprocess.DEVNULL, |