diff options
author | Stephen Hines <srhines@google.com> | 2022-01-05 21:47:21 -0800 |
---|---|---|
committer | Stephen Hines <srhines@google.com> | 2022-01-05 21:47:21 -0800 |
commit | 7bc245bad95c6f4c15bc3f819f61312705b59a5a (patch) | |
tree | cf4cdad04e1c146844e2a790910cf9cd1b97082a | |
parent | f103b9f78baa21b4edf25d5dd5a1226b6c0ff159 (diff) | |
parent | dbc8cb672a14f4a8bb136074817c71a22f743eaf (diff) | |
download | toolchain-utils-android-s-qpr3-beta-1.tar.gz |
Merging 32 commit(s) from Chromium's toolchain-utilsandroid-t-preview-2android-t-preview-1android-t-beta-3android-s-v2-beta-3android-s-qpr3-beta-1android-t-preview-1android-s-v2-beta-3android-s-qpr3-beta-1
Merged commit digest:
dbc8cb67 compiler_wrapper: Use ld.bfd to detect binutils path
0274702f patch_sync: Add show subcommand
7fd98520 patch_sync: Add patch_parsing module
72bb9a35 patch_sync: Base code, version control managing
457d96b5 pgo_tools: make monitor_pgo_profiles fail rather than emailing
ed486b7b afdo_metadata: Publish the new kernel profiles
a170d3e2 crosperf: remove dead bob DUT
d60f468c afdo_metadata: Publish the new kernel profiles
26b91865 llvm_tools: Update get_upstream_patch schema
811429e3 llvm_tools: Catch dup SHAs in get_upstream_patch
ed2aa295 compiler_wrapper: Drop "-fno-experimental-pass-manager"
e8d03f7e rust_watch: report bugs for new minor or major versions of rustc
443e0b36 cros_utils: Add a bug reporting module
86fe58c1 compiler_wrapper: adds ${sysroot}/usr/lib to linker path
d02a62da compiler_wrapper: add missing unused-but-set-var
50d42705 afdo_metadata: Publish the new kernel profiles
7afc3522 llvm_tools: Make skip_dependencies optional
9de6ecbf crosperf: Increase timeout for a crosfleet call
58a779e7 llvm_tools: adds a way to skip dependencies when pulling from Phab
28d6cbe1 crosperf: Replace non-inclusive language
c965dab6 afdo_metadata: Publish the new kernel profiles
58bc0488 crosperf: Update default_remotes to new DUTs
70c83ac3 Replace non-inclusive language
d080198a llvm_tools: fix yapf warnings
5200728b llvm_tools: Add lldb_server to updated_packages
afd2d4b0 compiler_wrapper: ignore unused_but_set_variable
217c68a1 Add crosperf_autolock.py wrapper
dcb168bf afdo_metadata: Publish the new kernel profiles
a7a19342 Auto-format the crosperf files
c5e12fba Add 'no_lock' crosperf setting
ec4d2aec afdo_metadata: Publish the new kernel profiles
2c5d6278 compiler_wrapper: move llvmnext flag to post flags
Change-Id: Id8af179e775f6fe23c22ce05b8be9b5b02ccbaf4
100 files changed, 3580 insertions, 1836 deletions
diff --git a/PRESUBMIT.cfg b/PRESUBMIT.cfg index d69d375b..5fccf3ac 100644 --- a/PRESUBMIT.cfg +++ b/PRESUBMIT.cfg @@ -1,2 +1,6 @@ [Hook Scripts] toolchain_utils_presubmits = ./toolchain_utils_githooks/check-presubmit ${PRESUBMIT_FILES} + +[Hook Overrides Options] +cros_license_check: --exclude_regex=\b(default_remotes)$ +long_line_check: --exclude_regex=\b(default_remotes)$ diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json index a4044c67..a3275004 100644 --- a/afdo_metadata/kernel_afdo.json +++ b/afdo_metadata/kernel_afdo.json @@ -1,14 +1,14 @@ { "chromeos-kernel-4_4": { - "name": "R96-14209.0-1632130382" + "name": "R99-14388.8-1640601492" }, "chromeos-kernel-4_14": { - "name": "R96-14221.0-1632130642" + "name": "R99-14407.0-1640601365" }, "chromeos-kernel-4_19": { - "name": "R96-14209.0-1632130511" + "name": "R99-14388.8-1640601108" }, "chromeos-kernel-5_4": { - "name": "R96-14209.0-1632130436" + "name": "R99-14397.0-1640601195" } } diff --git a/android_merge_from_upstream.sh b/android_merge_from_upstream.sh index 65b47d7b..4716a25a 100755 --- a/android_merge_from_upstream.sh +++ b/android_merge_from_upstream.sh @@ -11,9 +11,9 @@ # https://android-review.googlesource.com/c/platform/external/toolchain-utils/+/1132504/1 local_branch_name="merge_with_upstream" -local_upstream="aosp/master" +local_upstream="aosp/master" # nocheck remote="aosp" -remote_branch="${remote}/upstream-main" +remote_branch="${remote}/upstream-main" # nocheck my_dir="$(dirname "$(readlink -m "$0")")" cd "${my_dir}" @@ -21,7 +21,7 @@ cd "${my_dir}" ensure_head_is_upstream_main() { local current_rev main_rev current_rev="$(git rev-parse HEAD)" - main_rev="$(git rev-parse ${local_upstream})" + main_rev="$(git rev-parse "${local_upstream}")" if [[ "${current_rev}" != "${main_rev}" ]]; then echo "Please checkout ${local_upstream} and rerun this" >&2 exit @@ -49,7 +49,7 @@ ensure_no_local_branch_present() { get_merge_commit_list() { local merge_base - merge_base="$(git merge-base HEAD ${remote_branch})" + merge_base="$(git merge-base HEAD "${remote_branch}")" git log --oneline "${merge_base}..${remote_branch}" } @@ -62,6 +62,9 @@ repo start "${local_branch_name}" commit_list="$(get_merge_commit_list)" num_commits="$(wc -l <<< "${commit_list}")" + +# Disable shellcheck for the sed substitution warning. +# shellcheck disable=SC2001 commit_message="Merging ${num_commits} commit(s) from Chromium's toolchain-utils Merged commit digest: @@ -69,6 +72,6 @@ $(sed 's/^/ /' <<< "${commit_list}") " git merge "${remote_branch}" -m "${commit_message}" -echo 'NOTE: When you try to `repo upload`, repo might show a scary warning' +echo 'NOTE: When you try to "repo upload", repo might show a scary warning' echo 'about the number of changes are being uploaded. That should be fine,' echo 'since repo will only create CLs for commits not known to our remote.' diff --git a/compiler_wrapper/clang_flags.go b/compiler_wrapper/clang_flags.go index 84149bf6..e25ed74c 100644 --- a/compiler_wrapper/clang_flags.go +++ b/compiler_wrapper/clang_flags.go @@ -126,7 +126,7 @@ func processClangFlags(builder *commandBuilder) error { // Specify the target for clang. if !builder.cfg.isHostWrapper { - linkerPath := getLinkerPath(env, builder.target.target+"-ld", builder.rootPath) + linkerPath := getLinkerPath(env, builder.target.target+"-ld.bfd", builder.rootPath) relLinkerPath, err := filepath.Rel(env.getwd(), linkerPath) if err != nil { return wrapErrorwithSourceLocf(err, "failed to make linker path %s relative to %s", diff --git a/compiler_wrapper/clang_flags_test.go b/compiler_wrapper/clang_flags_test.go index 25c13c74..23aed7ef 100644 --- a/compiler_wrapper/clang_flags_test.go +++ b/compiler_wrapper/clang_flags_test.go @@ -248,7 +248,7 @@ func TestClangArchFlags(t *testing.T) { func TestClangLinkerPathProbesBinariesOnPath(t *testing.T) { withTestContext(t, func(ctx *testContext) { linkerPath := filepath.Join(ctx.tempDir, "a/b/c") - ctx.writeFile(filepath.Join(linkerPath, "x86_64-cros-linux-gnu-ld"), "") + ctx.writeFile(filepath.Join(linkerPath, "x86_64-cros-linux-gnu-ld.bfd"), "") ctx.env = []string{"PATH=nonExistantPath:" + linkerPath} cmd := ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand("./x86_64-cros-linux-gnu-clang", mainCc))) @@ -268,7 +268,7 @@ func TestClangLinkerPathEvaluatesSymlinksForBinariesOnPath(t *testing.T) { ctx.writeFile(realLinkerPath, "") firstLinkLinkerPath := filepath.Join(ctx.tempDir, "a/first/somelinker") ctx.symlink(realLinkerPath, firstLinkLinkerPath) - secondLinkLinkerPath := filepath.Join(ctx.tempDir, "a/second/x86_64-cros-linux-gnu-ld") + secondLinkLinkerPath := filepath.Join(ctx.tempDir, "a/second/x86_64-cros-linux-gnu-ld.bfd") ctx.symlink(firstLinkLinkerPath, secondLinkLinkerPath) ctx.env = []string{"PATH=nonExistantPath:" + filepath.Dir(secondLinkLinkerPath)} diff --git a/compiler_wrapper/config.go b/compiler_wrapper/config.go index 11d77ba7..6c28287c 100644 --- a/compiler_wrapper/config.go +++ b/compiler_wrapper/config.go @@ -154,6 +154,7 @@ var crosHardenedConfig = &config{ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", }, newWarningsDir: "/tmp/fatal_clang_warnings", triciumNitsDir: "/tmp/linting_output/clang-tidy", @@ -195,6 +196,7 @@ var crosNonHardenedConfig = &config{ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", }, newWarningsDir: "/tmp/fatal_clang_warnings", triciumNitsDir: "/tmp/linting_output/clang-tidy", @@ -243,6 +245,7 @@ var crosHostConfig = &config{ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", }, newWarningsDir: "/tmp/fatal_clang_warnings", triciumNitsDir: "/tmp/linting_output/clang-tidy", diff --git a/compiler_wrapper/cros_llvm_next_flags.go b/compiler_wrapper/cros_llvm_next_flags.go index d2ae413b..870e2885 100644 --- a/compiler_wrapper/cros_llvm_next_flags.go +++ b/compiler_wrapper/cros_llvm_next_flags.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +//go:build !android_llvm_next_flags // +build !android_llvm_next_flags package main @@ -13,8 +14,6 @@ package main // tag is set. // TODO: Enable test in config_test.go, once we have new llvm-next flags. -var llvmNextFlags = []string{ - "-Wno-unused-but-set-variable", -} +var llvmNextFlags = []string{} var llvmNextPostFlags = []string{} diff --git a/compiler_wrapper/sanitizer_flags.go b/compiler_wrapper/sanitizer_flags.go index fe8d1503..da0a64b3 100644 --- a/compiler_wrapper/sanitizer_flags.go +++ b/compiler_wrapper/sanitizer_flags.go @@ -9,20 +9,13 @@ import ( ) func processSanitizerFlags(builder *commandBuilder) { - hasCoverageFlags := false hasSanitizeFlags := false - hasSanitizeFuzzerFlags := false for _, arg := range builder.args { // TODO: This should probably be -fsanitize= to not match on - // e.g. -fsanitize-blacklist + // e.g. -fsanitize-blocklist if arg.fromUser { if strings.HasPrefix(arg.value, "-fsanitize") { hasSanitizeFlags = true - if strings.Contains(arg.value, "fuzzer") { - hasSanitizeFuzzerFlags = true - } - } else if arg.value == "-fprofile-instr-generate" { - hasCoverageFlags = true } } } @@ -44,15 +37,5 @@ func processSanitizerFlags(builder *commandBuilder) { } return arg.value }) - if builder.target.compilerType == clangType { - // hasSanitizeFlags && hasCoverageFlags is to work around crbug.com/1013622 - if hasSanitizeFuzzerFlags || (hasSanitizeFlags && hasCoverageFlags) { - fuzzerFlagsToAdd := []string{ - // TODO: This flag should be removed once fuzzer works with new pass manager - "-fno-experimental-new-pass-manager", - } - builder.addPreUserArgs(fuzzerFlagsToAdd...) - } - } } } diff --git a/compiler_wrapper/sanitizer_flags_test.go b/compiler_wrapper/sanitizer_flags_test.go index 8f50a900..a401d58e 100644 --- a/compiler_wrapper/sanitizer_flags_test.go +++ b/compiler_wrapper/sanitizer_flags_test.go @@ -98,55 +98,3 @@ func TestKeepSanitizerFlagsIfSanitizeGivenInCommonFlags(t *testing.T) { } }) } - -func TestAddFuzzerFlagsForClang(t *testing.T) { - withTestContext(t, func(ctx *testContext) { - cmd := ctx.must(callCompiler(ctx, ctx.cfg, - ctx.newCommand(clangX86_64, "-fsanitize=fuzzer", mainCc))) - if err := verifyArgOrder(cmd, "-fno-experimental-new-pass-manager", - "-fsanitize=fuzzer", mainCc); err != nil { - t.Error(err) - } - }) -} - -func TestOmitFuzzerFlagsForGcc(t *testing.T) { - withTestContext(t, func(ctx *testContext) { - cmd := ctx.must(callCompiler(ctx, ctx.cfg, - ctx.newCommand(gccX86_64, "-fsanitize=fuzzer", mainCc))) - if err := verifyArgCount(cmd, 0, "-fno-experimental-new-pass-manager"); err != nil { - t.Error(err) - } - }) -} - -func TestAddSanitizerCoverageFlagsForClang(t *testing.T) { - withTestContext(t, func(ctx *testContext) { - cmd := ctx.must(callCompiler(ctx, ctx.cfg, - ctx.newCommand(clangX86_64, "-fsanitize=address", "-fprofile-instr-generate", mainCc))) - if err := verifyArgOrder(cmd, "-fno-experimental-new-pass-manager", - "-fsanitize=address", "-fprofile-instr-generate", mainCc); err != nil { - t.Error(err) - } - }) -} - -func TestOmitSanitizerCoverageFlagsForClang(t *testing.T) { - withTestContext(t, func(ctx *testContext) { - cmd := ctx.must(callCompiler(ctx, ctx.cfg, - ctx.newCommand(clangX86_64, "-fsanitize=address", mainCc))) - if err := verifyArgCount(cmd, 0, "-fno-experimental-new-pass-manager"); err != nil { - t.Error(err) - } - }) -} - -func TestKeepSanitizerCoverageFlagsForClang(t *testing.T) { - withTestContext(t, func(ctx *testContext) { - cmd := ctx.must(callCompiler(ctx, ctx.cfg, - ctx.newCommand(clangX86_64, "-fprofile-instr-generate", mainCc))) - if err := verifyArgCount(cmd, 0, "-fno-experimental-new-pass-manager"); err != nil { - t.Error(err) - } - }) -} diff --git a/compiler_wrapper/sysroot_flag.go b/compiler_wrapper/sysroot_flag.go index 501fb20e..e0583b22 100644 --- a/compiler_wrapper/sysroot_flag.go +++ b/compiler_wrapper/sysroot_flag.go @@ -11,9 +11,14 @@ import ( func processSysrootFlag(builder *commandBuilder) { fromUser := false + userSysroot := "" for _, arg := range builder.args { if arg.fromUser && strings.HasPrefix(arg.value, "--sysroot=") { fromUser = true + sysrootArg := strings.Split(arg.value, "=") + if len(sysrootArg) == 2 { + userSysroot = sysrootArg[1] + } break } } @@ -27,5 +32,13 @@ func processSysrootFlag(builder *commandBuilder) { } if !fromUser { builder.addPreUserArgs("--sysroot=" + sysroot) + } else { + sysroot = userSysroot } + + libdir := "-L" + sysroot + "/usr/lib" + if strings.Contains(builder.target.target, "64") { + libdir += "64" + } + builder.addPostUserArgs(libdir) } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json index fae0f9f5..e237c7c7 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json @@ -43,7 +43,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ], "env_updates": [ "PYTHONPATH=/somepath/test_binary" @@ -97,7 +98,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ], "env_updates": [ "PYTHONPATH=/somepath/test_binary" @@ -154,7 +156,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ], "env_updates": [ "PYTHONPATH=/somepath/test_binary" diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json index 947f1a4d..07fccc6d 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_ftrapv_maincc_target_specific.json @@ -34,7 +34,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -75,7 +76,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -116,7 +118,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -157,7 +160,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -198,7 +202,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -239,7 +244,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -280,7 +286,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -321,7 +328,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -362,7 +370,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json index 9267f5b4..a221605e 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_host_wrapper.json @@ -33,7 +33,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json index c7a4a6a7..2130d528 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_maincc_target_specific.json @@ -33,7 +33,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -73,7 +74,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -113,7 +115,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -153,7 +156,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -193,7 +197,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -233,7 +238,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -273,7 +279,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -313,7 +320,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -353,7 +361,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json index 96b3fe50..43ae728a 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_path.json @@ -33,7 +33,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -76,7 +77,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] }, "stdout": "somemessage", @@ -119,7 +121,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -162,7 +165,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -214,7 +218,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -269,7 +274,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -324,7 +330,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] }, "stdout": "somemessage", @@ -367,7 +374,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -407,7 +415,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -447,7 +456,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -487,7 +497,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -530,7 +541,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json index 14e45b27..b8b28cd9 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_sanitizer_args.json @@ -36,7 +36,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -79,7 +80,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -122,7 +124,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -165,7 +168,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -202,13 +206,13 @@ "-Wno-deprecated-declarations", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", - "-fno-experimental-new-pass-manager", "-fsanitize=fuzzer", "main.cc", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -246,14 +250,14 @@ "-Wno-deprecated-declarations", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", - "-fno-experimental-new-pass-manager", "-fsanitize=address", "-fprofile-instr-generate", "main.cc", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -295,7 +299,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -337,7 +342,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json index 01a50a96..7c4afd32 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clang_specific_args.json @@ -51,7 +51,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -93,7 +94,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -135,7 +137,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -177,7 +180,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json b/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json index 2a563b89..f678ba64 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/clangtidy.json @@ -49,7 +49,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } }, @@ -76,7 +77,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -133,7 +135,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } }, @@ -160,7 +163,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -219,7 +223,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] }, "stdout": "somemessage", @@ -249,7 +254,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -309,7 +315,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } }, @@ -336,7 +343,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] }, "stdout": "somemessage", diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json index 139e351a..f5a77714 100644 --- a/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_clang_host_golden/force_disable_werror.json @@ -36,7 +36,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] } } @@ -79,7 +80,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] }, "stderr": "-Werror originalerror", @@ -109,6 +111,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-Wno-error", "-Wno-error=poison-system-directories" ] @@ -155,7 +158,8 @@ "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", - "-Wno-deprecated-copy" + "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable" ] }, "stderr": "-Werror originalerror", @@ -185,6 +189,7 @@ "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-Wno-error", "-Wno-error=poison-system-directories" ] diff --git a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json index c3bac4d2..05aea31f 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json @@ -50,10 +50,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -121,10 +123,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -195,10 +199,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json index 6efe1d2a..2b04d0c9 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_ftrapv_maincc_target_specific.json @@ -42,10 +42,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-ftrapv", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -103,10 +105,12 @@ "--prefix=../../bin/x86_64-cros-eabi-", "-ftrapv", "main.cc", + "-L/usr/x86_64-cros-eabi/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -164,10 +168,12 @@ "--prefix=../../bin/x86_64-cros-win-gnu-", "-ftrapv", "main.cc", + "-L/usr/x86_64-cros-win-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -225,10 +231,12 @@ "--prefix=../../bin/armv7m-cros-linux-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv7m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-linux-gnu" @@ -285,10 +293,12 @@ "--prefix=../../bin/armv7m-cros-eabi-", "-ftrapv", "main.cc", + "-L/usr/armv7m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-eabi" @@ -345,10 +355,12 @@ "--prefix=../../bin/armv7m-cros-win-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv7m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-win-gnu" @@ -405,10 +417,12 @@ "--prefix=../../bin/armv8m-cros-linux-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv8m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-linux-gnu" @@ -465,10 +479,12 @@ "--prefix=../../bin/armv8m-cros-eabi-", "-ftrapv", "main.cc", + "-L/usr/armv8m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-eabi" @@ -525,10 +541,12 @@ "--prefix=../../bin/armv8m-cros-win-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv8m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-win-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json index ac1109c1..ea834174 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_maincc_target_specific.json @@ -40,10 +40,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -99,10 +101,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-eabi-", "main.cc", + "-L/usr/x86_64-cros-eabi/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -158,10 +162,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-win-gnu-", "main.cc", + "-L/usr/x86_64-cros-win-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -217,10 +223,12 @@ "-mthumb", "--prefix=../../bin/armv7m-cros-linux-gnu-", "main.cc", + "-L/usr/armv7m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-linux-gnu" @@ -275,10 +283,12 @@ "-static-libgcc", "--prefix=../../bin/armv7m-cros-eabi-", "main.cc", + "-L/usr/armv7m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-eabi" @@ -333,10 +343,12 @@ "-mthumb", "--prefix=../../bin/armv7m-cros-win-gnu-", "main.cc", + "-L/usr/armv7m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-win-gnu" @@ -391,10 +403,12 @@ "-mthumb", "--prefix=../../bin/armv8m-cros-linux-gnu-", "main.cc", + "-L/usr/armv8m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-linux-gnu" @@ -449,10 +463,12 @@ "-static-libgcc", "--prefix=../../bin/armv8m-cros-eabi-", "main.cc", + "-L/usr/armv8m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-eabi" @@ -507,10 +523,12 @@ "-mthumb", "--prefix=../../bin/armv8m-cros-win-gnu-", "main.cc", + "-L/usr/armv8m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-win-gnu" diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json index d3dc7287..52d4184a 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_path.json @@ -40,10 +40,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -102,10 +104,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -164,10 +168,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -226,10 +232,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -297,10 +305,12 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -371,10 +381,12 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -440,10 +452,12 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -502,10 +516,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -561,10 +577,12 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -620,10 +638,12 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -679,10 +699,12 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", @@ -741,10 +763,12 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json index 8ef57484..b71a8805 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_sanitizer_args.json @@ -42,10 +42,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -103,10 +105,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -164,10 +168,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -225,10 +231,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -282,14 +290,15 @@ "-pie", "-fno-omit-frame-pointer", "-static-libgcc", - "-fno-experimental-new-pass-manager", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=fuzzer", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -344,15 +353,16 @@ "-pie", "-fno-omit-frame-pointer", "-static-libgcc", - "-fno-experimental-new-pass-manager", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=address", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -409,10 +419,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -470,10 +482,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json index f93c958b..d10e6cd0 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_specific_args.json @@ -58,10 +58,12 @@ "-Woverride-init", "-Wunsafe-loop-optimizations", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -119,10 +121,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-Wno-#warnings", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -180,10 +184,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-Wno-error=uninitialized", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -241,10 +247,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-someflag", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json index 5cfe476e..42209185 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clang_sysroot_wrapper_common.json @@ -27,6 +27,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -77,10 +78,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -139,10 +142,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -192,10 +197,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -251,10 +258,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-D__KERNEL__", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -310,10 +319,12 @@ "--prefix=../../bin/armv7a-cros-linux-gnueabihf-", "-D__KERNEL__", "main.cc", + "-L/usr/armv7a-cros-linux-gnueabihf/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7a-cros-linux-gnueabihf" @@ -369,10 +380,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "--sysroot=xyz", "main.cc", + "-Lxyz/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json index 658ecc26..ea1363e7 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/clangtidy.json @@ -55,10 +55,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -95,10 +97,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -165,10 +169,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -206,10 +212,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -278,10 +286,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -322,10 +332,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -395,10 +407,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -436,10 +450,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json index 14441d7b..4df81578 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/force_disable_werror.json @@ -43,10 +43,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -105,10 +107,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -153,10 +157,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -219,10 +225,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -267,10 +275,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json index 3ffd9e96..e2037e26 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_clang_syntax.json @@ -40,10 +40,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -70,6 +72,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -125,10 +128,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -155,6 +160,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -205,10 +211,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -267,10 +275,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -297,6 +307,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_maincc_target_specific.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_maincc_target_specific.json index 92fec35e..0cc3d8a9 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_maincc_target_specific.json @@ -27,6 +27,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -65,6 +66,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-eabi/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -103,6 +105,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-win-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -140,7 +143,8 @@ "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv7m-cros-linux-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -177,7 +181,8 @@ "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", - "main.cc" + "main.cc", + "-L/usr/armv7m-cros-eabi/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -214,7 +219,8 @@ "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv7m-cros-win-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -251,7 +257,8 @@ "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv8m-cros-linux-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -288,7 +295,8 @@ "-D_FORTIFY_SOURCE=2", "-fno-omit-frame-pointer", "-static-libgcc", - "main.cc" + "main.cc", + "-L/usr/armv8m-cros-eabi/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -325,7 +333,8 @@ "-D_FORTIFY_SOURCE=2", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv8m-cros-win-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_path.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_path.json index f44e69dd..5b129d7d 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_path.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_path.json @@ -27,6 +27,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -68,6 +69,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -109,6 +111,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -147,6 +150,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -185,6 +189,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -226,6 +231,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_sanitizer_args.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_sanitizer_args.json index 96215022..a61ead8e 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_sanitizer_args.json @@ -30,6 +30,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -71,6 +72,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -112,6 +114,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -153,6 +156,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -193,6 +197,7 @@ "-static-libgcc", "-fsanitize=fuzzer", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -235,6 +240,7 @@ "-fsanitize=address", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -275,6 +281,7 @@ "-static-libgcc", "-fsanitize=address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -315,6 +322,7 @@ "-static-libgcc", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_specific_args.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_specific_args.json index f5c311b4..f7b20001 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_specific_args.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_specific_args.json @@ -29,6 +29,7 @@ "-static-libgcc", "-march=goldmont", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -69,6 +70,7 @@ "-static-libgcc", "-march=goldmont-plus", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -109,6 +111,7 @@ "-static-libgcc", "-march=skylake", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_golden/gcc_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_hardened_golden/gcc_sysroot_wrapper_common.json index 9ca8403e..299b46b6 100644 --- a/compiler_wrapper/testdata/cros_hardened_golden/gcc_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_hardened_golden/gcc_sysroot_wrapper_common.json @@ -27,6 +27,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -64,6 +65,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -105,6 +107,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -138,6 +141,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -176,6 +180,7 @@ "-fno-stack-protector", "-D__KERNEL__", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -213,7 +218,8 @@ "-mthumb", "-fno-stack-protector", "-D__KERNEL__", - "main.cc" + "main.cc", + "-L/usr/armv7a-cros-linux-gnueabihf/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -252,6 +258,7 @@ "-static-libgcc", "--sysroot=xyz", "main.cc", + "-Lxyz/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json index b29f09d0..05aea31f 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json @@ -40,7 +40,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -51,10 +50,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -112,7 +113,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -123,10 +123,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -187,7 +189,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -198,10 +199,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json index 352ac4da..52d4184a 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clang_path.json @@ -30,7 +30,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -41,10 +40,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -93,7 +94,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -104,10 +104,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -156,7 +158,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -167,10 +168,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -219,7 +222,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -230,10 +232,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -289,7 +293,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -302,10 +305,12 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -364,7 +369,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -377,10 +381,12 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -434,7 +440,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -447,10 +452,12 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -499,7 +506,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -510,10 +516,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -559,7 +567,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -570,10 +577,12 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -619,7 +628,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -630,10 +638,12 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -679,7 +689,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -690,10 +699,12 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", @@ -742,7 +753,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -753,10 +763,12 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json index 11851288..ea1363e7 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/clangtidy.json @@ -45,7 +45,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -56,10 +55,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -86,7 +87,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -97,10 +97,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -157,7 +159,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -168,10 +169,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -199,7 +202,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -210,10 +212,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -272,7 +276,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -283,10 +286,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -317,7 +322,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -328,10 +332,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -391,7 +397,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -402,10 +407,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -433,7 +440,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -444,10 +450,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json index 44afa63d..4df81578 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/force_disable_werror.json @@ -33,7 +33,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -44,10 +43,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -96,7 +97,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -107,10 +107,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -145,7 +147,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -156,10 +157,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -212,7 +215,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -223,10 +225,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -261,7 +265,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -272,10 +275,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json index 2851f6d5..e2037e26 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_clang_syntax.json @@ -30,7 +30,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -41,10 +40,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -71,6 +72,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -116,7 +118,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -127,10 +128,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -157,6 +160,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -197,7 +201,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -208,10 +211,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -260,7 +265,6 @@ "-fexperimental-new-pass-manager", "-Wno-compound-token-split-by-macro", "-Wno-deprecated-declarations", - "-Wno-unused-but-set-variable", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-fcommon", "-fstack-protector-strong", @@ -271,10 +275,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -301,6 +307,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_path.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_path.json index f44e69dd..5b129d7d 100644 --- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_path.json +++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/gcc_path.json @@ -27,6 +27,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -68,6 +69,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -109,6 +111,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -147,6 +150,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -185,6 +189,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -226,6 +231,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json index 58a0846f..a676dc6e 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json @@ -49,10 +49,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -116,10 +118,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -186,10 +190,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json index 5865a12f..cc6d1ada 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clang_path.json @@ -39,10 +39,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -95,10 +97,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -151,10 +155,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -207,10 +213,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -272,10 +280,12 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -341,10 +351,12 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -409,10 +421,12 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -465,10 +479,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -518,10 +534,12 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -571,10 +589,12 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -624,10 +644,12 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", @@ -680,10 +702,12 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json index 658ecc26..ea1363e7 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/clangtidy.json @@ -55,10 +55,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -95,10 +97,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -165,10 +169,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -206,10 +212,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -278,10 +286,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -322,10 +332,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -395,10 +407,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -436,10 +450,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json index aa3de738..2c34edb4 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/force_disable_werror.json @@ -42,10 +42,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -98,10 +100,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -140,10 +144,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -200,10 +206,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -242,10 +250,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json index a532745f..368eb855 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_clang_syntax.json @@ -40,10 +40,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -69,6 +71,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -120,10 +123,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -150,6 +155,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -200,10 +206,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -262,10 +270,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -291,6 +301,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] }, diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_path.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_path.json index 9a964aad..9393c4b3 100644 --- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_path.json +++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/gcc_path.json @@ -26,6 +26,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -62,6 +63,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] }, @@ -98,6 +100,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -131,6 +134,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -164,6 +168,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -200,6 +205,7 @@ "-fno-omit-frame-pointer", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json index 7ccc0f66..6c46fee8 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json @@ -41,10 +41,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -103,10 +105,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -168,10 +172,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json index 36e8f7d0..86960cc9 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_ftrapv_maincc_target_specific.json @@ -33,10 +33,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-ftrapv", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -85,10 +87,12 @@ "--prefix=../../bin/x86_64-cros-eabi-", "-ftrapv", "main.cc", + "-L/usr/x86_64-cros-eabi/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -137,10 +141,12 @@ "--prefix=../../bin/x86_64-cros-win-gnu-", "-ftrapv", "main.cc", + "-L/usr/x86_64-cros-win-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -190,10 +196,12 @@ "--prefix=../../bin/armv7m-cros-linux-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv7m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-linux-gnu" @@ -241,10 +249,12 @@ "--prefix=../../bin/armv7m-cros-eabi-", "-ftrapv", "main.cc", + "-L/usr/armv7m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-eabi" @@ -293,10 +303,12 @@ "--prefix=../../bin/armv7m-cros-win-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv7m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-win-gnu" @@ -345,10 +357,12 @@ "--prefix=../../bin/armv8m-cros-linux-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv8m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-linux-gnu" @@ -396,10 +410,12 @@ "--prefix=../../bin/armv8m-cros-eabi-", "-ftrapv", "main.cc", + "-L/usr/armv8m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-eabi" @@ -448,10 +464,12 @@ "--prefix=../../bin/armv8m-cros-win-gnu-", "-ftrapv", "main.cc", + "-L/usr/armv8m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-win-gnu" diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json index af33ad3a..69af166d 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_maincc_target_specific.json @@ -31,10 +31,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -81,10 +83,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-eabi-", "main.cc", + "-L/usr/x86_64-cros-eabi/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -131,10 +135,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-win-gnu-", "main.cc", + "-L/usr/x86_64-cros-win-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -182,10 +188,12 @@ "-mthumb", "--prefix=../../bin/armv7m-cros-linux-gnu-", "main.cc", + "-L/usr/armv7m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-linux-gnu" @@ -231,10 +239,12 @@ "-static-libgcc", "--prefix=../../bin/armv7m-cros-eabi-", "main.cc", + "-L/usr/armv7m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-eabi" @@ -281,10 +291,12 @@ "-mthumb", "--prefix=../../bin/armv7m-cros-win-gnu-", "main.cc", + "-L/usr/armv7m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7m-cros-win-gnu" @@ -331,10 +343,12 @@ "-mthumb", "--prefix=../../bin/armv8m-cros-linux-gnu-", "main.cc", + "-L/usr/armv8m-cros-linux-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-linux-gnu" @@ -380,10 +394,12 @@ "-static-libgcc", "--prefix=../../bin/armv8m-cros-eabi-", "main.cc", + "-L/usr/armv8m-cros-eabi/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-eabi" @@ -430,10 +446,12 @@ "-mthumb", "--prefix=../../bin/armv8m-cros-win-gnu-", "main.cc", + "-L/usr/armv8m-cros-win-gnu/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv8m-cros-win-gnu" diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json index bb03aa25..afc79511 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_path.json @@ -31,10 +31,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -84,10 +86,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -137,10 +141,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -190,10 +196,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -252,10 +260,12 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -317,10 +327,12 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -377,10 +389,12 @@ "-resource-dir=someResourceDir", "--gcc-toolchain=/usr", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -430,10 +444,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -480,10 +496,12 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -530,10 +548,12 @@ "-static-libgcc", "--prefix=a/b/c/d/e/bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/stable/a/b/c/d/e/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-Ba/b/c/d/e/bin", "-target", @@ -580,10 +600,12 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", @@ -633,10 +655,12 @@ "-static-libgcc", "--prefix=../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/tmp/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json index 26addbcb..3e140d59 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sanitizer_args.json @@ -34,10 +34,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -87,10 +89,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -140,10 +144,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -193,10 +199,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -242,14 +250,15 @@ "-Wno-deprecated-declarations", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", - "-fno-experimental-new-pass-manager", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=fuzzer", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -296,15 +305,16 @@ "-Wno-deprecated-declarations", "-fcrash-diagnostics-dir=/tmp/stable/clang_crash_diagnostics", "-static-libgcc", - "-fno-experimental-new-pass-manager", "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=address", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -353,10 +363,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fsanitize=address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -405,10 +417,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json index 2b73d51b..fbe038e7 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_specific_args.json @@ -49,10 +49,12 @@ "-Woverride-init", "-Wunsafe-loop-optimizations", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -101,10 +103,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-Wno-#warnings", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -153,10 +157,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-Wno-error=uninitialized", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -205,10 +211,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-someflag", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json index 96d6e65a..f2ccadb9 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clang_sysroot_wrapper_common.json @@ -22,6 +22,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -63,10 +64,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -116,10 +119,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -162,10 +167,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -215,10 +222,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "-D__KERNEL__", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -269,10 +278,12 @@ "--prefix=../../bin/armv7a-cros-linux-gnueabihf-", "-D__KERNEL__", "main.cc", + "-L/usr/armv7a-cros-linux-gnueabihf/usr/lib", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-B../../bin", "-target", "armv7a-cros-linux-gnueabihf" @@ -319,10 +330,12 @@ "--prefix=../../bin/x86_64-cros-linux-gnu-", "--sysroot=xyz", "main.cc", + "-Lxyz/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json b/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json index ad6cb480..dc641c96 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/clangtidy.json @@ -46,10 +46,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -77,10 +79,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -138,10 +142,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -170,10 +176,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -233,10 +241,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -268,10 +278,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -332,10 +344,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -364,10 +378,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json b/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json index ba7f61f4..54b994cc 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/force_disable_werror.json @@ -34,10 +34,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -87,10 +89,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -126,10 +130,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -183,10 +189,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -222,10 +230,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json index 5e62ca2e..5234715d 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_clang_syntax.json @@ -31,10 +31,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -56,6 +58,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -102,10 +105,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -127,6 +132,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -168,10 +174,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -221,10 +229,12 @@ "-static-libgcc", "--prefix=../../bin/x86_64-cros-linux-gnu-", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-Wno-implicit-int-float-conversion", "-Wno-compound-token-split-by-space", "-Wno-string-concatenation", "-Wno-deprecated-copy", + "-Wno-unused-but-set-variable", "-mno-movbe", "-B../../bin", "-target", @@ -246,6 +256,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_maincc_target_specific.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_maincc_target_specific.json index 5bb170bd..24f90fbf 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_maincc_target_specific.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_maincc_target_specific.json @@ -22,6 +22,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -55,6 +56,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-eabi/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -88,6 +90,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-win-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -121,7 +124,8 @@ "-Wtrampolines", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv7m-cros-linux-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -153,7 +157,8 @@ "-Wno-deprecated-declarations", "-Wtrampolines", "-static-libgcc", - "main.cc" + "main.cc", + "-L/usr/armv7m-cros-eabi/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -186,7 +191,8 @@ "-Wtrampolines", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv7m-cros-win-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -219,7 +225,8 @@ "-Wtrampolines", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv8m-cros-linux-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -251,7 +258,8 @@ "-Wno-deprecated-declarations", "-Wtrampolines", "-static-libgcc", - "main.cc" + "main.cc", + "-L/usr/armv8m-cros-eabi/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -284,7 +292,8 @@ "-Wtrampolines", "-static-libgcc", "-mthumb", - "main.cc" + "main.cc", + "-L/usr/armv8m-cros-win-gnu/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_path.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_path.json index 00f2a364..b2662930 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_path.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_path.json @@ -22,6 +22,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -58,6 +59,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -94,6 +96,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -127,6 +130,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -160,6 +164,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/tmp/stable/a/b/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -196,6 +201,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sanitizer_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sanitizer_args.json index 10ff7383..79e02c3d 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sanitizer_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sanitizer_args.json @@ -25,6 +25,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -61,6 +62,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -97,6 +99,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -133,6 +136,7 @@ "-static-libgcc", "-fsanitize=kernel-address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -168,6 +172,7 @@ "-static-libgcc", "-fsanitize=fuzzer", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -205,6 +210,7 @@ "-fsanitize=address", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -240,6 +246,7 @@ "-static-libgcc", "-fsanitize=address", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -275,6 +282,7 @@ "-static-libgcc", "-fprofile-instr-generate", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_specific_args.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_specific_args.json index 5228e12b..e0fe5515 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_specific_args.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_specific_args.json @@ -24,6 +24,7 @@ "-static-libgcc", "-march=goldmont", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -59,6 +60,7 @@ "-static-libgcc", "-march=goldmont-plus", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -94,6 +96,7 @@ "-static-libgcc", "-march=skylake", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sysroot_wrapper_common.json b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sysroot_wrapper_common.json index 23109bc4..ba2f292c 100644 --- a/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sysroot_wrapper_common.json +++ b/compiler_wrapper/testdata/cros_nonhardened_golden/gcc_sysroot_wrapper_common.json @@ -22,6 +22,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -54,6 +55,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -90,6 +92,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ] } @@ -120,6 +123,7 @@ "-Wtrampolines", "-static-libgcc", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -156,6 +160,7 @@ "-fno-stack-protector", "-D__KERNEL__", "main.cc", + "-L/usr/x86_64-cros-linux-gnu/usr/lib64", "-mno-movbe" ], "env_updates": [ @@ -192,7 +197,8 @@ "-mthumb", "-fno-stack-protector", "-D__KERNEL__", - "main.cc" + "main.cc", + "-L/usr/armv7a-cros-linux-gnueabihf/usr/lib" ], "env_updates": [ "CCACHE_DIR=/var/cache/distfiles/ccache", @@ -226,6 +232,7 @@ "-static-libgcc", "--sysroot=xyz", "main.cc", + "-Lxyz/usr/lib64", "-mno-movbe" ], "env_updates": [ diff --git a/cros_utils/bugs.py b/cros_utils/bugs.py new file mode 100755 index 00000000..88fb7675 --- /dev/null +++ b/cros_utils/bugs.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# Copyright 2021 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Utilities to file bugs.""" + +import base64 +import datetime +import enum +import json +import os +from typing import Any, Dict, List, Optional + +X20_PATH = '/google/data/rw/teams/c-compiler-chrome/prod_bugs' + + +class WellKnownComponents(enum.IntEnum): + """A listing of "well-known" components recognized by our infra.""" + CrOSToolchainPublic = -1 + CrOSToolchainPrivate = -2 + + +def _WriteBugJSONFile(object_type: str, json_object: Dict[str, Any]): + """Writes a JSON file to X20_PATH with the given bug-ish object.""" + final_object = { + 'type': object_type, + 'value': json_object, + } + + # The name of this has two parts: + # - An easily sortable time, to provide uniqueness and let our service send + # things in the order they were put into the outbox. + # - 64 bits of entropy, so two racing bug writes don't clobber the same file. + now = datetime.datetime.utcnow().isoformat('T', 'seconds') + 'Z' + entropy = base64.urlsafe_b64encode(os.getrandom(8)) + entropy_str = entropy.rstrip(b'=').decode('utf-8') + file_path = os.path.join(X20_PATH, f'{now}_{entropy_str}.json') + + temp_path = file_path + '.in_progress' + try: + with open(temp_path, 'w') as f: + json.dump(final_object, f) + os.rename(temp_path, file_path) + except: + os.remove(temp_path) + raise + return file_path + + +def AppendToExistingBug(bug_id: int, body: str): + """Sends a reply to an existing bug.""" + _WriteBugJSONFile('AppendToExistingBugRequest', { + 'body': body, + 'bug_id': bug_id, + }) + + +def CreateNewBug(component_id: int, + title: str, + body: str, + assignee: Optional[str] = None, + cc: Optional[List[str]] = None): + """Sends a request to create a new bug. + + Args: + component_id: The component ID to add. Anything from WellKnownComponents + also works. + title: Title of the bug. Must be nonempty. + body: Body of the bug. Must be nonempty. + assignee: Assignee of the bug. Must be either an email address, or a + "well-known" assignee (detective, mage). + cc: A list of emails to add to the CC list. Must either be an email + address, or a "well-known" individual (detective, mage). + """ + obj = { + 'component_id': component_id, + 'subject': title, + 'body': body, + } + + if assignee: + obj['assignee'] = assignee + + if cc: + obj['cc'] = cc + + _WriteBugJSONFile('FileNewBugRequest', obj) + + +def SendCronjobLog(cronjob_name: str, failed: bool, message: str): + """Sends the record of a cronjob to our bug infra. + + cronjob_name: The name of the cronjob. Expected to remain consistent over + time. + failed: Whether the job failed or not. + message: Any seemingly relevant context. This is pasted verbatim in a bug, if + the cronjob infra deems it worthy. + """ + _WriteBugJSONFile('ChrotomationCronjobUpdate', { + 'name': cronjob_name, + 'message': message, + 'failed': failed, + }) diff --git a/cros_utils/bugs_test.py b/cros_utils/bugs_test.py new file mode 100755 index 00000000..03dee64d --- /dev/null +++ b/cros_utils/bugs_test.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# Copyright 2021 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# We're testing protected methods, so allow protected access. +# pylint: disable=protected-access + +"""Tests bug filing bits.""" + +import json +import tempfile +import unittest +from unittest.mock import patch + +import bugs + + +class Tests(unittest.TestCase): + """Tests for the bugs module.""" + def testWritingJSONFileSeemsToWork(self): + """Tests JSON file writing.""" + old_x20_path = bugs.X20_PATH + + def restore_x20_path(): + bugs.X20_PATH = old_x20_path + + self.addCleanup(restore_x20_path) + + with tempfile.TemporaryDirectory() as tempdir: + bugs.X20_PATH = tempdir + file_path = bugs._WriteBugJSONFile( + 'ObjectType', { + 'foo': 'bar', + 'baz': bugs.WellKnownComponents.CrOSToolchainPublic, + }) + + self.assertTrue(file_path.startswith(tempdir), + f'Expected {file_path} to start with {tempdir}') + + with open(file_path) as f: + self.assertEqual( + json.load(f), + { + 'type': 'ObjectType', + 'value': { + 'foo': 'bar', + 'baz': int(bugs.WellKnownComponents.CrOSToolchainPublic), + }, + }, + ) + + @patch('bugs._WriteBugJSONFile') + def testAppendingToBugsSeemsToWork(self, mock_write_json_file): + """Tests AppendToExistingBug.""" + bugs.AppendToExistingBug(1234, 'hello, world!') + mock_write_json_file.assert_called_once_with( + 'AppendToExistingBugRequest', + { + 'body': 'hello, world!', + 'bug_id': 1234, + }, + ) + + @patch('bugs._WriteBugJSONFile') + def testBugCreationSeemsToWork(self, mock_write_json_file): + """Tests CreateNewBug.""" + test_case_additions = ( + {}, + { + 'component_id': bugs.WellKnownComponents.CrOSToolchainPublic, + }, + { + 'assignee': 'foo@gbiv.com', + 'cc': ['bar@baz.com'], + }, + ) + + for additions in test_case_additions: + test_case = { + 'component_id': 123, + 'title': 'foo', + 'body': 'bar', + **additions, + } + + bugs.CreateNewBug(**test_case) + + expected_output = { + 'component_id': test_case['component_id'], + 'subject': test_case['title'], + 'body': test_case['body'], + } + + assignee = test_case.get('assignee') + if assignee: + expected_output['assignee'] = assignee + + cc = test_case.get('cc') + if cc: + expected_output['cc'] = cc + + mock_write_json_file.assert_called_once_with( + 'FileNewBugRequest', + expected_output, + ) + mock_write_json_file.reset_mock() + + @patch('bugs._WriteBugJSONFile') + def testCronjobLogSendingSeemsToWork(self, mock_write_json_file): + """Tests SendCronjobLog.""" + bugs.SendCronjobLog('my_name', False, 'hello, world!') + mock_write_json_file.assert_called_once_with( + 'ChrotomationCronjobUpdate', + { + 'name': 'my_name', + 'message': 'hello, world!', + 'failed': False, + }, + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/cros_utils/manifest_versions.py b/cros_utils/manifest_versions.py deleted file mode 100644 index 4838de3c..00000000 --- a/cros_utils/manifest_versions.py +++ /dev/null @@ -1,162 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Tools for searching/manipulating the manifests repository.""" - -from __future__ import print_function - -__author__ = 'llozano@google.com (Luis Lozano)' - -import copy -import os -import re -import shutil -import tempfile -import time - -from cros_utils import command_executer -from cros_utils import logger - -MANIFEST_VERSION_MAIN_BRANCH = 'master' - - -def IsCrosVersion(version): - match = re.search(r'(\d+\.\d+\.\d+\.\d+)', version) - return match is not None - - -def IsRFormatCrosVersion(version): - match = re.search(r'(R\d+-\d+\.\d+\.\d+)', version) - return match is not None - - -def RFormatCrosVersion(version): - assert IsCrosVersion(version) - tmp_major, tmp_minor = version.split('.', 1) - rformat = 'R' + tmp_major + '-' + tmp_minor - assert IsRFormatCrosVersion(rformat) - return rformat - - -class ManifestVersions(object): - """This class handles interactions with the manifests repo.""" - - def __init__(self, internal=True): - self.internal = internal - self.clone_location = tempfile.mkdtemp() - self.ce = command_executer.GetCommandExecuter() - if internal: - versions_git = ('https://chrome-internal.googlesource.com/' - 'chromeos/manifest-versions.git') - else: - versions_git = ( - 'https://chromium.googlesource.com/chromiumos/manifest-versions.git') - commands = [ - 'cd {0}'.format(self.clone_location), - 'git clone {0}'.format(versions_git) - ] - ret = self.ce.RunCommands(commands) - if ret: - logger.GetLogger().LogFatal('Failed to clone manifest-versions.') - - def __del__(self): - if self.clone_location: - shutil.rmtree(self.clone_location) - - def TimeToVersionChromeOS(self, my_time): - """Convert timestamp to version number, in ChromeOS/Paladin.""" - cur_time = time.mktime(time.gmtime()) - des_time = float(my_time) - if cur_time - des_time > 7000000: - logger.GetLogger().LogFatal('The time you specify is too early.') - commands = [ - 'cd {0}'.format(self.clone_location), 'cd manifest-versions', - 'git checkout -f $(git rev-list' + - ' --max-count=1 --before={0} origin/{1})'.format( - my_time, MANIFEST_VERSION_MAIN_BRANCH) - ] - ret = self.ce.RunCommands(commands) - if ret: - logger.GetLogger().LogFatal('Failed to checkout manifest at ' - 'specified time') - path = os.path.realpath('{0}/manifest-versions/LKGM/lkgm.xml'.format( - self.clone_location)) - pp = path.split('/') - new_list = copy.deepcopy(pp) - for i, e in enumerate(pp): - if e == 'android-LKGM-candidates': - new_list[i] = 'paladin' - chrome_path = '/'.join(new_list) - if not os.path.exists(chrome_path): - logger.GetLogger().LogOutput('LKGM path is %s' % path) - logger.GetLogger().LogOutput('Cannot find path %s' % chrome_path) - pieces = os.path.basename(chrome_path).split('.') - pieces = pieces[:-2] - new_base = '.'.join(pieces) + '*' - wild_path = os.path.join('/', '/'.join(new_list[:-1]), new_base) - command = 'ls %s' % wild_path - ret, out, _ = self.ce.RunCommandWOutput(command) - if ret == 0: - out = out.strip() - files = out.split('\n') - latest = files[-1] - small = os.path.basename(latest).split('.xml')[0] - version = pp[-2] + '.' + small - else: - small = os.path.basename(path).split('.xml')[0] - version = pp[-2] + '.' + small - commands = [ - 'cd {0}'.format(self.clone_location), 'cd manifest-versions', - 'git checkout {0}'.format(MANIFEST_VERSION_MAIN_BRANCH) - ] - self.ce.RunCommands(commands) - return version - - def TimeToVersion(self, my_time): - """Convert timestamp to version number.""" - cur_time = time.mktime(time.gmtime()) - des_time = float(my_time) - if cur_time - des_time > 7000000: - logger.GetLogger().LogFatal('The time you specify is too early.') - commands = [ - 'cd {0}'.format(self.clone_location), 'cd manifest-versions', - 'git checkout -f $(git rev-list' + - ' --max-count=1 --before={0} origin/{1})'.format( - my_time, MANIFEST_VERSION_MAIN_BRANCH) - ] - ret = self.ce.RunCommands(commands) - if ret: - logger.GetLogger().LogFatal('Failed to checkout manifest at ' - 'specified time') - path = os.path.realpath('{0}/manifest-versions/LKGM/lkgm.xml'.format( - self.clone_location)) - pp = path.split('/') - small = os.path.basename(path).split('.xml')[0] - version = pp[-2] + '.' + small - commands = [ - 'cd {0}'.format(self.clone_location), 'cd manifest-versions', - 'git checkout {0}'.format(MANIFEST_VERSION_MAIN_BRANCH) - ] - self.ce.RunCommands(commands) - return version - - def GetManifest(self, version, to_file): - """Get the manifest file from a given chromeos-internal version.""" - assert not IsRFormatCrosVersion(version) - version = version.split('.', 1)[1] - os.chdir(self.clone_location) - files = [ - os.path.join(r, f) - for r, _, fs in os.walk('.') - for f in fs - if version in f - ] - if files: - command = 'cp {0} {1}'.format(files[0], to_file) - ret = self.ce.RunCommand(command) - if ret: - raise RuntimeError('Cannot copy manifest to {0}'.format(to_file)) - else: - raise RuntimeError('Version {0} is not available.'.format(version)) diff --git a/cros_utils/misc.py b/cros_utils/misc.py index 93d1b3b6..a0d0de73 100644 --- a/cros_utils/misc.py +++ b/cros_utils/misc.py @@ -15,7 +15,6 @@ import os import re import shutil import sys -import traceback from cros_utils import command_executer from cros_utils import logger @@ -24,8 +23,6 @@ CHROMEOS_SCRIPTS_DIR = '/mnt/host/source/src/scripts' TOOLCHAIN_UTILS_PATH = ('/mnt/host/source/src/third_party/toolchain-utils/' 'cros_utils/toolchain_utils.sh') -CROS_MAIN_BRANCH = 'cros/master' - def GetChromeOSVersionFromLSBVersion(lsb_version): """Get Chromeos version from Lsb version.""" @@ -112,8 +109,8 @@ def FormatQuotedCommand(command): def FormatCommands(commands): - return ApplySubs( - str(commands), ('&&', '&&\n'), (';', ';\n'), (r'\n+\s*', '\n')) + return ApplySubs(str(commands), ('&&', '&&\n'), (';', ';\n'), + (r'\n+\s*', '\n')) def GetImageDir(chromeos_root, board): @@ -155,16 +152,16 @@ def GetBuildPackagesCommand(board, usepkg=False, debug=False): withdebug_flag = '--nowithdebug' return ('%s/build_packages %s --withdev --withtest --withautotest ' '--skip_toolchain_update %s --board=%s ' - '--accept_licenses=@CHROMEOS' % (CHROMEOS_SCRIPTS_DIR, usepkg_flag, - withdebug_flag, board)) + '--accept_licenses=@CHROMEOS' % + (CHROMEOS_SCRIPTS_DIR, usepkg_flag, withdebug_flag, board)) def GetBuildImageCommand(board, dev=False): dev_args = '' if dev: dev_args = '--noenable_rootfs_verification --disk_layout=2gb-rootfs' - return ('%s/build_image --board=%s %s test' % (CHROMEOS_SCRIPTS_DIR, board, - dev_args)) + return ('%s/build_image --board=%s %s test' % + (CHROMEOS_SCRIPTS_DIR, board, dev_args)) def GetSetupBoardCommand(board, usepkg=None, force=None): @@ -193,8 +190,8 @@ def CanonicalizePath(path): def GetCtargetFromBoard(board, chromeos_root): """Get Ctarget from board.""" base_board = board.split('_')[0] - command = ('source %s; get_ctarget_from_board %s' % (TOOLCHAIN_UTILS_PATH, - base_board)) + command = ('source %s; get_ctarget_from_board %s' % + (TOOLCHAIN_UTILS_PATH, base_board)) ce = command_executer.GetCommandExecuter() ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command) if ret != 0: @@ -207,8 +204,8 @@ def GetCtargetFromBoard(board, chromeos_root): def GetArchFromBoard(board, chromeos_root): """Get Arch from board.""" base_board = board.split('_')[0] - command = ( - 'source %s; get_board_arch %s' % (TOOLCHAIN_UTILS_PATH, base_board)) + command = ('source %s; get_board_arch %s' % + (TOOLCHAIN_UTILS_PATH, base_board)) ce = command_executer.GetCommandExecuter() ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command) if ret != 0: @@ -316,23 +313,22 @@ def WorkingDirectory(new_dir): def HasGitStagedChanges(git_dir): """Return True if git repository has staged changes.""" - command = 'cd {0} && git diff --quiet --cached --exit-code HEAD'.format( - git_dir) + command = f'cd {git_dir} && git diff --quiet --cached --exit-code HEAD' return command_executer.GetCommandExecuter().RunCommand( command, print_to_console=False) def HasGitUnstagedChanges(git_dir): """Return True if git repository has un-staged changes.""" - command = 'cd {0} && git diff --quiet --exit-code HEAD'.format(git_dir) + command = f'cd {git_dir} && git diff --quiet --exit-code HEAD' return command_executer.GetCommandExecuter().RunCommand( command, print_to_console=False) def HasGitUntrackedChanges(git_dir): """Return True if git repository has un-tracked changes.""" - command = ('cd {0} && test -z ' - '$(git ls-files --exclude-standard --others)').format(git_dir) + command = (f'cd {git_dir} && test -z ' + '$(git ls-files --exclude-standard --others)') return command_executer.GetCommandExecuter().RunCommand( command, print_to_console=False) @@ -352,8 +348,8 @@ def GitGetCommitHash(git_dir, commit_symbolic_name): The git hash for the symbolic name or None if fails. """ - command = ('cd {0} && git log -n 1 --pretty="format:%H" {1}').format( - git_dir, commit_symbolic_name) + command = (f'cd {git_dir} && git log -n 1' + f' --pretty="format:%H" {commit_symbolic_name}') rv, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( command, print_to_console=False) if rv == 0: @@ -393,7 +389,7 @@ def GetGitChangesAsList(git_dir, path=None, staged=False): Returns: A list containing all the changed files. """ - command = 'cd {0} && git diff --name-only'.format(git_dir) + command = f'cd {git_dir} && git diff --name-only' if staged: command += ' --cached' if path: @@ -408,8 +404,8 @@ def GetGitChangesAsList(git_dir, path=None, staged=False): def IsChromeOsTree(chromeos_root): return (os.path.isdir( - os.path.join(chromeos_root, 'src/third_party/chromiumos-overlay')) and - os.path.isdir(os.path.join(chromeos_root, 'manifest'))) + os.path.join(chromeos_root, 'src/third_party/chromiumos-overlay')) + and os.path.isdir(os.path.join(chromeos_root, 'manifest'))) def DeleteChromeOsTree(chromeos_root, dry_run=False): @@ -423,11 +419,10 @@ def DeleteChromeOsTree(chromeos_root, dry_run=False): True if everything is ok. """ if not IsChromeOsTree(chromeos_root): - logger.GetLogger().LogWarning( - '"{0}" does not seem to be a valid chromeos tree, do nothing.'.format( - chromeos_root)) + logger.GetLogger().LogWarning(f'"{chromeos_root}" does not seem to be a' + ' valid chromeos tree, do nothing.') return False - cmd0 = 'cd {0} && cros_sdk --delete'.format(chromeos_root) + cmd0 = f'cd {chromeos_root} && cros_sdk --delete' if dry_run: print(cmd0) else: @@ -435,10 +430,10 @@ def DeleteChromeOsTree(chromeos_root, dry_run=False): cmd0, print_to_console=True) != 0: return False - cmd1 = ('export CHROMEOSDIRNAME="$(dirname $(cd {0} && pwd))" && ' - 'export CHROMEOSBASENAME="$(basename $(cd {0} && pwd))" && ' - 'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME' - ).format(chromeos_root) + cmd1 = ( + f'export CHROMEOSDIRNAME="$(dirname $(cd {chromeos_root} && pwd))" && ' + f'export CHROMEOSBASENAME="$(basename $(cd {chromeos_root} && pwd))" && ' + 'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME') if dry_run: print(cmd1) return True @@ -447,54 +442,6 @@ def DeleteChromeOsTree(chromeos_root, dry_run=False): cmd1, print_to_console=True) == 0 -def ApplyGerritPatches(chromeos_root, - gerrit_patch_string, - branch=CROS_MAIN_BRANCH): - """Apply gerrit patches on a chromeos tree. - - Args: - chromeos_root: chromeos tree path - gerrit_patch_string: a patch string just like the one gives to cbuildbot, - 'id1 id2 *id3 ... idn'. A prefix of '* means this is an internal patch. - branch: the tree based on which to apply the patches. - - Returns: - True if success. - """ - - ### First of all, we need chromite libs - sys.path.append(os.path.join(chromeos_root, 'chromite')) - # Imports below are ok after modifying path to add chromite. - # Pylint cannot detect that and complains. - # pylint: disable=import-error, import-outside-toplevel - from lib import git - from lib import gerrit - manifest = git.ManifestCheckout(chromeos_root) - patch_list = gerrit_patch_string.split(' ') - ### This takes time, print log information. - logger.GetLogger().LogOutput('Retrieving patch information from server ...') - patch_info_list = gerrit.GetGerritPatchInfo(patch_list) - for pi in patch_info_list: - project_checkout = manifest.FindCheckout(pi.project, strict=False) - if not project_checkout: - logger.GetLogger().LogError( - 'Failed to find patch project "{project}" in manifest.'.format( - project=pi.project)) - return False - - pi_str = '{project}:{ref}'.format(project=pi.project, ref=pi.ref) - try: - project_git_path = project_checkout.GetPath(absolute=True) - logger.GetLogger().LogOutput('Applying patch "{0}" in "{1}" ...'.format( - pi_str, project_git_path)) - pi.Apply(project_git_path, branch, trivial=False) - except Exception: - traceback.print_exc(file=sys.stdout) - logger.GetLogger().LogError('Failed to apply patch "{0}"'.format(pi_str)) - return False - return True - - def BooleanPrompt(prompt='Do you want to continue?', default=True, true_value='yes', @@ -515,8 +462,8 @@ def BooleanPrompt(prompt='Do you want to continue?', true_value, false_value = true_value.lower(), false_value.lower() true_text, false_text = true_value, false_value if true_value == false_value: - raise ValueError( - 'true_value and false_value must differ: got %r' % true_value) + raise ValueError('true_value and false_value must differ: got %r' % + true_value) if default: true_text = true_text[0].upper() + true_text[1:] diff --git a/crosperf/crosperf_autolock.py b/crosperf/crosperf_autolock.py new file mode 100755 index 00000000..b593fa9c --- /dev/null +++ b/crosperf/crosperf_autolock.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 + +# Copyright 2021 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Wrapper script to automatically lock devices for crosperf.""" + +import os +import sys +import argparse +import subprocess +import contextlib +import json +from typing import Optional, Any +import dataclasses + +# Have to do sys.path hackery because crosperf relies on PYTHONPATH +# modifications. +PARENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PARENT_DIR) + + +def main(sys_args: list[str]) -> Optional[str]: + """Run crosperf_autolock. Returns error msg or None""" + args, leftover_args = parse_args(sys_args) + fleet_params = [ + CrosfleetParams(board=args.board, + pool=args.pool, + lease_time=args.lease_time) + for _ in range(args.num_leases) + ] + if not fleet_params: + return ('No board names identified. If you want to use' + ' a known host, just use crosperf directly.') + try: + _run_crosperf(fleet_params, args.dut_lock_timeout, leftover_args) + except BoardLockError as e: + _eprint('ERROR:', e) + _eprint('May need to login to crosfleet? Run "crosfleet login"') + _eprint('The leases may also be successful later on. ' + 'Check with "crosfleet dut leases"') + return 'crosperf_autolock failed' + except BoardReleaseError as e: + _eprint('ERROR:', e) + _eprint('May need to re-run "crosfleet dut abandon"') + return 'crosperf_autolock failed' + return None + + +def parse_args(args: list[str]) -> tuple[Any, list]: + """Parse the CLI arguments.""" + parser = argparse.ArgumentParser( + 'crosperf_autolock', + description='Wrapper around crosperf' + ' to autolock DUTs from crosfleet.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--board', + type=str, + help='Space or comma separated list of boards to lock', + required=True, + default=argparse.SUPPRESS) + parser.add_argument('--num-leases', + type=int, + help='Number of boards to lock.', + metavar='NUM', + default=1) + parser.add_argument('--pool', + type=str, + help='Pool to pull from.', + default='DUT_POOL_QUOTA') + parser.add_argument('--dut-lock-timeout', + type=float, + metavar='SEC', + help='Number of seconds we want to try to lease a board' + ' from crosfleet. This option does NOT change the' + ' lease length.', + default=600) + parser.add_argument('--lease-time', + type=int, + metavar='MIN', + help='Number of minutes to lock the board. Max is 1440.', + default=1440) + parser.epilog = ( + 'For more detailed flags, you have to read the args taken by the' + ' crosperf executable. Args are passed transparently to crosperf.') + return parser.parse_known_args(args) + + +class BoardLockError(Exception): + """Error to indicate failure to lock a board.""" + + def __init__(self, msg: str): + self.msg = 'BoardLockError: ' + msg + super().__init__(self.msg) + + +class BoardReleaseError(Exception): + """Error to indicate failure to release a board.""" + + def __init__(self, msg: str): + self.msg = 'BoardReleaseError: ' + msg + super().__init__(self.msg) + + +@dataclasses.dataclass(frozen=True) +class CrosfleetParams: + """Dataclass to hold all crosfleet parameterizations.""" + board: str + pool: str + lease_time: int + + +def _eprint(*msg, **kwargs): + print(*msg, file=sys.stderr, **kwargs) + + +def _run_crosperf(crosfleet_params: list[CrosfleetParams], lock_timeout: float, + leftover_args: list[str]): + """Autolock devices and run crosperf with leftover arguments. + + Raises: + BoardLockError: When board was unable to be locked. + BoardReleaseError: When board was unable to be released. + """ + if not crosfleet_params: + raise ValueError('No crosfleet params given; cannot call crosfleet.') + + # We'll assume all the boards are the same type, which seems to be the case + # in experiments that actually get used. + passed_board_arg = crosfleet_params[0].board + with contextlib.ExitStack() as stack: + dut_hostnames = [] + for param in crosfleet_params: + print( + f'Sent lock request for {param.board} for {param.lease_time} minutes' + '\nIf this fails, you may need to run "crosfleet dut abandon <...>"') + # May raise BoardLockError, abandoning previous DUTs. + dut_hostname = stack.enter_context( + crosfleet_machine_ctx( + param.board, + param.lease_time, + lock_timeout, + {'label-pool': param.pool}, + )) + if dut_hostname: + print(f'Locked {param.board} machine: {dut_hostname}') + dut_hostnames.append(dut_hostname) + + # We import crosperf late, because this import is extremely slow. + # We don't want the user to wait several seconds just to get + # help info. + import crosperf + for dut_hostname in dut_hostnames: + crosperf.Main([ + sys.argv[0], + '--no_lock', + 'True', + '--remote', + dut_hostname, + '--board', + passed_board_arg, + ] + leftover_args) + + +@contextlib.contextmanager +def crosfleet_machine_ctx(board: str, + lease_minutes: int, + lock_timeout: float, + dims: dict[str, Any], + abandon_timeout: float = 120.0) -> Any: + """Acquire dut from crosfleet, and release once it leaves the context. + + Args: + board: Board type to lease. + lease_minutes: Length of lease, in minutes. + lock_timeout: How long to wait for a lock until quitting. + dims: Dictionary of dimension arguments to pass to crosfleet's '-dims' + abandon_timeout (optional): How long to wait for releasing until quitting. + + Yields: + A string representing the crosfleet DUT hostname. + + Raises: + BoardLockError: When board was unable to be locked. + BoardReleaseError: When board was unable to be released. + """ + # This lock may raise an exception, but if it does, we can't release + # the DUT anyways as we won't have the dut_hostname. + dut_hostname = crosfleet_autolock(board, lease_minutes, dims, lock_timeout) + try: + yield dut_hostname + finally: + if dut_hostname: + crosfleet_release(dut_hostname, abandon_timeout) + + +def crosfleet_autolock(board: str, lease_minutes: int, dims: dict[str, Any], + timeout_sec: float) -> str: + """Lock a device using crosfleet, paramaterized by the board type. + + Args: + board: Board of the DUT we want to lock. + lease_minutes: Number of minutes we're trying to lease the DUT for. + dims: Dictionary of dimension arguments to pass to crosfleet's '-dims' + timeout_sec: Number of seconds to try to lease the DUT. Default 120s. + + Returns: + The hostname of the board, or empty string if it couldn't be parsed. + + Raises: + BoardLockError: When board was unable to be locked. + """ + crosfleet_cmd_args = [ + 'crosfleet', + 'dut', + 'lease', + '-json', + '-reason="crosperf autolock"', + f'-board={board}', + f'-minutes={lease_minutes}', + ] + if dims: + dims_arg = ','.join('{}={}'.format(k, v) for k, v in dims.items()) + crosfleet_cmd_args.extend(['-dims', f'{dims_arg}']) + + try: + output = subprocess.check_output(crosfleet_cmd_args, + timeout=timeout_sec, + encoding='utf-8') + except subprocess.CalledProcessError as e: + raise BoardLockError( + f'crosfleet dut lease failed with exit code: {e.returncode}') + except subprocess.TimeoutExpired as e: + raise BoardLockError(f'crosfleet dut lease timed out after {timeout_sec}s;' + ' please abandon the dut manually.') + + try: + json_obj = json.loads(output) + dut_hostname = json_obj['DUT']['Hostname'] + if not isinstance(dut_hostname, str): + raise TypeError('dut_hostname was not a string') + except (json.JSONDecodeError, IndexError, KeyError, TypeError) as e: + raise BoardLockError( + f'crosfleet dut lease output was parsed incorrectly: {e!r};' + f' observed output was {output}') + return _maybe_append_suffix(dut_hostname) + + +def crosfleet_release(dut_hostname: str, timeout_sec: float = 120.0): + """Release a crosfleet device. + + Consider using the context managed crosfleet_machine_context + + Args: + dut_hostname: Name of the device we want to release. + timeout_sec: Number of seconds to try to release the DUT. Default is 120s. + + Raises: + BoardReleaseError: Potentially failed to abandon the lease. + """ + crosfleet_cmd_args = [ + 'crosfleet', + 'dut', + 'abandon', + dut_hostname, + ] + exit_code = subprocess.call(crosfleet_cmd_args, timeout=timeout_sec) + if exit_code != 0: + raise BoardReleaseError( + f'"crosfleet dut abandon" had exit code {exit_code}') + + +def _maybe_append_suffix(hostname: str) -> str: + if hostname.endswith('.cros') or '.cros.' in hostname: + return hostname + return hostname + '.cros' + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/crosperf/crosperf_unittest.py b/crosperf/crosperf_unittest.py index 9c7d52a1..774159ff 100755 --- a/crosperf/crosperf_unittest.py +++ b/crosperf/crosperf_unittest.py @@ -55,20 +55,19 @@ class CrosperfTest(unittest.TestCase): def testConvertOptionsToSettings(self): parser = argparse.ArgumentParser() - parser.add_argument( - '-l', - '--log_dir', - dest='log_dir', - default='', - help='The log_dir, default is under ' - '<crosperf_logs>/logs') + parser.add_argument('-l', + '--log_dir', + dest='log_dir', + default='', + help='The log_dir, default is under ' + '<crosperf_logs>/logs') crosperf.SetupParserOptions(parser) argv = ['crosperf/crosperf.py', 'temp.exp', '--rerun=True'] options, _ = parser.parse_known_args(argv) settings = crosperf.ConvertOptionsToSettings(options) self.assertIsNotNone(settings) self.assertIsInstance(settings, settings_factory.GlobalSettings) - self.assertEqual(len(settings.fields), 39) + self.assertEqual(len(settings.fields), 40) self.assertTrue(settings.GetField('rerun')) argv = ['crosperf/crosperf.py', 'temp.exp'] options, _ = parser.parse_known_args(argv) diff --git a/crosperf/default_remotes b/crosperf/default_remotes index 45110752..faecb833 100644 --- a/crosperf/default_remotes +++ b/crosperf/default_remotes @@ -1,9 +1,8 @@ -bob : chromeos2-row10-rack9-host3.cros chromeos6-row3-rack13-host15.cros -chell : chromeos2-row9-rack9-host1.cros chromeos2-row9-rack9-host3.cros -coral : chromeos2-row9-rack9-host9.cros chromeos2-row9-rack9-host11.cros chromeos2-row9-rack9-host13.cros +bob : chromeos6-row4-rack13-host6.cros +chell : chromeos2-row1-rack10-host2.cros chromeos2-row1-rack10-host4.cros +coral : chromeos6-row5-rack6-host1.cros chromeos6-row5-rack6-host3.cros chromeos6-row5-rack6-host5.cros elm : chromeos6-row14-rack15-host21.cros -kefka : chromeos6-row6-rack22-host2.cros chromeos6-row6-rack22-host3.cros -lulu : chromeos2-row9-rack9-host5.cros chromeos2-row9-rack9-host7.cros -nautilus : chromeos2-row10-rack9-host9.cros chromeos2-row10-rack9-host11.cros -snappy : chromeos2-row10-rack9-host5.cros chromeos2-row10-rack9-host7.cros -veyron_tiger : chromeos2-row9-rack9-host17.cros +kefka : chromeos6-row6-rack22-host2.cros chromeos6-row6-rack22-host3.cros chromeos6-row11-rack22-host7.cros +nautilus : chromeos6-row5-rack10-host1.cros chromeos6-row5-rack10-host3.cros +snappy : chromeos6-row3-rack20-host1.cros chromeos6-row3-rack20-host3.cros +veyron_tiger : chromeos6-row3-rack7-host1.cros diff --git a/crosperf/experiment.py b/crosperf/experiment.py index 854d7f77..e919f6ee 100644 --- a/crosperf/experiment.py +++ b/crosperf/experiment.py @@ -29,7 +29,7 @@ class Experiment(object): cache_conditions, labels, benchmarks, experiment_file, email_to, acquire_timeout, log_dir, log_level, share_cache, results_directory, compress_results, locks_directory, cwp_dso, - ignore_min_max, crosfleet, dut_config): + ignore_min_max, crosfleet, dut_config, no_lock: bool): self.name = name self.working_directory = working_directory self.remote = remote @@ -57,6 +57,7 @@ class Experiment(object): self.cwp_dso = cwp_dso self.ignore_min_max = ignore_min_max self.crosfleet = crosfleet + self.no_lock = no_lock self.l = logger.GetLogger(log_dir) if not self.benchmarks: diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py index 73928756..a9594a20 100644 --- a/crosperf/experiment_factory.py +++ b/crosperf/experiment_factory.py @@ -101,7 +101,8 @@ class ExperimentFactory(object): def AppendBenchmarkSet(self, benchmarks, benchmark_list, test_args, iterations, rm_chroot_tmp, perf_args, suite, - show_all_results, retries, run_local, cwp_dso, weight): + show_all_results, retries, run_local, cwp_dso, + weight): """Add all the tests in a set to the benchmarks list.""" for test_name in benchmark_list: telemetry_benchmark = Benchmark(test_name, test_name, test_args, @@ -121,6 +122,7 @@ class ExperimentFactory(object): log_level = 'verbose' crosfleet = global_settings.GetField('crosfleet') + no_lock = bool(global_settings.GetField('no_lock')) # Check whether crosfleet tool is installed correctly for crosfleet mode. if crosfleet and not self.CheckCrosfleetTool(chromeos_root, log_level): sys.exit(0) @@ -257,10 +259,10 @@ class ExperimentFactory(object): if suite == 'telemetry_Crosperf': if test_name == 'all_perfv2': - self.AppendBenchmarkSet(benchmarks, telemetry_perfv2_tests, test_args, - iterations, rm_chroot_tmp, perf_args, suite, - show_all_results, retries, run_local, cwp_dso, - weight) + self.AppendBenchmarkSet(benchmarks, telemetry_perfv2_tests, + test_args, iterations, rm_chroot_tmp, + perf_args, suite, show_all_results, retries, + run_local, cwp_dso, weight) elif test_name == 'all_pagecyclers': self.AppendBenchmarkSet(benchmarks, telemetry_pagecycler_tests, test_args, iterations, rm_chroot_tmp, @@ -270,21 +272,20 @@ class ExperimentFactory(object): self.AppendBenchmarkSet(benchmarks, telemetry_crosbolt_perf_tests, test_args, iterations, rm_chroot_tmp, perf_args, 'telemetry_Crosperf', - show_all_results, retries, run_local, cwp_dso, - weight) - self.AppendBenchmarkSet( - benchmarks, - crosbolt_perf_tests, - '', - iterations, - rm_chroot_tmp, - perf_args, - '', - show_all_results, - retries, - run_local=False, - cwp_dso=cwp_dso, - weight=weight) + show_all_results, retries, run_local, + cwp_dso, weight) + self.AppendBenchmarkSet(benchmarks, + crosbolt_perf_tests, + '', + iterations, + rm_chroot_tmp, + perf_args, + '', + show_all_results, + retries, + run_local=False, + cwp_dso=cwp_dso, + weight=weight) elif test_name == 'all_toolchain_perf': self.AppendBenchmarkSet(benchmarks, telemetry_toolchain_perf_tests, test_args, iterations, rm_chroot_tmp, @@ -324,10 +325,10 @@ class ExperimentFactory(object): # weight=weight)) elif test_name == 'all_toolchain_perf_old': self.AppendBenchmarkSet(benchmarks, - telemetry_toolchain_old_perf_tests, test_args, - iterations, rm_chroot_tmp, perf_args, suite, - show_all_results, retries, run_local, cwp_dso, - weight) + telemetry_toolchain_old_perf_tests, + test_args, iterations, rm_chroot_tmp, + perf_args, suite, show_all_results, retries, + run_local, cwp_dso, weight) else: benchmark = Benchmark(benchmark_name, test_name, test_args, iterations, rm_chroot_tmp, perf_args, suite, @@ -336,34 +337,32 @@ class ExperimentFactory(object): benchmarks.append(benchmark) else: if test_name == 'all_graphics_perf': - self.AppendBenchmarkSet( - benchmarks, - graphics_perf_tests, - '', - iterations, - rm_chroot_tmp, - perf_args, - '', - show_all_results, - retries, - run_local=False, - cwp_dso=cwp_dso, - weight=weight) + self.AppendBenchmarkSet(benchmarks, + graphics_perf_tests, + '', + iterations, + rm_chroot_tmp, + perf_args, + '', + show_all_results, + retries, + run_local=False, + cwp_dso=cwp_dso, + weight=weight) else: # Add the single benchmark. - benchmark = Benchmark( - benchmark_name, - test_name, - test_args, - iterations, - rm_chroot_tmp, - perf_args, - suite, - show_all_results, - retries, - run_local=False, - cwp_dso=cwp_dso, - weight=weight) + benchmark = Benchmark(benchmark_name, + test_name, + test_args, + iterations, + rm_chroot_tmp, + perf_args, + suite, + show_all_results, + retries, + run_local=False, + cwp_dso=cwp_dso, + weight=weight) benchmarks.append(benchmark) if not benchmarks: @@ -410,8 +409,8 @@ class ExperimentFactory(object): # TODO(yunlian): We should consolidate code in machine_manager.py # to derermine whether we are running from within google or not - if ('corp.google.com' in socket.gethostname() and not my_remote and - not crosfleet): + if ('corp.google.com' in socket.gethostname() and not my_remote + and not crosfleet): my_remote = self.GetDefaultRemotes(board) if global_settings.GetField('same_machine') and len(my_remote) > 1: raise RuntimeError('Only one remote is allowed when same_machine ' @@ -422,8 +421,8 @@ class ExperimentFactory(object): # pylint: disable=too-many-function-args label = MockLabel(label_name, build, image, autotest_path, debug_path, chromeos_root, board, my_remote, image_args, - cache_dir, cache_only, log_level, compiler, crosfleet, - chrome_src) + cache_dir, cache_only, log_level, compiler, + crosfleet, chrome_src) else: label = Label(label_name, build, image, autotest_path, debug_path, chromeos_root, board, my_remote, image_args, cache_dir, @@ -439,18 +438,33 @@ class ExperimentFactory(object): if crosfleet: for remote in all_remote: self.CheckRemotesInCrosfleet(remote) - experiment = Experiment(experiment_name, all_remote, working_directory, - chromeos_root, cache_conditions, labels, benchmarks, - experiment_file.Canonicalize(), email, - acquire_timeout, log_dir, log_level, share_cache, - results_dir, compress_results, locks_dir, cwp_dso, - ignore_min_max, crosfleet, dut_config) + experiment = Experiment(experiment_name, + all_remote, + working_directory, + chromeos_root, + cache_conditions, + labels, + benchmarks, + experiment_file.Canonicalize(), + email, + acquire_timeout, + log_dir, + log_level, + share_cache, + results_dir, + compress_results, + locks_dir, + cwp_dso, + ignore_min_max, + crosfleet, + dut_config, + no_lock=no_lock) return experiment def GetDefaultRemotes(self, board): - default_remotes_file = os.path.join( - os.path.dirname(__file__), 'default_remotes') + default_remotes_file = os.path.join(os.path.dirname(__file__), + 'default_remotes') try: with open(default_remotes_file) as f: for line in f: @@ -480,8 +494,8 @@ class ExperimentFactory(object): l = logger.GetLogger() l.LogOutput('Crosfleet tool not installed, trying to install it.') ce = command_executer.GetCommandExecuter(l, log_level=log_level) - setup_lab_tools = os.path.join(chromeos_root, 'chromeos-admin', 'lab-tools', - 'setup_lab_tools') + setup_lab_tools = os.path.join(chromeos_root, 'chromeos-admin', + 'lab-tools', 'setup_lab_tools') cmd = '%s' % setup_lab_tools status = ce.RunCommand(cmd) if status != 0: diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py index 78cf780c..9637c108 100755 --- a/crosperf/experiment_factory_unittest.py +++ b/crosperf/experiment_factory_unittest.py @@ -79,14 +79,14 @@ EXPERIMENT_FILE_2 = """ class ExperimentFactoryTest(unittest.TestCase): """Class for running experiment factory unittests.""" - def setUp(self): self.append_benchmark_call_args = [] def testLoadExperimentFile1(self): experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1)) - exp = ExperimentFactory().GetExperiment( - experiment_file, working_directory='', log_dir='') + exp = ExperimentFactory().GetExperiment(experiment_file, + working_directory='', + log_dir='') self.assertEqual(exp.remote, ['chromeos-alex3']) self.assertEqual(len(exp.benchmarks), 2) @@ -104,8 +104,9 @@ class ExperimentFactoryTest(unittest.TestCase): def testLoadExperimentFile2CWP(self): experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_2)) - exp = ExperimentFactory().GetExperiment( - experiment_file, working_directory='', log_dir='') + exp = ExperimentFactory().GetExperiment(experiment_file, + working_directory='', + log_dir='') self.assertEqual(exp.cwp_dso, 'kallsyms') self.assertEqual(len(exp.benchmarks), 2) self.assertEqual(exp.benchmarks[0].weight, 0.8) @@ -240,11 +241,12 @@ class ExperimentFactoryTest(unittest.TestCase): ef = ExperimentFactory() bench_list = [] - ef.AppendBenchmarkSet(bench_list, experiment_factory.telemetry_perfv2_tests, - '', 1, False, '', 'telemetry_Crosperf', False, 0, - False, '', 0) - self.assertEqual( - len(bench_list), len(experiment_factory.telemetry_perfv2_tests)) + ef.AppendBenchmarkSet(bench_list, + experiment_factory.telemetry_perfv2_tests, '', 1, + False, '', 'telemetry_Crosperf', False, 0, False, '', + 0) + self.assertEqual(len(bench_list), + len(experiment_factory.telemetry_perfv2_tests)) self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark)) bench_list = [] @@ -252,17 +254,17 @@ class ExperimentFactoryTest(unittest.TestCase): experiment_factory.telemetry_pagecycler_tests, '', 1, False, '', 'telemetry_Crosperf', False, 0, False, '', 0) - self.assertEqual( - len(bench_list), len(experiment_factory.telemetry_pagecycler_tests)) + self.assertEqual(len(bench_list), + len(experiment_factory.telemetry_pagecycler_tests)) self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark)) bench_list = [] ef.AppendBenchmarkSet(bench_list, - experiment_factory.telemetry_toolchain_perf_tests, '', - 1, False, '', 'telemetry_Crosperf', False, 0, False, - '', 0) - self.assertEqual( - len(bench_list), len(experiment_factory.telemetry_toolchain_perf_tests)) + experiment_factory.telemetry_toolchain_perf_tests, + '', 1, False, '', 'telemetry_Crosperf', False, 0, + False, '', 0) + self.assertEqual(len(bench_list), + len(experiment_factory.telemetry_toolchain_perf_tests)) self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark)) @mock.patch.object(socket, 'gethostname') @@ -370,7 +372,8 @@ class ExperimentFactoryTest(unittest.TestCase): global_settings.SetField('same_machine', 'true') global_settings.SetField('same_specs', 'true') - self.assertRaises(Exception, ef.GetExperiment, mock_experiment_file, '', '') + self.assertRaises(Exception, ef.GetExperiment, mock_experiment_file, '', + '') label_settings.SetField('remote', '') global_settings.SetField('remote', '123.45.67.89') exp = ef.GetExperiment(mock_experiment_file, '', '') @@ -399,7 +402,7 @@ class ExperimentFactoryTest(unittest.TestCase): def test_get_default_remotes(self): board_list = [ - 'bob', 'chell', 'coral', 'elm', 'kefka', 'lulu', 'nautilus', 'snappy', + 'bob', 'chell', 'coral', 'elm', 'kefka', 'nautilus', 'snappy', 'veyron_tiger' ] diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py index 49aff425..6daef780 100644 --- a/crosperf/experiment_runner.py +++ b/crosperf/experiment_runner.py @@ -160,8 +160,8 @@ class ExperimentRunner(object): cache.Init(br.label.chromeos_image, br.label.chromeos_root, br.benchmark.test_name, br.iteration, br.test_args, br.profiler_args, br.machine_manager, br.machine, - br.label.board, br.cache_conditions, br.logger(), br.log_level, - br.label, br.share_cache, br.benchmark.suite, + br.label.board, br.cache_conditions, br.logger(), + br.log_level, br.label, br.share_cache, br.benchmark.suite, br.benchmark.show_all_results, br.benchmark.run_local, br.benchmark.cwp_dso) cache_dir = cache.GetCacheDirForWrite() @@ -176,7 +176,7 @@ class ExperimentRunner(object): # no-op task on the DUT and new test created will be hanging there. # TODO(zhizhouy): Need to check whether machine is ready or not before # assigning a test to it. - if not experiment.crosfleet: + if not experiment.no_lock and not experiment.crosfleet: self._LockAllMachines(experiment) # Calculate all checksums of avaiable/locked machines, to ensure same # label has same machines for testing @@ -236,8 +236,8 @@ class ExperimentRunner(object): if not benchmark_run.cache_hit: send_mail = True break - if (not send_mail and not experiment.email_to or - config.GetConfig('no_email')): + if (not send_mail and not experiment.email_to + or config.GetConfig('no_email')): return label_names = [] @@ -245,7 +245,8 @@ class ExperimentRunner(object): label_names.append(label.name) subject = '%s: %s' % (experiment.name, ' vs. '.join(label_names)) - text_report = TextResultsReport.FromExperiment(experiment, True).GetReport() + text_report = TextResultsReport.FromExperiment(experiment, + True).GetReport() text_report += ('\nResults are stored in %s.\n' % experiment.results_directory) text_report = "<pre style='font-size: 13px'>%s</pre>" % text_report @@ -253,12 +254,11 @@ class ExperimentRunner(object): attachment = EmailSender.Attachment('report.html', html_report) email_to = experiment.email_to or [] email_to.append(getpass.getuser()) - EmailSender().SendEmail( - email_to, - subject, - text_report, - attachments=[attachment], - msg_type='html') + EmailSender().SendEmail(email_to, + subject, + text_report, + attachments=[attachment], + msg_type='html') def _StoreResults(self, experiment): if self._terminated: @@ -300,9 +300,10 @@ class ExperimentRunner(object): self.l.LogOutput('Storing results of each benchmark run.') for benchmark_run in experiment.benchmark_runs: if benchmark_run.result: - benchmark_run_name = ''.join( - ch for ch in benchmark_run.name if ch.isalnum()) - benchmark_run_path = os.path.join(results_directory, benchmark_run_name) + benchmark_run_name = ''.join(ch for ch in benchmark_run.name + if ch.isalnum()) + benchmark_run_path = os.path.join(results_directory, + benchmark_run_name) if experiment.compress_results: benchmark_run.result.CompressResultsTo(benchmark_run_path) else: @@ -313,15 +314,16 @@ class ExperimentRunner(object): results_table_path = os.path.join(results_directory, 'results.html') report = HTMLResultsReport.FromExperiment(experiment).GetReport() if self.json_report: - json_report = JSONResultsReport.FromExperiment( - experiment, json_args={'indent': 2}) + json_report = JSONResultsReport.FromExperiment(experiment, + json_args={'indent': 2}) _WriteJSONReportToFile(experiment, results_directory, json_report) FileUtils().WriteFile(results_table_path, report) self.l.LogOutput('Storing email message body in %s.' % results_directory) msg_file_path = os.path.join(results_directory, 'msg_body.html') - text_report = TextResultsReport.FromExperiment(experiment, True).GetReport() + text_report = TextResultsReport.FromExperiment(experiment, + True).GetReport() text_report += ('\nResults are stored in %s.\n' % experiment.results_directory) msg_body = "<pre style='font-size: 13px'>%s</pre>" % text_report diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py index 87e30ecc..5525858c 100644 --- a/crosperf/results_cache.py +++ b/crosperf/results_cache.py @@ -27,7 +27,7 @@ import results_report import test_flag SCRATCH_DIR = os.path.expanduser('~/cros_scratch') -RESULTS_FILE = 'results.txt' +RESULTS_FILE = 'results.pickle' MACHINE_FILE = 'machine.txt' AUTOTEST_TARBALL = 'autotest.tbz2' RESULTS_TARBALL = 'results.tbz2' @@ -197,9 +197,9 @@ class Result(object): keyvals_dict[key] = result_dict['value'] elif 'values' in result_dict: values = result_dict['values'] - if ('type' in result_dict and - result_dict['type'] == 'list_of_scalar_values' and values and - values != 'null'): + if ('type' in result_dict + and result_dict['type'] == 'list_of_scalar_values' and values + and values != 'null'): keyvals_dict[key] = sum(values) / float(len(values)) else: keyvals_dict[key] = values @@ -245,13 +245,14 @@ class Result(object): results_in_chroot = os.path.join(self.chromeos_root, 'chroot', 'tmp') if not self.temp_dir: self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot) - command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir) + command = f'cp -r {self.results_dir}/* {self.temp_dir}' self.ce.RunCommand(command, print_to_console=False) command = ('./generate_test_report --no-color --csv %s' % (os.path.join('/tmp', os.path.basename(self.temp_dir)))) - _, out, _ = self.ce.ChrootRunCommandWOutput( - self.chromeos_root, command, print_to_console=False) + _, out, _ = self.ce.ChrootRunCommandWOutput(self.chromeos_root, + command, + print_to_console=False) keyvals_dict = {} tmp_dir_in_chroot = misc.GetInsideChrootPath(self.chromeos_root, self.temp_dir) @@ -322,8 +323,8 @@ class Result(object): idle_functions = { '[kernel.kallsyms]': - ('intel_idle', 'arch_cpu_idle', 'intel_idle', 'cpu_startup_entry', - 'default_idle', 'cpu_idle_loop', 'do_idle'), + ('intel_idle', 'arch_cpu_idle', 'intel_idle', 'cpu_startup_entry', + 'default_idle', 'cpu_idle_loop', 'do_idle'), } idle_samples = 0 @@ -390,8 +391,8 @@ class Result(object): result = ( self.FindFilesInResultsDir('-name histograms.json').splitlines()) else: - result = ( - self.FindFilesInResultsDir('-name results-chart.json').splitlines()) + result = (self.FindFilesInResultsDir( + '-name results-chart.json').splitlines()) return result def GetTurbostatFile(self): @@ -449,8 +450,8 @@ class Result(object): if debug_path: symfs = '--symfs ' + debug_path - vmlinux = '--vmlinux ' + os.path.join(debug_path, 'usr', 'lib', 'debug', - 'boot', 'vmlinux') + vmlinux = '--vmlinux ' + os.path.join(debug_path, 'usr', 'lib', + 'debug', 'boot', 'vmlinux') kallsyms = '' print('** WARNING **: --kallsyms option not applied, no System.map-* ' 'for downloaded image.') @@ -546,9 +547,9 @@ class Result(object): values = value_dict['values'] if not values: continue - if ('type' in value_dict and - value_dict['type'] == 'list_of_scalar_values' and - values != 'null'): + if ('type' in value_dict + and value_dict['type'] == 'list_of_scalar_values' + and values != 'null'): result = sum(values) / float(len(values)) else: result = values @@ -746,8 +747,9 @@ class Result(object): # order. heapq.heappush(cmd_top5_cpu_use[cmd_with_pid], round(cpu_use, 1)) - for consumer, usage in sorted( - cmd_total_cpu_use.items(), key=lambda x: x[1], reverse=True): + for consumer, usage in sorted(cmd_total_cpu_use.items(), + key=lambda x: x[1], + reverse=True): # Iterate through commands by descending order of total CPU usage. topcmd = { 'cmd': consumer, @@ -913,7 +915,8 @@ class Result(object): self.chromeos_root, path_str) if status: # Error of reading a perf.data profile is fatal. - raise PerfDataReadError(f'Failed to read perf.data profile: {path_str}') + raise PerfDataReadError( + f'Failed to read perf.data profile: {path_str}') # Pattern to search a line with "perf record" command line: # # cmdline : /usr/bin/perf record -e instructions -p 123" @@ -938,7 +941,8 @@ class Result(object): break else: # cmdline wasn't found in the header. It's a fatal error. - raise PerfDataReadError(f'Perf command line is not found in {path_str}') + raise PerfDataReadError( + f'Perf command line is not found in {path_str}') return pids def VerifyPerfDataPID(self): @@ -976,11 +980,11 @@ class Result(object): # Note that this function doesn't know anything about whether there is a # cache hit or miss. It should process results agnostic of the cache hit # state. - if (self.results_file and self.suite == 'telemetry_Crosperf' and - 'histograms.json' in self.results_file[0]): + if (self.results_file and self.suite == 'telemetry_Crosperf' + and 'histograms.json' in self.results_file[0]): self.keyvals = self.ProcessHistogramsResults() - elif (self.results_file and self.suite != 'telemetry_Crosperf' and - 'results-chart.json' in self.results_file[0]): + elif (self.results_file and self.suite != 'telemetry_Crosperf' + and 'results-chart.json' in self.results_file[0]): self.keyvals = self.ProcessChartResults() else: if not use_cache: @@ -1134,15 +1138,16 @@ class Result(object): f.write(machine_manager.machine_checksum_string[self.label.name]) if os.path.exists(cache_dir): - command = 'rm -rf {0}'.format(cache_dir) + command = f'rm -rf {cache_dir}' self.ce.RunCommand(command) - command = 'mkdir -p {0} && '.format(os.path.dirname(cache_dir)) - command += 'chmod g+x {0} && '.format(temp_dir) - command += 'mv {0} {1}'.format(temp_dir, cache_dir) + parent_dir = os.path.dirname(cache_dir) + command = f'mkdir -p {parent_dir} && ' + command += f'chmod g+x {temp_dir} && ' + command += f'mv {temp_dir} {cache_dir}' ret = self.ce.RunCommand(command) if ret: - command = 'rm -rf {0}'.format(temp_dir) + command = f'rm -rf {temp_dir}' self.ce.RunCommand(command) raise RuntimeError('Could not move dir %s to dir %s' % (temp_dir, cache_dir)) @@ -1241,8 +1246,8 @@ class TelemetryResult(Result): self.err = pickle.load(f) self.retval = pickle.load(f) - self.chrome_version = ( - super(TelemetryResult, self).GetChromeVersionFromCache(cache_dir)) + self.chrome_version = (super(TelemetryResult, + self).GetChromeVersionFromCache(cache_dir)) self.ProcessResults() @@ -1304,10 +1309,10 @@ class ResultsCache(object): self.run_local = None self.cwp_dso = None - def Init(self, chromeos_image, chromeos_root, test_name, iteration, test_args, - profiler_args, machine_manager, machine, board, cache_conditions, - logger_to_use, log_level, label, share_cache, suite, - show_all_results, run_local, cwp_dso): + def Init(self, chromeos_image, chromeos_root, test_name, iteration, + test_args, profiler_args, machine_manager, machine, board, + cache_conditions, logger_to_use, log_level, label, share_cache, + suite, show_all_results, run_local, cwp_dso): self.chromeos_image = chromeos_image self.chromeos_root = chromeos_root self.test_name = test_name @@ -1319,8 +1324,8 @@ class ResultsCache(object): self.machine_manager = machine_manager self.machine = machine self._logger = logger_to_use - self.ce = command_executer.GetCommandExecuter( - self._logger, log_level=log_level) + self.ce = command_executer.GetCommandExecuter(self._logger, + log_level=log_level) self.label = label self.share_cache = share_cache self.suite = suite @@ -1406,15 +1411,16 @@ class ResultsCache(object): temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args, self.run_local) - test_args_checksum = hashlib.md5(temp_test_args.encode('utf-8')).hexdigest() + test_args_checksum = hashlib.md5( + temp_test_args.encode('utf-8')).hexdigest() return (image_path_checksum, self.test_name, str(self.iteration), - test_args_checksum, checksum, machine_checksum, machine_id_checksum, - str(self.CACHE_VERSION)) + test_args_checksum, checksum, machine_checksum, + machine_id_checksum, str(self.CACHE_VERSION)) def ReadResult(self): if CacheConditions.FALSE in self.cache_conditions: cache_dir = self.GetCacheDirForWrite() - command = 'rm -rf %s' % (cache_dir,) + command = 'rm -rf %s' % (cache_dir, ) self.ce.RunCommand(command) return None cache_dir = self.GetCacheDirForRead() @@ -1427,14 +1433,15 @@ class ResultsCache(object): if self.log_level == 'verbose': self._logger.LogOutput('Trying to read from cache dir: %s' % cache_dir) - result = Result.CreateFromCacheHit(self._logger, self.log_level, self.label, - self.machine, cache_dir, self.test_name, - self.suite, self.cwp_dso) + result = Result.CreateFromCacheHit(self._logger, self.log_level, + self.label, self.machine, cache_dir, + self.test_name, self.suite, + self.cwp_dso) if not result: return None - if (result.retval == 0 or - CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): + if (result.retval == 0 + or CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): return result return None diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py index df3a35e9..d6953eed 100755 --- a/crosperf/results_cache_unittest.py +++ b/crosperf/results_cache_unittest.py @@ -11,6 +11,7 @@ from __future__ import print_function import io import os +import pickle import shutil import tempfile import unittest @@ -31,6 +32,8 @@ from cros_utils import command_executer from cros_utils import logger from cros_utils import misc +# The following hardcoded string has blocked words replaced, and thus +# is not representative of a true crosperf output. # pylint: disable=line-too-long OUTPUT = """CMD (True): ./test_that.sh\ --remote=172.17.128.241 --board=lumpy LibCBench @@ -42,13 +45,13 @@ INFO : Running the following control files 1 times: INFO : * 'client/site_tests/platform_LibCBench/control' INFO : Running client test client/site_tests/platform_LibCBench/control -./server/autoserv -m 172.17.128.241 --ssh-port 22 -c client/site_tests/platform_LibCBench/control -r /tmp/test_that.PO1234567/platform_LibCBench --test-retry=0 --args +./server/autoserv -m 172.17.128.241 --ssh-port 22 -c client/site_tests/platform_LibCBench/control -r /tmp/test_that.PO1234567/platform_LibCBench --test-retry=0 --args ERROR:root:import statsd failed, no stats will be reported. 14:20:22 INFO | Results placed in /tmp/test_that.PO1234567/platform_LibCBench 14:20:22 INFO | Processing control file -14:20:23 INFO | Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_VIIP67ssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' +14:20:23 INFO | Starting main ssh connection '/usr/bin/ssh -a -x -N -o ControlMain=yes -o ControlPath=/tmp/_autotmp_VIIP67ssh-main/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' 14:20:23 ERROR| [stderr] Warning: Permanently added '172.17.128.241' (RSA) to the list of known hosts. -14:20:23 INFO | INFO ---- ---- kernel=3.8.11 localtime=May 22 14:20:23 timestamp=1369257623 +14:20:23 INFO | INFO\t----\t----\tkernel=3.8.11\tlocaltime=May 22 14:20:23\ttimestamp=1369257623 14:20:23 INFO | Installing autotest on 172.17.128.241 14:20:23 INFO | Using installation dir /usr/local/autotest 14:20:23 WARNI| No job_repo_url for <remote host: 172.17.128.241> @@ -59,11 +62,11 @@ ERROR:root:import statsd failed, no stats will be reported. 14:20:24 INFO | Entered autotestd_monitor. 14:20:24 INFO | Finished launching tail subprocesses. 14:20:24 INFO | Finished waiting on autotestd to start. -14:20:26 INFO | START ---- ---- timestamp=1369257625 localtime=May 22 14:20:25 -14:20:26 INFO | START platform_LibCBench platform_LibCBench timestamp=1369257625 localtime=May 22 14:20:25 -14:20:30 INFO | GOOD platform_LibCBench platform_LibCBench timestamp=1369257630 localtime=May 22 14:20:30 completed successfully -14:20:30 INFO | END GOOD platform_LibCBench platform_LibCBench timestamp=1369257630 localtime=May 22 14:20:30 -14:20:31 INFO | END GOOD ---- ---- timestamp=1369257630 localtime=May 22 14:20:30 +14:20:26 INFO | START\t----\t----\ttimestamp=1369257625\tlocaltime=May 22 14:20:25 +14:20:26 INFO | \tSTART\tplatform_LibCBench\tplatform_LibCBench\ttimestamp=1369257625\tlocaltime=May 22 14:20:25 +14:20:30 INFO | \t\tGOOD\tplatform_LibCBench\tplatform_LibCBench\ttimestamp=1369257630\tlocaltime=May 22 14:20:30\tcompleted successfully +14:20:30 INFO | \tEND GOOD\tplatform_LibCBench\tplatform_LibCBench\ttimestamp=1369257630\tlocaltime=May 22 14:20:30 +14:20:31 INFO | END GOOD\t----\t----\ttimestamp=1369257630\tlocaltime=May 22 14:20:30 14:20:31 INFO | Got lock of exit_code_file. 14:20:31 INFO | Released lock of exit_code_file and closed it. OUTPUT: ============================== @@ -72,14 +75,14 @@ Done: 0% [ ] OUTPUT: Thread Status: RUNNING: 1 ('ttt: LibCBench (1)' 0:01:21) Machine Status: -Machine Thread Lock Status Checksum +Machine Thread Lock Status Checksum 172.17.128.241 ttt: LibCBench (1) True RUNNING 3ba9f2ecbb222f20887daea5583d86ba OUTPUT: ============================== 14:20:33 INFO | Killing child processes. 14:20:33 INFO | Client complete 14:20:33 INFO | Finished processing control file -14:20:33 INFO | Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_aVJUgmssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' +14:20:33 INFO | Starting main ssh connection '/usr/bin/ssh -a -x -N -o ControlMain=yes -o ControlPath=/tmp/_autotmp_aVJUgmssh-main/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' 14:20:33 ERROR| [stderr] Warning: Permanently added '172.17.128.241' (RSA) to the list of known hosts. INFO : Test results: @@ -116,7 +119,7 @@ platform_LibCBench/platform_LibCBench b_utf8_onebyone__0_ ------------------------------------------------------------------- Total PASS: 2/2 (100%) -INFO : Elapsed time: 0m16s +INFO : Elapsed time: 0m16s """ error = """ @@ -177,7 +180,7 @@ PERF_DATA_HEADER = """ # total memory : 5911496 kB # cmdline : /usr/bin/perf record -e instructions -p {pid} # event : name = instructions, , id = ( 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193 ), type = 8, size = 112 -# event : name = dummy:u, , id = ( 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204 ), type = 1, size = 112, config = 0x9 +# event : name = placeholder:u, , id = ( 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204 ), type = 1, size = 112, config = 0x9 # CPU_TOPOLOGY info available, use -I to display # pmu mappings: software = 1, uprobe = 6, cs_etm = 8, breakpoint = 5, tracepoint = 2, armv8_pmuv3 = 7 # contains AUX area data (e.g. instruction trace) @@ -439,7 +442,6 @@ HISTOGRAMSET = (""" class MockResult(Result): """Mock result class.""" - def __init__(self, mylogger, label, logging_level, machine): super(MockResult, self).__init__(mylogger, label, logging_level, machine) @@ -455,7 +457,6 @@ class MockResult(Result): class ResultTest(unittest.TestCase): """Result test class.""" - def __init__(self, *args, **kwargs): super(ResultTest, self).__init__(*args, **kwargs) self.callFakeProcessResults = False @@ -484,8 +485,8 @@ class ResultTest(unittest.TestCase): def testCreateFromRun(self): result = MockResult.CreateFromRun(logger.GetLogger(), 'average', - self.mock_label, 'remote1', OUTPUT, error, - 0, True) + self.mock_label, 'remote1', OUTPUT, + error, 0, True) self.assertEqual(result.keyvals, keyvals) self.assertEqual(result.chroot_results_dir, '/tmp/test_that.PO1234567/platform_LibCBench') @@ -533,7 +534,8 @@ class ResultTest(unittest.TestCase): mock_runcmd.call_args_list[1]) self.assertEqual(mock_runcmd.call_args_list[0], mock_runcmd.call_args_list[2]) - self.assertEqual(mock_runcmd.call_args_list[0][0], ('mkdir -p /tmp/test',)) + self.assertEqual(mock_runcmd.call_args_list[0][0], + ('mkdir -p /tmp/test', )) # test 3. CopyFiles returns 1 (fails). mock_copyfiles.return_value = 1 @@ -715,7 +717,8 @@ class ResultTest(unittest.TestCase): mock_mkdtemp.return_value = TMP_DIR1 mock_chrootruncmd.return_value = [ - '', ('%s,PASS\n%s/telemetry_Crosperf,PASS\n') % (TMP_DIR1, TMP_DIR1), '' + '', ('%s,PASS\n%s/telemetry_Crosperf,PASS\n') % (TMP_DIR1, TMP_DIR1), + '' ] mock_getpath.return_value = TMP_DIR1 self.result.ce.ChrootRunCommandWOutput = mock_chrootruncmd @@ -730,7 +733,7 @@ class ResultTest(unittest.TestCase): self.assertEqual(self.kv_dict, {'': 'PASS', 'telemetry_Crosperf': 'PASS'}) self.assertEqual(mock_runcmd.call_count, 1) self.assertEqual(mock_runcmd.call_args_list[0][0], - ('cp -r /tmp/test_that_resultsNmq/* %s' % TMP_DIR1,)) + ('cp -r /tmp/test_that_resultsNmq/* %s' % TMP_DIR1, )) self.assertEqual(mock_chrootruncmd.call_count, 1) self.assertEqual( mock_chrootruncmd.call_args_list[0][0], @@ -770,7 +773,8 @@ class ResultTest(unittest.TestCase): @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommandWOutput') @mock.patch.object(os.path, 'exists') - def test_get_samples(self, mock_exists, mock_get_total_samples, mock_getpath): + def test_get_samples(self, mock_exists, mock_get_total_samples, + mock_getpath): self.result.perf_data_files = ['/tmp/results/perf.data'] self.result.board = 'samus' mock_getpath.return_value = '/usr/chromeos/chroot/tmp/results/perf.data' @@ -811,7 +815,7 @@ class ResultTest(unittest.TestCase): res = self.result.FindFilesInResultsDir('-name perf.data') self.assertEqual(mock_runcmd.call_count, 1) self.assertEqual(mock_runcmd.call_args_list[0][0], - ('find /tmp/test_results -name perf.data',)) + ('find /tmp/test_results -name perf.data', )) self.assertEqual(res, '/tmp/test_results/perf.data') mock_runcmd.reset_mock() @@ -827,7 +831,8 @@ class ResultTest(unittest.TestCase): self.result.FindFilesInResultsDir = mock_findfiles res = self.result.GetPerfDataFiles() self.assertEqual(res, ['line1', 'line1']) - self.assertEqual(mock_findfiles.call_args_list[0][0], ('-name perf.data',)) + self.assertEqual(mock_findfiles.call_args_list[0][0], + ('-name perf.data', )) def test_get_perf_report_files(self): self.args = None @@ -958,16 +963,18 @@ class ResultTest(unittest.TestCase): """Verify perf PID which is present in TOP_DATA.""" self.result.top_cmds = TOP_DATA # pid is present in TOP_DATA. - with mock.patch.object( - Result, 'ReadPidFromPerfData', return_value=['5713']): + with mock.patch.object(Result, + 'ReadPidFromPerfData', + return_value=['5713']): self.result.VerifyPerfDataPID() def test_verify_perf_data_pid_fail(self): """Test perf PID missing in top raises the error.""" self.result.top_cmds = TOP_DATA # pid is not in the list of top processes. - with mock.patch.object( - Result, 'ReadPidFromPerfData', return_value=['9999']): + with mock.patch.object(Result, + 'ReadPidFromPerfData', + return_value=['9999']): with self.assertRaises(PidVerificationError): self.result.VerifyPerfDataPID() @@ -976,7 +983,9 @@ class ResultTest(unittest.TestCase): def test_read_pid_from_perf_data_ok(self, mock_runcmd): """Test perf header parser, normal flow.""" self.result.ce.ChrootRunCommandWOutput = mock_runcmd - self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + self.result.perf_data_files = [ + '/tmp/chromeos/chroot/tmp/results/perf.data' + ] exp_pid = '12345' mock_runcmd.return_value = (0, PERF_DATA_HEADER.format(pid=exp_pid), '') pids = self.result.ReadPidFromPerfData() @@ -1007,7 +1016,9 @@ class ResultTest(unittest.TestCase): def test_read_pid_from_perf_data_no_pid(self, mock_runcmd): """Test perf.data without PID.""" self.result.ce.ChrootRunCommandWOutput = mock_runcmd - self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + self.result.perf_data_files = [ + '/tmp/chromeos/chroot/tmp/results/perf.data' + ] cmd_line = '# cmdline : /usr/bin/perf record -e instructions' mock_runcmd.return_value = (0, cmd_line, '') pids = self.result.ReadPidFromPerfData() @@ -1019,7 +1030,9 @@ class ResultTest(unittest.TestCase): def test_read_pid_from_perf_data_system_wide(self, mock_runcmd): """Test reading from system-wide profile with PID.""" self.result.ce.ChrootRunCommandWOutput = mock_runcmd - self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + self.result.perf_data_files = [ + '/tmp/chromeos/chroot/tmp/results/perf.data' + ] # There is '-p <pid>' in command line but it's still system-wide: '-a'. cmd_line = '# cmdline : /usr/bin/perf record -e instructions -a -p 1234' mock_runcmd.return_value = (0, cmd_line, '') @@ -1032,7 +1045,9 @@ class ResultTest(unittest.TestCase): def test_read_pid_from_perf_data_read_fail(self, mock_runcmd): """Failure to read perf.data raises the error.""" self.result.ce.ChrootRunCommandWOutput = mock_runcmd - self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + self.result.perf_data_files = [ + '/tmp/chromeos/chroot/tmp/results/perf.data' + ] # Error status of the profile read. mock_runcmd.return_value = (1, '', '') with self.assertRaises(PerfDataReadError): @@ -1043,7 +1058,9 @@ class ResultTest(unittest.TestCase): def test_read_pid_from_perf_data_fail(self, mock_runcmd): """Failure to find cmdline in perf.data header raises the error.""" self.result.ce.ChrootRunCommandWOutput = mock_runcmd - self.result.perf_data_files = ['/tmp/chromeos/chroot/tmp/results/perf.data'] + self.result.perf_data_files = [ + '/tmp/chromeos/chroot/tmp/results/perf.data' + ] # Empty output. mock_runcmd.return_value = (0, '', '') with self.assertRaises(PerfDataReadError): @@ -1273,7 +1290,6 @@ class ResultTest(unittest.TestCase): @mock.patch.object(misc, 'GetOutsideChrootPath') def test_populate_from_run(self, mock_getpath): - def FakeGetResultsDir(): self.callGetResultsDir = True return '/tmp/results_dir' @@ -1361,7 +1377,6 @@ class ResultTest(unittest.TestCase): return {'Total': 10} def test_process_results(self): - def FakeGatherPerfResults(): self.callGatherPerfResults = True @@ -1407,16 +1422,17 @@ class ResultTest(unittest.TestCase): self.result.ProcessResults() shutil.rmtree(os.path.dirname(self.result.results_file[0])) # Verify the summary for the story is correct - self.assertEqual(self.result.keyvals['timeToFirstContentfulPaint__typical'], - [880.000, u'ms_smallerIsBetter']) + self.assertEqual( + self.result.keyvals['timeToFirstContentfulPaint__typical'], + [880.000, u'ms_smallerIsBetter']) # Veirfy the summary for a certain stroy tag is correct self.assertEqual( - self.result - .keyvals['timeToFirstContentfulPaint__cache_temperature:cold'], + self.result. + keyvals['timeToFirstContentfulPaint__cache_temperature:cold'], [1000.000, u'ms_smallerIsBetter']) self.assertEqual( - self.result - .keyvals['timeToFirstContentfulPaint__cache_temperature:warm'], + self.result. + keyvals['timeToFirstContentfulPaint__cache_temperature:warm'], [800.000, u'ms_smallerIsBetter']) @mock.patch.object(Result, 'ProcessCpustatsResults') @@ -1572,7 +1588,8 @@ class ResultTest(unittest.TestCase): u'telemetry_page_measurement_results__num_errored': [0, u'count'], u'string-fasta__string-fasta': [23.2, u'ms'], u'crypto-sha1__crypto-sha1': [11.6, u'ms'], - u'bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte': [3.2, u'ms'], + u'bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte': + [3.2, u'ms'], u'access-nsieve__access-nsieve': [7.9, u'ms'], u'bitops-nsieve-bits__bitops-nsieve-bits': [9.4, u'ms'], u'string-validate-input__string-validate-input': [19.3, u'ms'], @@ -1610,7 +1627,8 @@ class ResultTest(unittest.TestCase): u'telemetry_page_measurement_results__num_errored': [0, u'count'], u'string-fasta__string-fasta': [23.2, u'ms'], u'crypto-sha1__crypto-sha1': [11.6, u'ms'], - u'bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte': [3.2, u'ms'], + u'bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte': + [3.2, u'ms'], u'access-nsieve__access-nsieve': [7.9, u'ms'], u'bitops-nsieve-bits__bitops-nsieve-bits': [9.4, u'ms'], u'string-validate-input__string-validate-input': [19.3, u'ms'], @@ -1657,8 +1675,9 @@ class ResultTest(unittest.TestCase): self.assertEqual(mock_getroot.call_count, 1) self.assertEqual(mock_runcmd.call_count, 2) self.assertEqual(mock_runcmd.call_args_list[0][0], - ('rm -rf test_results_dir',)) - self.assertEqual(mock_runcmd.call_args_list[1][0], ('rm -rf testtemp_dir',)) + ('rm -rf test_results_dir', )) + self.assertEqual(mock_runcmd.call_args_list[1][0], + ('rm -rf testtemp_dir', )) # Test 2. Same, except ath results_dir name does not contain # 'test_that_results_' @@ -1672,8 +1691,9 @@ class ResultTest(unittest.TestCase): self.assertEqual(mock_getroot.call_count, 1) self.assertEqual(mock_runcmd.call_count, 2) self.assertEqual(mock_runcmd.call_args_list[0][0], - ('rm -rf /tmp/tmp_AbcXyz',)) - self.assertEqual(mock_runcmd.call_args_list[1][0], ('rm -rf testtemp_dir',)) + ('rm -rf /tmp/tmp_AbcXyz', )) + self.assertEqual(mock_runcmd.call_args_list[1][0], + ('rm -rf testtemp_dir', )) # Test 3. mock_getroot returns nothing; 'rm_chroot_tmp' is False. mock_getroot.reset_mock() @@ -1681,7 +1701,8 @@ class ResultTest(unittest.TestCase): self.result.CleanUp(False) self.assertEqual(mock_getroot.call_count, 0) self.assertEqual(mock_runcmd.call_count, 1) - self.assertEqual(mock_runcmd.call_args_list[0][0], ('rm -rf testtemp_dir',)) + self.assertEqual(mock_runcmd.call_args_list[0][0], + ('rm -rf testtemp_dir', )) # Test 4. 'rm_chroot_tmp' is True, but result_dir & temp_dir are None. mock_getroot.reset_mock() @@ -1695,7 +1716,6 @@ class ResultTest(unittest.TestCase): @mock.patch.object(misc, 'GetInsideChrootPath') @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand') def test_store_to_cache_dir(self, mock_chrootruncmd, mock_getpath): - def FakeMkdtemp(directory=''): if directory: pass @@ -1730,7 +1750,7 @@ class ResultTest(unittest.TestCase): base_dir = os.path.join(os.getcwd(), 'test_cache/compare_output') self.assertTrue(os.path.exists(os.path.join(test_dir, 'autotest.tbz2'))) self.assertTrue(os.path.exists(os.path.join(test_dir, 'machine.txt'))) - self.assertTrue(os.path.exists(os.path.join(test_dir, 'results.txt'))) + self.assertTrue(os.path.exists(os.path.join(test_dir, 'results.pickle'))) f1 = os.path.join(test_dir, 'machine.txt') f2 = os.path.join(base_dir, 'machine.txt') @@ -1738,11 +1758,13 @@ class ResultTest(unittest.TestCase): [_, out, _] = self.result.ce.RunCommandWOutput(cmd) self.assertEqual(len(out), 0) - f1 = os.path.join(test_dir, 'results.txt') - f2 = os.path.join(base_dir, 'results.txt') - cmd = 'diff %s %s' % (f1, f2) - [_, out, _] = self.result.ce.RunCommandWOutput(cmd) - self.assertEqual(len(out), 0) + f1 = os.path.join(test_dir, 'results.pickle') + f2 = os.path.join(base_dir, 'results.pickle') + with open(f1, 'rb') as f: + f1_obj = pickle.load(f) + with open(f2, 'rb') as f: + f2_obj = pickle.load(f) + self.assertEqual(f1_obj, f2_obj) # Clean up after test. tempfile.mkdtemp = save_real_mkdtemp @@ -1753,87 +1775,87 @@ class ResultTest(unittest.TestCase): TELEMETRY_RESULT_KEYVALS = { 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'math-cordic (ms)': - '11.4', + '11.4', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'access-nbody (ms)': - '6.9', + '6.9', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'access-fannkuch (ms)': - '26.3', + '26.3', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'math-spectral-norm (ms)': - '6.3', + '6.3', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'bitops-nsieve-bits (ms)': - '9.3', + '9.3', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'math-partial-sums (ms)': - '32.8', + '32.8', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'regexp-dna (ms)': - '16.1', + '16.1', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' '3d-cube (ms)': - '42.7', + '42.7', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'crypto-md5 (ms)': - '10.8', + '10.8', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'crypto-sha1 (ms)': - '12.4', + '12.4', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'string-tagcloud (ms)': - '47.2', + '47.2', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'string-fasta (ms)': - '36.3', + '36.3', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'access-binary-trees (ms)': - '7.3', + '7.3', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'date-format-xparb (ms)': - '138.1', + '138.1', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'crypto-aes (ms)': - '19.2', + '19.2', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'Total (ms)': - '656.5', + '656.5', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'string-base64 (ms)': - '17.5', + '17.5', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'string-validate-input (ms)': - '24.8', + '24.8', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' '3d-raytrace (ms)': - '28.7', + '28.7', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'controlflow-recursive (ms)': - '5.3', + '5.3', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'bitops-bits-in-byte (ms)': - '9.8', + '9.8', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' '3d-morph (ms)': - '50.2', + '50.2', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'bitops-bitwise-and (ms)': - '8.8', + '8.8', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'access-nsieve (ms)': - '8.6', + '8.6', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'date-format-tofte (ms)': - '31.2', + '31.2', 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'bitops-3bit-bits-in-byte (ms)': - '3.5', + '3.5', 'retval': - 0, + 0, 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 'string-unpack-code (ms)': - '45.0' + '45.0' } PURE_TELEMETRY_OUTPUT = """ @@ -1843,7 +1865,6 @@ page_name,3d-cube (ms),3d-morph (ms),3d-raytrace (ms),Total (ms),access-binary-t class TelemetryResultTest(unittest.TestCase): """Telemetry result test.""" - def __init__(self, *args, **kwargs): super(TelemetryResultTest, self).__init__(*args, **kwargs) self.callFakeProcessResults = False @@ -1854,12 +1875,10 @@ class TelemetryResultTest(unittest.TestCase): 'autotest_dir', 'debug_dir', '/tmp', 'lumpy', 'remote', 'image_args', 'cache_dir', 'average', 'gcc', False, None) - self.mock_machine = machine_manager.MockCrosMachine('falco.cros', - '/tmp/chromeos', - 'average') + self.mock_machine = machine_manager.MockCrosMachine( + 'falco.cros', '/tmp/chromeos', 'average') def test_populate_from_run(self): - def FakeProcessResults(): self.callFakeProcessResults = True @@ -1890,7 +1909,6 @@ class TelemetryResultTest(unittest.TestCase): class ResultsCacheTest(unittest.TestCase): """Resultcache test class.""" - def __init__(self, *args, **kwargs): super(ResultsCacheTest, self).__init__(*args, **kwargs) self.fakeCacheReturnResult = None @@ -1932,7 +1950,6 @@ class ResultsCacheTest(unittest.TestCase): @mock.patch.object(image_checksummer.ImageChecksummer, 'Checksum') def test_get_cache_dir_for_write(self, mock_checksum): - def FakeGetMachines(label): if label: pass @@ -2041,7 +2058,8 @@ class ResultsCacheTest(unittest.TestCase): # Test 5. Generating cache name for writing, with local image type, and # specifying that the image path must match the cached image path. self.results_cache.label.image_type = 'local' - self.results_cache.cache_conditions.append(CacheConditions.IMAGE_PATH_MATCH) + self.results_cache.cache_conditions.append( + CacheConditions.IMAGE_PATH_MATCH) key_list = self.results_cache.GetCacheKeyList(False) self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py index 34326b68..78834c63 100644 --- a/crosperf/settings_factory.py +++ b/crosperf/settings_factory.py @@ -22,14 +22,13 @@ class BenchmarkSettings(Settings): def __init__(self, name): super(BenchmarkSettings, self).__init__(name, 'benchmark') self.AddField( - TextField( - 'test_name', - description='The name of the test to run. ' - 'Defaults to the name of the benchmark.')) + TextField('test_name', + description='The name of the test to run. ' + 'Defaults to the name of the benchmark.')) self.AddField( - TextField( - 'test_args', description='Arguments to be passed to the ' - 'test.')) + TextField('test_args', + description='Arguments to be passed to the ' + 'test.')) self.AddField( IntegerField( 'iterations', @@ -39,24 +38,21 @@ class BenchmarkSettings(Settings): 'If not set, will run each benchmark test the optimum number of ' 'times to get a stable result.')) self.AddField( - TextField( - 'suite', - default='test_that', - description='The type of the benchmark.')) + TextField('suite', + default='test_that', + description='The type of the benchmark.')) self.AddField( - IntegerField( - 'retries', - default=0, - description='Number of times to retry a ' - 'benchmark run.')) + IntegerField('retries', + default=0, + description='Number of times to retry a ' + 'benchmark run.')) self.AddField( - BooleanField( - 'run_local', - description='Run benchmark harness on the DUT. ' - 'Currently only compatible with the suite: ' - 'telemetry_Crosperf.', - required=False, - default=True)) + BooleanField('run_local', + description='Run benchmark harness on the DUT. ' + 'Currently only compatible with the suite: ' + 'telemetry_Crosperf.', + required=False, + default=True)) self.AddField( FloatField( 'weight', @@ -70,12 +66,11 @@ class LabelSettings(Settings): def __init__(self, name): super(LabelSettings, self).__init__(name, 'label') self.AddField( - TextField( - 'chromeos_image', - required=False, - description='The path to the image to run tests ' - 'on, for local/custom-built images. See the ' - "'build' option for official or trybot images.")) + TextField('chromeos_image', + required=False, + description='The path to the image to run tests ' + 'on, for local/custom-built images. See the ' + "'build' option for official or trybot images.")) self.AddField( TextField( 'autotest_path', @@ -90,53 +85,46 @@ class LabelSettings(Settings): description='Debug info directory relative to chroot which has ' 'symbols and vmlinux that can be used by perf tool.')) self.AddField( - TextField( - 'chromeos_root', - description='The path to a chromeos checkout which ' - 'contains a src/scripts directory. Defaults to ' - 'the chromeos checkout which contains the ' - 'chromeos_image.')) - self.AddField( - ListField( - 'remote', - description='A comma-separated list of IPs of chromeos' - 'devices to run experiments on.')) - self.AddField( - TextField( - 'image_args', - required=False, - default='', - description='Extra arguments to pass to ' - 'image_chromeos.py.')) - self.AddField( - TextField( - 'cache_dir', - default='', - description='The cache dir for this image.')) - self.AddField( - TextField( - 'compiler', - default='gcc', - description='The compiler used to build the ' - 'ChromeOS image (gcc or llvm).')) - self.AddField( - TextField( - 'chrome_src', - description='The path to the source of chrome. ' - 'This is used to run telemetry benchmarks. ' - 'The default one is the src inside chroot.', - required=False, - default='')) - self.AddField( - TextField( - 'build', - description='The xbuddy specification for an ' - 'official or trybot image to use for tests. ' - "'/remote' is assumed, and the board is given " - "elsewhere, so omit the '/remote/<board>/' xbuddy " - 'prefix.', - required=False, - default='')) + TextField('chromeos_root', + description='The path to a chromeos checkout which ' + 'contains a src/scripts directory. Defaults to ' + 'the chromeos checkout which contains the ' + 'chromeos_image.')) + self.AddField( + ListField('remote', + description='A comma-separated list of IPs of chromeos' + 'devices to run experiments on.')) + self.AddField( + TextField('image_args', + required=False, + default='', + description='Extra arguments to pass to ' + 'image_chromeos.py.')) + self.AddField( + TextField('cache_dir', + default='', + description='The cache dir for this image.')) + self.AddField( + TextField('compiler', + default='gcc', + description='The compiler used to build the ' + 'ChromeOS image (gcc or llvm).')) + self.AddField( + TextField('chrome_src', + description='The path to the source of chrome. ' + 'This is used to run telemetry benchmarks. ' + 'The default one is the src inside chroot.', + required=False, + default='')) + self.AddField( + TextField('build', + description='The xbuddy specification for an ' + 'official or trybot image to use for tests. ' + "'/remote' is assumed, and the board is given " + "elsewhere, so omit the '/remote/<board>/' xbuddy " + 'prefix.', + required=False, + default='')) class GlobalSettings(Settings): @@ -145,67 +133,56 @@ class GlobalSettings(Settings): def __init__(self, name): super(GlobalSettings, self).__init__(name, 'global') self.AddField( - TextField( - 'name', - description='The name of the experiment. Just an ' - 'identifier.')) - self.AddField( - TextField( - 'board', - description='The target board for running ' - 'experiments on, e.g. x86-alex.')) - self.AddField( - BooleanField( - 'crosfleet', - description='Whether to run experiments via crosfleet.', - default=False)) - self.AddField( - ListField( - 'remote', - description='A comma-separated list of IPs of ' - 'chromeos devices to run experiments on.')) - self.AddField( - BooleanField( - 'rerun_if_failed', - description='Whether to re-run failed test runs ' - 'or not.', - default=False)) - self.AddField( - BooleanField( - 'rm_chroot_tmp', - default=False, - description='Whether to remove the test_that ' - 'result in the chroot.')) - self.AddField( - ListField( - 'email', - description='Space-separated list of email ' - 'addresses to send email to.')) - self.AddField( - BooleanField( - 'rerun', - description='Whether to ignore the cache and ' - 'for tests to be re-run.', - default=False)) - self.AddField( - BooleanField( - 'same_specs', - default=True, - description='Ensure cached runs are run on the ' - 'same kind of devices which are specified as a ' - 'remote.')) - self.AddField( - BooleanField( - 'same_machine', - default=False, - description='Ensure cached runs are run on the ' - 'same remote.')) - self.AddField( - BooleanField( - 'use_file_locks', - default=False, - description='DEPRECATED: Whether to use the file locks ' - 'or AFE server lock mechanism.')) + TextField('name', + description='The name of the experiment. Just an ' + 'identifier.')) + self.AddField( + TextField('board', + description='The target board for running ' + 'experiments on, e.g. x86-alex.')) + self.AddField( + BooleanField('crosfleet', + description='Whether to run experiments via crosfleet.', + default=False)) + self.AddField( + ListField('remote', + description='A comma-separated list of IPs of ' + 'chromeos devices to run experiments on.')) + self.AddField( + BooleanField('rerun_if_failed', + description='Whether to re-run failed test runs ' + 'or not.', + default=False)) + self.AddField( + BooleanField('rm_chroot_tmp', + default=False, + description='Whether to remove the test_that ' + 'result in the chroot.')) + self.AddField( + ListField('email', + description='Space-separated list of email ' + 'addresses to send email to.')) + self.AddField( + BooleanField('rerun', + description='Whether to ignore the cache and ' + 'for tests to be re-run.', + default=False)) + self.AddField( + BooleanField('same_specs', + default=True, + description='Ensure cached runs are run on the ' + 'same kind of devices which are specified as a ' + 'remote.')) + self.AddField( + BooleanField('same_machine', + default=False, + description='Ensure cached runs are run on the ' + 'same remote.')) + self.AddField( + BooleanField('use_file_locks', + default=False, + description='DEPRECATED: Whether to use the file locks ' + 'or AFE server lock mechanism.')) self.AddField( IntegerField( 'iterations', @@ -215,79 +192,68 @@ class GlobalSettings(Settings): 'If not set, will run each benchmark test the optimum number of ' 'times to get a stable result.')) self.AddField( - TextField( - 'chromeos_root', - description='The path to a chromeos checkout which ' - 'contains a src/scripts directory. Defaults to ' - 'the chromeos checkout which contains the ' - 'chromeos_image.')) - self.AddField( - TextField( - 'logging_level', - default='average', - description='The level of logging desired. ' - "Options are 'quiet', 'average', and 'verbose'.")) - self.AddField( - IntegerField( - 'acquire_timeout', - default=0, - description='Number of seconds to wait for ' - 'machine before exit if all the machines in ' - 'the experiment file are busy. Default is 0.')) - self.AddField( - TextField( - 'perf_args', - default='', - description='The optional profile command. It ' - 'enables perf commands to record perforamance ' - 'related counters. It must start with perf ' - 'command record or stat followed by arguments.')) - self.AddField( - BooleanField( - 'download_debug', - default=True, - description='Download compressed debug symbols alongwith ' - 'image. This can provide more info matching symbols for' - 'profiles, but takes larger space. By default, download' - 'it only when perf_args is specified.')) - self.AddField( - TextField( - 'cache_dir', - default='', - description='The abs path of cache dir. ' - 'Default is /home/$(whoami)/cros_scratch.')) - self.AddField( - BooleanField( - 'cache_only', - default=False, - description='Whether to use only cached ' - 'results (do not rerun failed tests).')) - self.AddField( - BooleanField( - 'no_email', - default=False, - description='Whether to disable the email to ' - 'user after crosperf finishes.')) - self.AddField( - BooleanField( - 'json_report', - default=False, - description='Whether to generate a json version ' - 'of the report, for archiving.')) - self.AddField( - BooleanField( - 'show_all_results', - default=False, - description='When running Telemetry tests, ' - 'whether to all the results, instead of just ' - 'the default (summary) results.')) - self.AddField( - TextField( - 'share_cache', - default='', - description='Path to alternate cache whose data ' - 'you want to use. It accepts multiple directories ' - 'separated by a ",".')) + TextField('chromeos_root', + description='The path to a chromeos checkout which ' + 'contains a src/scripts directory. Defaults to ' + 'the chromeos checkout which contains the ' + 'chromeos_image.')) + self.AddField( + TextField('logging_level', + default='average', + description='The level of logging desired. ' + "Options are 'quiet', 'average', and 'verbose'.")) + self.AddField( + IntegerField('acquire_timeout', + default=0, + description='Number of seconds to wait for ' + 'machine before exit if all the machines in ' + 'the experiment file are busy. Default is 0.')) + self.AddField( + TextField('perf_args', + default='', + description='The optional profile command. It ' + 'enables perf commands to record perforamance ' + 'related counters. It must start with perf ' + 'command record or stat followed by arguments.')) + self.AddField( + BooleanField('download_debug', + default=True, + description='Download compressed debug symbols alongwith ' + 'image. This can provide more info matching symbols for' + 'profiles, but takes larger space. By default, download' + 'it only when perf_args is specified.')) + self.AddField( + TextField('cache_dir', + default='', + description='The abs path of cache dir. ' + 'Default is /home/$(whoami)/cros_scratch.')) + self.AddField( + BooleanField('cache_only', + default=False, + description='Whether to use only cached ' + 'results (do not rerun failed tests).')) + self.AddField( + BooleanField('no_email', + default=False, + description='Whether to disable the email to ' + 'user after crosperf finishes.')) + self.AddField( + BooleanField('json_report', + default=False, + description='Whether to generate a json version ' + 'of the report, for archiving.')) + self.AddField( + BooleanField('show_all_results', + default=False, + description='When running Telemetry tests, ' + 'whether to all the results, instead of just ' + 'the default (summary) results.')) + self.AddField( + TextField('share_cache', + default='', + description='Path to alternate cache whose data ' + 'you want to use. It accepts multiple directories ' + 'separated by a ",".')) self.AddField( TextField('results_dir', default='', description='The results dir.')) self.AddField( @@ -297,55 +263,49 @@ class GlobalSettings(Settings): description='Whether to compress all test results other than ' 'reports into a tarball to save disk space.')) self.AddField( - TextField( - 'locks_dir', - default='', - description='An alternate directory to use for ' - 'storing/checking machine file locks for local machines. ' - 'By default the file locks directory is ' - '/google/data/rw/users/mo/mobiletc-prebuild/locks.\n' - 'WARNING: If you use your own locks directory, ' - 'there is no guarantee that someone else might not ' - 'hold a lock on the same machine in a different ' - 'locks directory.')) - self.AddField( - TextField( - 'chrome_src', - description='The path to the source of chrome. ' - 'This is used to run telemetry benchmarks. ' - 'The default one is the src inside chroot.', - required=False, - default='')) - self.AddField( - IntegerField( - 'retries', - default=0, - description='Number of times to retry a ' - 'benchmark run.')) - self.AddField( - TextField( - 'cwp_dso', - description='The DSO type that we want to use for ' - 'CWP approximation. This is used to run telemetry ' - 'benchmarks. Valid DSO types can be found from dso_list ' - 'in experiment_factory.py. The default value is set to ' - 'be empty.', - required=False, - default='')) - self.AddField( - BooleanField( - 'enable_aslr', - description='Enable ASLR on the machine to run the ' - 'benchmarks. ASLR is disabled by default', - required=False, - default=False)) - self.AddField( - BooleanField( - 'ignore_min_max', - description='When doing math for the raw results, ' - 'ignore min and max values to reduce noise.', - required=False, - default=False)) + TextField('locks_dir', + default='', + description='An alternate directory to use for ' + 'storing/checking machine file locks for local machines. ' + 'By default the file locks directory is ' + '/google/data/rw/users/mo/mobiletc-prebuild/locks.\n' + 'WARNING: If you use your own locks directory, ' + 'there is no guarantee that someone else might not ' + 'hold a lock on the same machine in a different ' + 'locks directory.')) + self.AddField( + TextField('chrome_src', + description='The path to the source of chrome. ' + 'This is used to run telemetry benchmarks. ' + 'The default one is the src inside chroot.', + required=False, + default='')) + self.AddField( + IntegerField('retries', + default=0, + description='Number of times to retry a ' + 'benchmark run.')) + self.AddField( + TextField('cwp_dso', + description='The DSO type that we want to use for ' + 'CWP approximation. This is used to run telemetry ' + 'benchmarks. Valid DSO types can be found from dso_list ' + 'in experiment_factory.py. The default value is set to ' + 'be empty.', + required=False, + default='')) + self.AddField( + BooleanField('enable_aslr', + description='Enable ASLR on the machine to run the ' + 'benchmarks. ASLR is disabled by default', + required=False, + default=False)) + self.AddField( + BooleanField('ignore_min_max', + description='When doing math for the raw results, ' + 'ignore min and max values to reduce noise.', + required=False, + default=False)) self.AddField( TextField( 'intel_pstate', @@ -356,12 +316,11 @@ class GlobalSettings(Settings): required=False, default='no_hwp')) self.AddField( - BooleanField( - 'turbostat', - description='Run turbostat process in the background' - ' of a benchmark. Enabled by default.', - required=False, - default=True)) + BooleanField('turbostat', + description='Run turbostat process in the background' + ' of a benchmark. Enabled by default.', + required=False, + default=True)) self.AddField( FloatField( 'top_interval', @@ -377,22 +336,20 @@ class GlobalSettings(Settings): required=False, default=1)) self.AddField( - IntegerField( - 'cooldown_temp', - required=False, - default=40, - description='Wait until CPU temperature goes down below' - ' specified temperature in Celsius' - ' prior starting a benchmark. ' - 'By default the value is set to 40 degrees.')) - self.AddField( - IntegerField( - 'cooldown_time', - required=False, - default=10, - description='Wait specified time in minutes allowing' - ' CPU to cool down. Zero value disables cooldown. ' - 'The default value is 10 minutes.')) + IntegerField('cooldown_temp', + required=False, + default=40, + description='Wait until CPU temperature goes down below' + ' specified temperature in Celsius' + ' prior starting a benchmark. ' + 'By default the value is set to 40 degrees.')) + self.AddField( + IntegerField('cooldown_time', + required=False, + default=10, + description='Wait specified time in minutes allowing' + ' CPU to cool down. Zero value disables cooldown. ' + 'The default value is 10 minutes.')) self.AddField( EnumField( 'governor', @@ -439,6 +396,12 @@ class GlobalSettings(Settings): ' or equal to a percent of max_freq. ' 'CPU frequency is reduced to 95%% by default to reduce thermal ' 'throttling.')) + self.AddField( + BooleanField( + 'no_lock', + default=False, + description='Do not attempt to lock the DUT.' + ' Useful when lock is held externally, say with crosfleet.')) class SettingsFactory(object): diff --git a/crosperf/settings_factory_unittest.py b/crosperf/settings_factory_unittest.py index 035da8d7..8277e870 100755 --- a/crosperf/settings_factory_unittest.py +++ b/crosperf/settings_factory_unittest.py @@ -50,7 +50,7 @@ class GlobalSettingsTest(unittest.TestCase): def test_init(self): res = settings_factory.GlobalSettings('g_settings') self.assertIsNotNone(res) - self.assertEqual(len(res.fields), 39) + self.assertEqual(len(res.fields), 40) self.assertEqual(res.GetField('name'), '') self.assertEqual(res.GetField('board'), '') self.assertEqual(res.GetField('crosfleet'), False) @@ -108,7 +108,7 @@ class SettingsFactoryTest(unittest.TestCase): g_settings = settings_factory.SettingsFactory().GetSettings( 'global', 'global') self.assertIsInstance(g_settings, settings_factory.GlobalSettings) - self.assertEqual(len(g_settings.fields), 39) + self.assertEqual(len(g_settings.fields), 40) if __name__ == '__main__': diff --git a/crosperf/test_cache/compare_output/results.txt b/crosperf/test_cache/compare_output/results.pickle Binary files differindex 592e7161..587863c5 100644 --- a/crosperf/test_cache/compare_output/results.txt +++ b/crosperf/test_cache/compare_output/results.pickle diff --git a/crosperf/test_cache/test_input/results.txt b/crosperf/test_cache/test_input/results.pickle index 33ba6ab7..33ba6ab7 100644 --- a/crosperf/test_cache/test_input/results.txt +++ b/crosperf/test_cache/test_input/results.pickle diff --git a/crosperf/test_cache/test_puretelemetry_input/results.txt b/crosperf/test_cache/test_puretelemetry_input/results.pickle index 497d1cf3..497d1cf3 100644 --- a/crosperf/test_cache/test_puretelemetry_input/results.txt +++ b/crosperf/test_cache/test_puretelemetry_input/results.pickle diff --git a/llvm_extra/create_llvm_extra.sh b/llvm_extra/create_llvm_extra.sh index 6f34a0b2..b58e0508 100755 --- a/llvm_extra/create_llvm_extra.sh +++ b/llvm_extra/create_llvm_extra.sh @@ -82,7 +82,7 @@ function create_llvm_extra_ebuild() { set -e -# Sanity checks. +# Confidence checks. check_cmd "${@}" # Create llvm-extra ebuild. create_llvm_extra_ebuild "${@}" diff --git a/llvm_tools/README.md b/llvm_tools/README.md index 43c80ad6..74fad6c9 100644 --- a/llvm_tools/README.md +++ b/llvm_tools/README.md @@ -119,6 +119,7 @@ For example, to create a roll CL to the git hash of revision 367622: $ ./update_chromeos_llvm_hash.py \ --update_packages sys-devel/llvm sys-libs/compiler-rt \ sys-libs/libcxx sys-libs/libcxxabi sys-libs/llvm-libunwind \ + 'dev-util/lldb-server' \ --llvm_version 367622 \ --failure_mode disable_patches ``` diff --git a/llvm_tools/get_upstream_patch.py b/llvm_tools/get_upstream_patch.py index 77783d41..7a4be3eb 100755 --- a/llvm_tools/get_upstream_patch.py +++ b/llvm_tools/get_upstream_patch.py @@ -6,8 +6,6 @@ """Get an upstream patch to LLVM's PATCHES.json.""" -from __future__ import print_function - import argparse import json import logging @@ -18,12 +16,20 @@ import sys import typing as t from datetime import datetime +import dataclasses + import chroot import get_llvm_hash import git import git_llvm_rev import update_chromeos_llvm_hash +__DOC_EPILOGUE = """ +Example Usage: + get_upstream_patch --chroot_path ~/chromiumos --platform chromiumos \ +--sha 1234567 --sha 890abdc +""" + class CherrypickError(ValueError): """A ValueError that highlights the cherry-pick has been seen before""" @@ -32,7 +38,7 @@ class CherrypickError(ValueError): def add_patch(patches_json_path: str, patches_dir: str, relative_patches_dir: str, start_version: git_llvm_rev.Rev, llvm_dir: str, rev: t.Union[git_llvm_rev.Rev, str], sha: str, - package: str): + package: str, platforms: t.List[str]): """Gets the start and end intervals in 'json_file'. Args: @@ -47,6 +53,7 @@ def add_patch(patches_json_path: str, patches_dir: str, revisions, the git sha from the local commit created by 'arc patch' is used. package: The LLVM project name this patch applies to. + platforms: List of platforms this patch applies to. Raises: CherrypickError: A ValueError that highlights the cherry-pick has been @@ -85,20 +92,29 @@ def add_patch(patches_json_path: str, patches_dir: str, subprocess.check_call(cmd, stdout=f, cwd=llvm_dir) commit_subject = subprocess.check_output( - ['git', 'log', '-n1', '--format=%s', sha], cwd=llvm_dir, encoding='utf-8') + ['git', 'log', '-n1', '--format=%s', sha], + cwd=llvm_dir, + encoding='utf-8') - patch_metadata = { - 'comment': commit_subject.strip(), + patch_props = { 'rel_patch_path': rel_patch_path, 'start_version': start_version.number, + 'metadata': { + 'title': commit_subject.strip(), + 'info': [], + }, + 'platforms': sorted(platforms), + 'end_version': rev.number if isinstance(rev, git_llvm_rev.Rev) else None, } - if isinstance(rev, git_llvm_rev.Rev): - patch_metadata['end_version'] = rev.number - patches_json.append(patch_metadata) + patches_json.append(patch_props) temp_file = patches_json_path + '.tmp' with open(temp_file, 'w', encoding='utf-8') as f: - json.dump(patches_json, f, indent=4, separators=(',', ': ')) + json.dump(patches_json, + f, + indent=4, + separators=(',', ': '), + sort_keys=True) f.write('\n') os.rename(temp_file, patches_json_path) @@ -178,7 +194,7 @@ def get_package_names(sha: str, llvm_dir: str) -> list: def create_patch_for_packages(packages: t.List[str], symlinks: t.List[str], start_rev: git_llvm_rev.Rev, rev: t.Union[git_llvm_rev.Rev, str], sha: str, - llvm_dir: str): + llvm_dir: str, platforms: t.List[str]): """Create a patch and add its metadata for each package""" for package, symlink in zip(packages, symlinks): symlink_dir = os.path.dirname(symlink) @@ -186,8 +202,15 @@ def create_patch_for_packages(packages: t.List[str], symlinks: t.List[str], relative_patches_dir = 'cherry' if package == 'llvm' else '' patches_dir = os.path.join(symlink_dir, 'files', relative_patches_dir) logging.info('Getting %s (%s) into %s', rev, sha, package) - add_patch(patches_json_path, patches_dir, relative_patches_dir, start_rev, - llvm_dir, rev, sha, package) + add_patch(patches_json_path, + patches_dir, + relative_patches_dir, + start_rev, + llvm_dir, + rev, + sha, + package, + platforms=platforms) def make_cl(symlinks_to_uprev: t.List[str], llvm_symlink_dir: str, branch: str, @@ -212,35 +235,35 @@ def resolve_symbolic_sha(start_sha: str, llvm_symlink_dir: str) -> str: return start_sha -def find_patches_and_make_cl(chroot_path: str, patches: t.List[str], - start_rev: git_llvm_rev.Rev, - llvm_config: git_llvm_rev.LLVMConfig, - llvm_symlink_dir: str, create_cl: bool, - reviewers: t.Optional[t.List[str]], - cc: t.Optional[t.List[str]]): +def find_patches_and_make_cl( + chroot_path: str, patches: t.List[str], start_rev: git_llvm_rev.Rev, + llvm_config: git_llvm_rev.LLVMConfig, llvm_symlink_dir: str, + create_cl: bool, skip_dependencies: bool, + reviewers: t.Optional[t.List[str]], cc: t.Optional[t.List[str]], + platforms: t.List[str]): + + converted_patches = [ + _convert_patch(llvm_config, skip_dependencies, p) for p in patches + ] + potential_duplicates = _get_duplicate_shas(converted_patches) + if potential_duplicates: + err_msg = '\n'.join(f'{a.patch} == {b.patch}' + for a, b in potential_duplicates) + raise RuntimeError(f'Found Duplicate SHAs:\n{err_msg}') + + # CL Related variables, only used if `create_cl` + symlinks_to_uprev = [] + commit_messages = [ + 'llvm: get patches from upstream\n', + ] + branch = f'get-upstream-{datetime.now().strftime("%Y%m%d%H%M%S%f")}' + if create_cl: - branch = f'get-upstream-{datetime.now().strftime("%Y%m%d%H%M%S%f")}' git.CreateBranch(llvm_symlink_dir, branch) - symlinks_to_uprev = [] - commit_messages = [ - 'llvm: get patches from upstream\n', - ] - - for patch in patches: - # git hash should only have lower-case letters - is_differential = patch.startswith('D') - if is_differential: - subprocess.check_output( - ['arc', 'patch', '--nobranch', '--revision', patch], - cwd=llvm_config.dir, - ) - sha = resolve_llvm_ref(llvm_config.dir, 'HEAD') - rev = patch - else: - sha = resolve_llvm_ref(llvm_config.dir, patch) - rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha) + + for parsed_patch in converted_patches: # Find out the llvm projects changed in this commit - packages = get_package_names(sha, llvm_config.dir) + packages = get_package_names(parsed_patch.sha, llvm_config.dir) # Find out the ebuild symlinks of the corresponding ChromeOS packages symlinks = chroot.GetChrootEbuildPaths(chroot_path, [ 'sys-devel/llvm' if package == 'llvm' else 'sys-libs/' + package @@ -248,23 +271,25 @@ def find_patches_and_make_cl(chroot_path: str, patches: t.List[str], ]) symlinks = chroot.ConvertChrootPathsToAbsolutePaths(chroot_path, symlinks) # Create a local patch for all the affected llvm projects - create_patch_for_packages(packages, symlinks, start_rev, rev, sha, - llvm_config.dir) + create_patch_for_packages(packages, + symlinks, + start_rev, + parsed_patch.rev, + parsed_patch.sha, + llvm_config.dir, + platforms=platforms) if create_cl: symlinks_to_uprev.extend(symlinks) - if is_differential: - msg = f'\n\nreviews.llvm.org/{patch}\n' - else: - msg = f'\n\nreviews.llvm.org/rG{sha}\n' commit_messages.extend([ - msg, - subprocess.check_output(['git', 'log', '-n1', '--oneline', sha], - cwd=llvm_config.dir, - encoding='utf-8') + parsed_patch.git_msg(), + subprocess.check_output( + ['git', 'log', '-n1', '--oneline', parsed_patch.sha], + cwd=llvm_config.dir, + encoding='utf-8') ]) - if is_differential: + if parsed_patch.is_differential: subprocess.check_output(['git', 'reset', '--hard', 'HEAD^'], cwd=llvm_config.dir) @@ -273,10 +298,67 @@ def find_patches_and_make_cl(chroot_path: str, patches: t.List[str], reviewers, cc) +@dataclasses.dataclass(frozen=True) +class ParsedPatch: + """Class to keep track of bundled patch info.""" + patch: str + sha: str + is_differential: bool + rev: t.Union[git_llvm_rev.Rev, str] + + def git_msg(self) -> str: + if self.is_differential: + return f'\n\nreviews.llvm.org/{self.patch}\n' + return f'\n\nreviews.llvm.org/rG{self.sha}\n' + + +def _convert_patch(llvm_config: git_llvm_rev.LLVMConfig, + skip_dependencies: bool, patch: str) -> ParsedPatch: + """Extract git revision info from a patch. + + Args: + llvm_config: LLVM configuration object. + skip_dependencies: Pass --skip-dependecies for to `arc` + patch: A single patch referent string. + + Returns: + A [ParsedPatch] object. + """ + + # git hash should only have lower-case letters + is_differential = patch.startswith('D') + if is_differential: + subprocess.check_output( + [ + 'arc', 'patch', '--nobranch', + '--skip-dependencies' if skip_dependencies else '--revision', patch + ], + cwd=llvm_config.dir, + ) + sha = resolve_llvm_ref(llvm_config.dir, 'HEAD') + rev = patch + else: + sha = resolve_llvm_ref(llvm_config.dir, patch) + rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha) + return ParsedPatch(patch=patch, + sha=sha, + rev=rev, + is_differential=is_differential) + + +def _get_duplicate_shas( + patches: t.List[ParsedPatch]) -> t.List[t.Tuple[ParsedPatch, ParsedPatch]]: + """Return a list of Patches which have duplicate SHA's""" + return [(left, right) for i, left in enumerate(patches) + for right in patches[i + 1:] if left.sha == right.sha] + + def get_from_upstream(chroot_path: str, create_cl: bool, start_sha: str, patches: t.List[str], + platforms: t.List[str], + skip_dependencies: bool = False, reviewers: t.List[str] = None, cc: t.List[str] = None): llvm_symlink = chroot.ConvertChrootPathsToAbsolutePaths( @@ -299,15 +381,17 @@ def get_from_upstream(chroot_path: str, remote='origin', dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools()) start_sha = resolve_llvm_ref(llvm_config.dir, start_sha) - find_patches_and_make_cl( - chroot_path=chroot_path, - patches=patches, - start_rev=git_llvm_rev.translate_sha_to_rev(llvm_config, start_sha), - llvm_config=llvm_config, - llvm_symlink_dir=llvm_symlink_dir, - create_cl=create_cl, - reviewers=reviewers, - cc=cc) + find_patches_and_make_cl(chroot_path=chroot_path, + patches=patches, + platforms=platforms, + start_rev=git_llvm_rev.translate_sha_to_rev( + llvm_config, start_sha), + llvm_config=llvm_config, + llvm_symlink_dir=llvm_symlink_dir, + create_cl=create_cl, + skip_dependencies=skip_dependencies, + reviewers=reviewers, + cc=cc) logging.info('Complete.') @@ -318,41 +402,60 @@ def main(): level=logging.INFO, ) - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - '--chroot_path', - default=os.path.join(os.path.expanduser('~'), 'chromiumos'), - help='the path to the chroot (default: %(default)s)') + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__DOC_EPILOGUE) + parser.add_argument('--chroot_path', + default=os.path.join(os.path.expanduser('~'), + 'chromiumos'), + help='the path to the chroot (default: %(default)s)') parser.add_argument( '--start_sha', default='llvm-next', help='LLVM SHA that the patch should start applying at. You can specify ' '"llvm" or "llvm-next", as well. Defaults to %(default)s.') - parser.add_argument( - '--sha', - action='append', - default=[], - help='The LLVM git SHA to cherry-pick.') + parser.add_argument('--sha', + action='append', + default=[], + help='The LLVM git SHA to cherry-pick.') parser.add_argument( '--differential', action='append', default=[], help='The LLVM differential revision to apply. Example: D1234') parser.add_argument( - '--create_cl', - default=False, + '--platform', + action='append', + required=True, + help='Apply this patch to the give platform. Common options include ' + '"chromiumos" and "android". Can be specified multiple times to ' + 'apply to multiple platforms') + parser.add_argument('--create_cl', + action='store_true', + help='Automatically create a CL if specified') + parser.add_argument( + '--skip_dependencies', action='store_true', - help='Automatically create a CL if specified') + help="Skips a LLVM differential revision's dependencies. Only valid " + 'when --differential appears exactly once.') args = parser.parse_args() if not (args.sha or args.differential): parser.error('--sha or --differential required') + if args.skip_dependencies and len(args.differential) != 1: + parser.error("--skip_dependencies is only valid when there's exactly one " + 'supplied differential') + get_from_upstream( chroot_path=args.chroot_path, create_cl=args.create_cl, start_sha=args.start_sha, - patches=args.sha + args.differential) + patches=args.sha + args.differential, + skip_dependencies=args.skip_dependencies, + platforms=args.platform, + ) if __name__ == '__main__': diff --git a/llvm_tools/llvm_bisection.py b/llvm_tools/llvm_bisection.py index b1898ea9..0148efd2 100755 --- a/llvm_tools/llvm_bisection.py +++ b/llvm_tools/llvm_bisection.py @@ -20,6 +20,7 @@ import chroot import get_llvm_hash import git_llvm_rev import modify_a_tryjob +import update_chromeos_llvm_hash import update_tryjob_status @@ -51,18 +52,16 @@ def GetCommandLineArgs(): 'the first bad version (default: %(default)s)') # Add argument for the good LLVM revision for bisection. - parser.add_argument( - '--start_rev', - required=True, - type=int, - help='The good revision for the bisection.') + parser.add_argument('--start_rev', + required=True, + type=int, + help='The good revision for the bisection.') # Add argument for the bad LLVM revision for bisection. - parser.add_argument( - '--end_rev', - required=True, - type=int, - help='The bad revision for the bisection.') + parser.add_argument('--end_rev', + required=True, + type=int, + help='The bad revision for the bisection.') # Add argument for the absolute path to the file that contains information on # the previous tested svn version. @@ -88,42 +87,38 @@ def GetCommandLineArgs(): 'of updating the packages') # Add argument for custom options for the tryjob. - parser.add_argument( - '--options', - required=False, - nargs='+', - help='options to use for the tryjob testing') + parser.add_argument('--options', + required=False, + nargs='+', + help='options to use for the tryjob testing') # Add argument for the builder to use for the tryjob. - parser.add_argument( - '--builder', required=True, help='builder to use for the tryjob testing') + parser.add_argument('--builder', + required=True, + help='builder to use for the tryjob testing') # Add argument for the description of the tryjob. - parser.add_argument( - '--description', - required=False, - nargs='+', - help='the description of the tryjob') + parser.add_argument('--description', + required=False, + nargs='+', + help='the description of the tryjob') # Add argument for a specific chroot path. - parser.add_argument( - '--chroot_path', - default=cros_root, - help='the path to the chroot (default: %(default)s)') + parser.add_argument('--chroot_path', + default=cros_root, + help='the path to the chroot (default: %(default)s)') # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - '--verbose', - action='store_true', - help='display contents of a command to the terminal ' - '(default: %(default)s)') + parser.add_argument('--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - '--nocleanup', - action='store_false', - dest='cleanup', - help='Abandon CLs created for bisectoin') + parser.add_argument('--nocleanup', + action='store_false', + dest='cleanup', + help='Abandon CLs created for bisectoin') args_output = parser.parse_args() @@ -174,8 +169,7 @@ def GetRemainingRange(start, end, tryjobs): all_bad_revisions = [end] all_bad_revisions.extend( - cur_tryjob['rev'] - for cur_tryjob in tryjobs + cur_tryjob['rev'] for cur_tryjob in tryjobs if cur_tryjob['status'] == update_tryjob_status.TryjobStatus.BAD.value) # The minimum value for the 'bad' field in the tryjobs is the new end @@ -184,8 +178,7 @@ def GetRemainingRange(start, end, tryjobs): all_good_revisions = [start] all_good_revisions.extend( - cur_tryjob['rev'] - for cur_tryjob in tryjobs + cur_tryjob['rev'] for cur_tryjob in tryjobs if cur_tryjob['status'] == update_tryjob_status.TryjobStatus.GOOD.value) # The maximum value for the 'good' field in the tryjobs is the new start @@ -205,8 +198,8 @@ def GetRemainingRange(start, end, tryjobs): pending_revisions = { tryjob['rev'] for tryjob in tryjobs - if tryjob['status'] == update_tryjob_status.TryjobStatus.PENDING.value and - good_rev < tryjob['rev'] < bad_rev + if tryjob['status'] == update_tryjob_status.TryjobStatus.PENDING.value + and good_rev < tryjob['rev'] < bad_rev } # Find all revisions that are to be skipped within 'good_rev' and 'bad_rev'. @@ -217,8 +210,8 @@ def GetRemainingRange(start, end, tryjobs): skip_revisions = { tryjob['rev'] for tryjob in tryjobs - if tryjob['status'] == update_tryjob_status.TryjobStatus.SKIP.value and - good_rev < tryjob['rev'] < bad_rev + if tryjob['status'] == update_tryjob_status.TryjobStatus.SKIP.value + and good_rev < tryjob['rev'] < bad_rev } return good_rev, bad_rev, pending_revisions, skip_revisions @@ -295,66 +288,62 @@ def main(args_output): """ chroot.VerifyOutsideChroot() - update_packages = [ - 'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx', - 'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind' - ] patch_metadata_file = 'PATCHES.json' start = args_output.start_rev end = args_output.end_rev bisect_state = LoadStatusFile(args_output.last_tested, start, end) if start != bisect_state['start'] or end != bisect_state['end']: - raise ValueError(f'The start {start} or the end {end} version provided is ' - f'different than "start" {bisect_state["start"]} or "end" ' - f'{bisect_state["end"]} in the .JSON file') + raise ValueError( + f'The start {start} or the end {end} version provided is ' + f'different than "start" {bisect_state["start"]} or "end" ' + f'{bisect_state["end"]} in the .JSON file') - # Pending and skipped revisions are between 'start_revision' and - # 'end_revision'. - start_revision, end_revision, pending_revisions, skip_revisions = \ - GetRemainingRange(start, end, bisect_state['jobs']) + # Pending and skipped revisions are between 'start_rev' and 'end_rev'. + start_rev, end_rev, pending_revs, skip_revs = GetRemainingRange( + start, end, bisect_state['jobs']) - revisions, git_hashes = GetCommitsBetween(start_revision, end_revision, + revisions, git_hashes = GetCommitsBetween(start_rev, end_rev, args_output.parallel, - args_output.src_path, - pending_revisions, skip_revisions) + args_output.src_path, pending_revs, + skip_revs) - # No more revisions between 'start_revision' and 'end_revision', so + # No more revisions between 'start_rev' and 'end_rev', so # bisection is complete. # - # This is determined by finding all valid revisions between 'start_revision' - # and 'end_revision' and that are NOT in the 'pending' and 'skipped' set. + # This is determined by finding all valid revisions between 'start_rev' + # and 'end_rev' and that are NOT in the 'pending' and 'skipped' set. if not revisions: - if pending_revisions: + if pending_revs: # Some tryjobs are not finished which may change the actual bad # commit/revision when those tryjobs are finished. - no_revisions_message = (f'No revisions between start {start_revision} ' - f'and end {end_revision} to create tryjobs\n') + no_revisions_message = (f'No revisions between start {start_rev} ' + f'and end {end_rev} to create tryjobs\n') - if pending_revisions: - no_revisions_message += ( - 'The following tryjobs are pending:\n' + - '\n'.join(str(rev) for rev in pending_revisions) + '\n') + if pending_revs: + no_revisions_message += ('The following tryjobs are pending:\n' + + '\n'.join(str(rev) + for rev in pending_revs) + '\n') - if skip_revisions: + if skip_revs: no_revisions_message += ('The following tryjobs were skipped:\n' + - '\n'.join(str(rev) for rev in skip_revisions) + - '\n') + '\n'.join(str(rev) + for rev in skip_revs) + '\n') raise ValueError(no_revisions_message) print(f'Finished bisecting for {args_output.last_tested}') if args_output.src_path: bad_llvm_hash = get_llvm_hash.GetGitHashFrom(args_output.src_path, - end_revision) + end_rev) else: - bad_llvm_hash = get_llvm_hash.LLVMHash().GetLLVMHash(end_revision) - print(f'The bad revision is {end_revision} and its commit hash is ' + bad_llvm_hash = get_llvm_hash.LLVMHash().GetLLVMHash(end_rev) + print(f'The bad revision is {end_rev} and its commit hash is ' f'{bad_llvm_hash}') - if skip_revisions: - skip_revisions_message = ('\nThe following revisions were skipped:\n' + - '\n'.join(str(rev) for rev in skip_revisions)) - print(skip_revisions_message) + if skip_revs: + skip_revs_message = ('\nThe following revisions were skipped:\n' + + '\n'.join(str(rev) for rev in skip_revs)) + print(skip_revs_message) if args_output.cleanup: # Abandon all the CLs created for bisection @@ -378,9 +367,9 @@ def main(args_output): raise ValueError(f'Revision {rev} exists already in "jobs"') Bisect(revisions, git_hashes, bisect_state, args_output.last_tested, - update_packages, args_output.chroot_path, patch_metadata_file, - args_output.extra_change_lists, args_output.options, - args_output.builder, args_output.verbose) + update_chromeos_llvm_hash.DEFAULT_PACKAGES, args_output.chroot_path, + patch_metadata_file, args_output.extra_change_lists, + args_output.options, args_output.builder, args_output.verbose) if __name__ == '__main__': diff --git a/llvm_tools/modify_a_tryjob.py b/llvm_tools/modify_a_tryjob.py index 4d41e6b2..519fb51e 100755 --- a/llvm_tools/modify_a_tryjob.py +++ b/llvm_tools/modify_a_tryjob.py @@ -17,9 +17,9 @@ import sys import chroot import failure_modes import get_llvm_hash +import update_chromeos_llvm_hash import update_packages_and_run_tests import update_tryjob_status -import update_chromeos_llvm_hash class ModifyTryjob(enum.Enum): @@ -57,11 +57,10 @@ def GetCommandLineArgs(): # Add argument that determines which revision to search for in the list of # tryjobs. - parser.add_argument( - '--revision', - required=True, - type=int, - help='The revision to either remove or relaunch.') + parser.add_argument('--revision', + required=True, + type=int, + help='The revision to either remove or relaunch.') # Add argument for other change lists that want to run alongside the tryjob. parser.add_argument( @@ -72,40 +71,38 @@ def GetCommandLineArgs(): 'of updating the packages') # Add argument for custom options for the tryjob. - parser.add_argument( - '--options', - required=False, - nargs='+', - help='options to use for the tryjob testing') + parser.add_argument('--options', + required=False, + nargs='+', + help='options to use for the tryjob testing') # Add argument for the builder to use for the tryjob. - parser.add_argument('--builder', help='builder to use for the tryjob testing') + parser.add_argument('--builder', + help='builder to use for the tryjob testing') # Add argument for a specific chroot path. - parser.add_argument( - '--chroot_path', - default=cros_root, - help='the path to the chroot (default: %(default)s)') + parser.add_argument('--chroot_path', + default=cros_root, + help='the path to the chroot (default: %(default)s)') # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - '--verbose', - action='store_true', - help='display contents of a command to the terminal ' - '(default: %(default)s)') + parser.add_argument('--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') args_output = parser.parse_args() - if not os.path.isfile(args_output.status_file) or \ - not args_output.status_file.endswith('.json'): + if (not os.path.isfile(args_output.status_file) + or not args_output.status_file.endswith('.json')): raise ValueError('File does not exist or does not ending in ".json" ' ': %s' % args_output.status_file) - if args_output.modify_tryjob == ModifyTryjob.ADD.value and \ - not args_output.builder: + if (args_output.modify_tryjob == ModifyTryjob.ADD.value + and not args_output.builder): raise ValueError('A builder is required for adding a tryjob.') - elif args_output.modify_tryjob != ModifyTryjob.ADD.value and \ - args_output.builder: + elif (args_output.modify_tryjob != ModifyTryjob.ADD.value + and args_output.builder): raise ValueError('Specifying a builder is only available when adding a ' 'tryjob.') @@ -234,13 +231,13 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, bisect_contents['jobs'][tryjob_index]['cl'], bisect_contents['jobs'][tryjob_index]['extra_cls'], bisect_contents['jobs'][tryjob_index]['options'], - bisect_contents['jobs'][tryjob_index]['builder'], chroot_path, verbose) + bisect_contents['jobs'][tryjob_index]['builder'], chroot_path) bisect_contents['jobs'][tryjob_index][ 'status'] = update_tryjob_status.TryjobStatus.PENDING.value bisect_contents['jobs'][tryjob_index]['link'] = tryjob_results[0]['link'] - bisect_contents['jobs'][tryjob_index]['buildbucket_id'] = tryjob_results[0][ - 'buildbucket_id'] + bisect_contents['jobs'][tryjob_index]['buildbucket_id'] = tryjob_results[ + 0]['buildbucket_id'] print('Successfully relaunched the tryjob for revision %d and updated ' 'the tryjob link to %s' % (revision, tryjob_results[0]['link'])) @@ -253,17 +250,14 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, # Make sure the revision is within the bounds of the start and end of the # bisection. elif bisect_contents['start'] < revision < bisect_contents['end']: - update_packages = [ - 'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx', - 'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind' - ] patch_metadata_file = 'PATCHES.json' git_hash, revision = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption( revision) - tryjob_dict = AddTryjob(update_packages, git_hash, revision, chroot_path, + tryjob_dict = AddTryjob(update_chromeos_llvm_hash.DEFAULT_PACKAGES, + git_hash, revision, chroot_path, patch_metadata_file, extra_cls, options, builder, verbose, revision) @@ -277,7 +271,10 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, modify_tryjob) with open(status_file, 'w') as update_tryjobs: - json.dump(bisect_contents, update_tryjobs, indent=4, separators=(',', ': ')) + json.dump(bisect_contents, + update_tryjobs, + indent=4, + separators=(',', ': ')) def main(): @@ -290,9 +287,9 @@ def main(): PerformTryjobModification(args_output.revision, ModifyTryjob(args_output.modify_tryjob), args_output.status_file, - args_output.extra_change_lists, args_output.options, - args_output.builder, args_output.chroot_path, - args_output.verbose) + args_output.extra_change_lists, + args_output.options, args_output.builder, + args_output.chroot_path, args_output.verbose) if __name__ == '__main__': diff --git a/llvm_tools/nightly_revert_checker.py b/llvm_tools/nightly_revert_checker.py index 6941f3d6..5e878816 100755 --- a/llvm_tools/nightly_revert_checker.py +++ b/llvm_tools/nightly_revert_checker.py @@ -33,7 +33,7 @@ State = t.Any def _find_interesting_android_shas(android_llvm_toolchain_dir: str - ) -> t.List[t.Tuple[str, str]]: + ) -> t.List[t.Tuple[str, str]]: llvm_project = os.path.join(android_llvm_toolchain_dir, 'toolchain/llvm-project') @@ -44,7 +44,7 @@ def _find_interesting_android_shas(android_llvm_toolchain_dir: str encoding='utf-8', ).strip() - main_legacy = get_llvm_merge_base('aosp/master-legacy') + main_legacy = get_llvm_merge_base('aosp/master-legacy') # nocheck testing_upstream = get_llvm_merge_base('aosp/testing-upstream') result = [('main-legacy', main_legacy)] @@ -55,8 +55,7 @@ def _find_interesting_android_shas(android_llvm_toolchain_dir: str def _parse_llvm_ebuild_for_shas(ebuild_file: io.TextIOWrapper - ) -> t.List[t.Tuple[str, str]]: - + ) -> t.List[t.Tuple[str, str]]: def parse_ebuild_assignment(line: str) -> str: no_comments = line.split('#')[0] no_assign = no_comments.split('=', 1)[1].strip() @@ -84,12 +83,11 @@ def _parse_llvm_ebuild_for_shas(ebuild_file: io.TextIOWrapper def _find_interesting_chromeos_shas(chromeos_base: str - ) -> t.List[t.Tuple[str, str]]: + ) -> t.List[t.Tuple[str, str]]: llvm_dir = os.path.join(chromeos_base, 'src/third_party/chromiumos-overlay/sys-devel/llvm') candidate_ebuilds = [ - os.path.join(llvm_dir, x) - for x in os.listdir(llvm_dir) + os.path.join(llvm_dir, x) for x in os.listdir(llvm_dir) if '_pre' in x and not os.path.islink(os.path.join(llvm_dir, x)) ] @@ -197,8 +195,10 @@ def find_shas(llvm_dir: str, interesting_shas: t.List[t.Tuple[str, str]], state: State, new_state: State): for friendly_name, sha in interesting_shas: logging.info('Finding reverts across %s (%s)', friendly_name, sha) - all_reverts = revert_checker.find_reverts( - llvm_dir, sha, root='origin/' + git_llvm_rev.MAIN_BRANCH) + all_reverts = revert_checker.find_reverts(llvm_dir, + sha, + root='origin/' + + git_llvm_rev.MAIN_BRANCH) logging.info('Detected the following revert(s) across %s:\n%s', friendly_name, pprint.pformat(all_reverts)) @@ -230,13 +230,12 @@ def do_cherrypick(chroot_path: str, llvm_dir: str, seen.add(friendly_name) for sha, reverted_sha in reverts: try: - get_upstream_patch.get_from_upstream( - chroot_path=chroot_path, - create_cl=True, - start_sha=reverted_sha, - patches=[sha], - reviewers=reviewers, - cc=cc) + get_upstream_patch.get_from_upstream(chroot_path=chroot_path, + create_cl=True, + start_sha=reverted_sha, + patches=[sha], + reviewers=reviewers, + cc=cc) except get_upstream_patch.CherrypickError as e: logging.info('%s, skipping...', str(e)) return new_state @@ -245,7 +244,6 @@ def do_cherrypick(chroot_path: str, llvm_dir: str, def do_email(is_dry_run: bool, llvm_dir: str, repository: str, interesting_shas: t.List[t.Tuple[str, str]], state: State, recipients: _EmailRecipients) -> State: - def prettify_sha(sha: str) -> tiny_render.Piece: rev = get_llvm_hash.GetVersionFrom(llvm_dir, sha) @@ -253,8 +251,8 @@ def do_email(is_dry_run: bool, llvm_dir: str, repository: str, short_sha = sha[:12] return tiny_render.Switch( text=f'r{rev} ({short_sha})', - html=tiny_render.Link( - href='https://reviews.llvm.org/rG' + sha, inner='r' + str(rev)), + html=tiny_render.Link(href='https://reviews.llvm.org/rG' + sha, + inner='r' + str(rev)), ) def get_sha_description(sha: str) -> tiny_render.Piece: @@ -267,11 +265,12 @@ def do_email(is_dry_run: bool, llvm_dir: str, repository: str, new_state: State = {} for friendly_name, sha, new_reverts in find_shas(llvm_dir, interesting_shas, state, new_state): - email = _generate_revert_email(repository, friendly_name, sha, prettify_sha, - get_sha_description, new_reverts) + email = _generate_revert_email(repository, friendly_name, sha, + prettify_sha, get_sha_description, + new_reverts) if is_dry_run: - logging.info('Would send email:\nSubject: %s\nBody:\n%s\n', email.subject, - tiny_render.render_text_pieces(email.body)) + logging.info('Would send email:\nSubject: %s\nBody:\n%s\n', + email.subject, tiny_render.render_text_pieces(email.body)) else: logging.info('Sending email with subject %r...', email.subject) _send_revert_email(recipients, email) @@ -281,16 +280,19 @@ def do_email(is_dry_run: bool, llvm_dir: str, repository: str, def parse_args(argv: t.List[str]) -> t.Any: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( 'action', choices=['cherry-pick', 'email', 'dry-run'], help='Automatically cherry-pick upstream reverts, send an email, or ' 'write to stdout.') - parser.add_argument( - '--state_file', required=True, help='File to store persistent state in.') - parser.add_argument( - '--llvm_dir', required=True, help='Up-to-date LLVM directory to use.') + parser.add_argument('--state_file', + required=True, + help='File to store persistent state in.') + parser.add_argument('--llvm_dir', + required=True, + help='Up-to-date LLVM directory to use.') parser.add_argument('--debug', action='store_true') parser.add_argument( '--reviewers', @@ -309,8 +311,9 @@ def parse_args(argv: t.List[str]) -> t.Any: subparsers.required = True chromeos_subparser = subparsers.add_parser('chromeos') - chromeos_subparser.add_argument( - '--chromeos_dir', required=True, help='Up-to-date CrOS directory to use.') + chromeos_subparser.add_argument('--chromeos_dir', + required=True, + help='Up-to-date CrOS directory to use.') android_subparser = subparsers.add_parser('android') android_subparser.add_argument( @@ -322,7 +325,7 @@ def parse_args(argv: t.List[str]) -> t.Any: def find_chroot(opts: t.Any, reviewers: t.List[str], cc: t.List[str] - ) -> t.Tuple[str, t.List[t.Tuple[str, str]], _EmailRecipients]: + ) -> t.Tuple[str, t.List[t.Tuple[str, str]], _EmailRecipients]: recipients = reviewers + cc if opts.repository == 'chromeos': chroot_path = opts.chromeos_dir @@ -335,9 +338,9 @@ def find_chroot(opts: t.Any, reviewers: t.List[str], cc: t.List[str] chroot_path = opts.android_llvm_toolchain_dir return (chroot_path, _find_interesting_android_shas(chroot_path), - _EmailRecipients( - well_known=[], - direct=['android-llvm-dev@google.com'] + recipients)) + _EmailRecipients(well_known=[], + direct=['android-llvm-dev@google.com'] + + recipients)) else: raise ValueError(f'Unknown repository {opts.repository}') @@ -366,21 +369,19 @@ def main(argv: t.List[str]) -> int: # We want to be as free of obvious side-effects as possible in case something # above breaks. Hence, action as late as possible. if action == 'cherry-pick': - new_state = do_cherrypick( - chroot_path=chroot_path, - llvm_dir=llvm_dir, - interesting_shas=interesting_shas, - state=state, - reviewers=reviewers, - cc=cc) + new_state = do_cherrypick(chroot_path=chroot_path, + llvm_dir=llvm_dir, + interesting_shas=interesting_shas, + state=state, + reviewers=reviewers, + cc=cc) else: - new_state = do_email( - is_dry_run=action == 'dry-run', - llvm_dir=llvm_dir, - repository=repository, - interesting_shas=interesting_shas, - state=state, - recipients=recipients) + new_state = do_email(is_dry_run=action == 'dry-run', + llvm_dir=llvm_dir, + repository=repository, + interesting_shas=interesting_shas, + state=state, + recipients=recipients) _write_state(state_file, new_state) return 0 diff --git a/llvm_tools/patch_sync/.gitignore b/llvm_tools/patch_sync/.gitignore new file mode 100644 index 00000000..2f7896d1 --- /dev/null +++ b/llvm_tools/patch_sync/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/llvm_tools/patch_sync/Cargo.lock b/llvm_tools/patch_sync/Cargo.lock new file mode 100644 index 00000000..63a9fcf8 --- /dev/null +++ b/llvm_tools/patch_sync/Cargo.lock @@ -0,0 +1,453 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b26702f315f53b6071259e15dd9d64528213b44d61de1ec926eca7715d62203" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "cpufeatures" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +dependencies = [ + "libc", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.112" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "patch_sync" +version = "0.1.0" +dependencies = [ + "anyhow", + "rand", + "regex", + "serde", + "serde_json", + "sha2", + "structopt", + "time", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f84e92c0f7c9d58328b85a78557813e4bd845130db68d7184635344399423b1" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "ryu" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" + +[[package]] +name = "serde" +version = "1.0.132" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9875c23cf305cd1fd7eb77234cbb705f21ea6a72c637a5c6db5fe4b8e7f008" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.132" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc0db5cb2556c0e558887d9bbdcf6ac4471e83ff66cf696e5419024d1606276" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcbd0344bc6533bc7ec56df11d42fb70f1b912351c0825ccb7211b59d8af7cf5" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" +dependencies = [ + "block-buffer", + "cfg-if", + "cpufeatures", + "digest", + "opaque-debug", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23a1dfb999630e338648c83e91c59a4e9fb7620f520c3194b6b89e276f2f1959" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "time" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41effe7cfa8af36f439fac33861b66b049edc6f9a32331e2312660529c1c24ad" +dependencies = [ + "libc", +] + +[[package]] +name = "typenum" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" + +[[package]] +name = "unicode-segmentation" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/llvm_tools/patch_sync/Cargo.toml b/llvm_tools/patch_sync/Cargo.toml new file mode 100644 index 00000000..43082627 --- /dev/null +++ b/llvm_tools/patch_sync/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "patch_sync" +version = "0.1.0" +authors = ["Jordan R Abrahams-Whitehead <ajordanr@google.com>"] +edition = "2018" + +[profile.release] +panic = "abort" + +[dependencies] +anyhow = "1.0" +regex = "1.5" +serde = {version = "1.0", features = ["derive"]} +serde_json = "1.0" +sha2 = "0.9" +structopt = "0.3" +time = "0.3" + +[dev-dependencies] +rand = "0.8" diff --git a/llvm_tools/patch_sync/src/main.rs b/llvm_tools/patch_sync/src/main.rs new file mode 100644 index 00000000..081ce01a --- /dev/null +++ b/llvm_tools/patch_sync/src/main.rs @@ -0,0 +1,173 @@ +mod patch_parsing; +mod version_control; + +use anyhow::{Context, Result}; +use std::path::PathBuf; +use structopt::StructOpt; + +fn main() -> Result<()> { + match Opt::from_args() { + Opt::Show { + cros_checkout_path, + android_checkout_path, + sync, + } => show_subcmd(cros_checkout_path, android_checkout_path, sync), + Opt::Transpose { + cros_checkout_path, + old_cros_ref, + android_checkout_path, + old_android_ref, + sync, + verbose, + dry_run, + no_commit, + } => transpose_subcmd(TransposeOpt { + cros_checkout_path, + old_cros_ref, + android_checkout_path, + old_android_ref, + sync, + verbose, + dry_run, + no_commit, + }), + } +} + +fn show_subcmd( + cros_checkout_path: PathBuf, + android_checkout_path: PathBuf, + sync: bool, +) -> Result<()> { + let ctx = version_control::RepoSetupContext { + cros_checkout: cros_checkout_path, + android_checkout: android_checkout_path, + sync_before: sync, + }; + ctx.setup()?; + let cros_patches_path = ctx.cros_patches_path(); + let android_patches_path = ctx.android_patches_path(); + let cur_cros_collection = patch_parsing::PatchCollection::parse_from_file(&cros_patches_path) + .context("could not parse cros PATCHES.json")?; + let cur_android_collection = + patch_parsing::PatchCollection::parse_from_file(&android_patches_path) + .context("could not parse android PATCHES.json")?; + let merged = cur_cros_collection.union(&cur_android_collection)?; + println!("{}", merged.serialize_patches()?); + Ok(()) +} + +#[allow(dead_code)] +struct TransposeOpt { + cros_checkout_path: PathBuf, + old_cros_ref: String, + android_checkout_path: PathBuf, + old_android_ref: String, + sync: bool, + verbose: bool, + dry_run: bool, + no_commit: bool, +} + +fn transpose_subcmd(args: TransposeOpt) -> Result<()> { + let ctx = version_control::RepoSetupContext { + cros_checkout: args.cros_checkout_path, + android_checkout: args.android_checkout_path, + sync_before: args.sync, + }; + ctx.setup()?; + let cros_patches_path = ctx.cros_patches_path(); + let android_patches_path = ctx.android_patches_path(); + + // Chromium OS Patches ---------------------------------------------------- + let mut cur_cros_collection = + patch_parsing::PatchCollection::parse_from_file(&cros_patches_path) + .context("parsing cros PATCHES.json")?; + let new_cros_patches: patch_parsing::PatchCollection = { + let cros_old_patches_json = ctx.old_cros_patch_contents(&args.old_cros_ref)?; + let old_cros_collection = patch_parsing::PatchCollection::parse_from_str( + cros_patches_path.parent().unwrap().to_path_buf(), + &cros_old_patches_json, + )?; + cur_cros_collection.subtract(&old_cros_collection)? + }; + + // Android Patches ------------------------------------------------------- + let mut cur_android_collection = + patch_parsing::PatchCollection::parse_from_file(&android_patches_path) + .context("parsing android PATCHES.json")?; + let new_android_patches: patch_parsing::PatchCollection = { + let android_old_patches_json = ctx.old_android_patch_contents(&args.old_android_ref)?; + let old_android_collection = patch_parsing::PatchCollection::parse_from_str( + android_patches_path.parent().unwrap().to_path_buf(), + &android_old_patches_json, + )?; + cur_android_collection.subtract(&old_android_collection)? + }; + + // Transpose Patches ----------------------------------------------------- + new_cros_patches.transpose_write(&mut cur_cros_collection)?; + new_android_patches.transpose_write(&mut cur_android_collection)?; + + if !args.no_commit { + return Ok(()); + } + // Commit and upload for review ------------------------------------------ + ctx.cros_repo_upload() + .context("uploading chromiumos changes")?; + ctx.android_repo_upload() + .context("uploading android changes")?; + Ok(()) +} + +#[derive(Debug, structopt::StructOpt)] +#[structopt(name = "patch_sync", about = "A pipeline for syncing the patch code")] +enum Opt { + /// Show a combined view of the PATCHES.json file, without making any changes. + #[allow(dead_code)] + Show { + #[structopt(parse(from_os_str))] + cros_checkout_path: PathBuf, + #[structopt(parse(from_os_str))] + android_checkout_path: PathBuf, + #[structopt(short, long)] + sync: bool, + }, + /// Transpose patches from two PATCHES.json files + /// to each other. + Transpose { + /// Path to the ChromiumOS source repo checkout. + #[structopt(long = "cros-checkout", parse(from_os_str))] + cros_checkout_path: PathBuf, + + /// Git ref (e.g. hash) for the ChromiumOS overlay to use as the base. + #[structopt(long = "overlay-base-ref")] + old_cros_ref: String, + + /// Path to the Android Open Source Project source repo checkout. + #[structopt(long = "aosp-checkout", parse(from_os_str))] + android_checkout_path: PathBuf, + + /// Git ref (e.g. hash) for the llvm_android repo to use as the base. + #[structopt(long = "aosp-base-ref")] + old_android_ref: String, + + /// Run repo sync before transposing. + #[structopt(short, long)] + sync: bool, + + /// Print information to stdout + #[structopt(short, long)] + verbose: bool, + + /// Do not change any files. Useful in combination with `--verbose` + /// Implies `--no-commit` and `--no-upload`. + #[structopt(long)] + dry_run: bool, + + /// Do not commit any changes made. + /// Implies `--no-upload`. + #[structopt(long)] + no_commit: bool, + }, +} diff --git a/llvm_tools/patch_sync/src/patch_parsing.rs b/llvm_tools/patch_sync/src/patch_parsing.rs new file mode 100644 index 00000000..733451ae --- /dev/null +++ b/llvm_tools/patch_sync/src/patch_parsing.rs @@ -0,0 +1,313 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{copy, File}; +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// JSON serde struct. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PatchDictSchema { + pub rel_patch_path: String, + pub start_version: Option<u64>, + pub end_version: Option<u64>, + pub platforms: BTreeSet<String>, + pub metadata: Option<BTreeMap<String, serde_json::Value>>, +} + +/// Struct to keep track of patches and their relative paths. +#[derive(Debug, Clone)] +pub struct PatchCollection { + pub patches: Vec<PatchDictSchema>, + pub workdir: PathBuf, +} + +impl PatchCollection { + /// Create a `PatchCollection` from a PATCHES. + pub fn parse_from_file(json_file: &Path) -> Result<Self> { + Ok(Self { + patches: serde_json::from_reader(File::open(json_file)?)?, + workdir: json_file + .parent() + .ok_or_else(|| anyhow!("failed to get json_file parent"))? + .to_path_buf(), + }) + } + + /// Create a `PatchCollection` from a string literal and a workdir. + pub fn parse_from_str(workdir: PathBuf, contents: &str) -> Result<Self> { + Ok(Self { + patches: serde_json::from_str(contents).context("parsing from str")?, + workdir, + }) + } + + #[allow(dead_code)] + /// Return true if the collection is tracking any patches. + pub fn is_empty(&self) -> bool { + self.patches.is_empty() + } + + /// Compute the set-set subtraction, returning a new `PatchCollection` which + /// keeps the minuend's wordir. + pub fn subtract(&self, subtrahend: &Self) -> Result<Self> { + let mut new_patches = Vec::new(); + // This is O(n^2) when it could be much faster, but n is always going to be less + // than 1k and speed is not important here. + for our_patch in &self.patches { + let found_in_sub = subtrahend.patches.iter().any(|sub_patch| { + let hash1 = subtrahend + .hash_from_rel_patch(sub_patch) + .expect("getting hash from subtrahend patch"); + let hash2 = self + .hash_from_rel_patch(our_patch) + .expect("getting hash from our patch"); + hash1 == hash2 + }); + if !found_in_sub { + new_patches.push(our_patch.clone()); + } + } + Ok(Self { + patches: new_patches, + workdir: self.workdir.clone(), + }) + } + + pub fn union(&self, other: &Self) -> Result<Self> { + self.union_helper( + other, + |p| self.hash_from_rel_patch(p), + |p| other.hash_from_rel_patch(p), + ) + } + + fn union_helper( + &self, + other: &Self, + our_hash_f: impl Fn(&PatchDictSchema) -> Result<String>, + their_hash_f: impl Fn(&PatchDictSchema) -> Result<String>, + ) -> Result<Self> { + // 1. For all our patches: + // a. If there exists a matching patch hash from `other`: + // i. Create a new patch with merged platform info, + // ii. add the new patch to our new collection. + // iii. Mark the other patch as "merged" + // b. Otherwise, copy our patch to the new collection + // 2. For all unmerged patches from the `other` + // a. Copy their patch into the new collection + let mut combined_patches = Vec::new(); + let mut other_merged = vec![false; other.patches.len()]; + + // 1. + for p in &self.patches { + let our_hash = our_hash_f(p)?; + let mut found = false; + // a. + for (idx, merged) in other_merged.iter_mut().enumerate() { + if !*merged { + let other_p = &other.patches[idx]; + let their_hash = their_hash_f(other_p)?; + if our_hash == their_hash { + // i. + let new_platforms = + p.platforms.union(&other_p.platforms).cloned().collect(); + // ii. + combined_patches.push(PatchDictSchema { + rel_patch_path: p.rel_patch_path.clone(), + start_version: p.start_version, + end_version: p.end_version, + platforms: new_platforms, + metadata: p.metadata.clone(), + }); + // iii. + *merged = true; + found = true; + break; + } + } + } + // b. + if !found { + combined_patches.push(p.clone()); + } + } + // 2. + // Add any remaining, other-only patches. + for (idx, merged) in other_merged.iter().enumerate() { + if !*merged { + combined_patches.push(other.patches[idx].clone()); + } + } + + Ok(Self { + workdir: self.workdir.clone(), + patches: combined_patches, + }) + } + + /// Copy all patches from this collection into another existing collection, and write that + /// to the existing collection's file. + pub fn transpose_write(&self, existing_collection: &mut Self) -> Result<()> { + for p in &self.patches { + let original_file_path = self.workdir.join(&p.rel_patch_path); + let copy_file_path = existing_collection.workdir.join(&p.rel_patch_path); + copy_create_parents(&original_file_path, ©_file_path)?; + existing_collection.patches.push(p.clone()); + } + existing_collection.write_patches_json("PATCHES.json") + } + + /// Write out the patch collection contents to a PATCHES.json file. + fn write_patches_json(&self, filename: &str) -> Result<()> { + let write_path = self.workdir.join(filename); + let mut new_patches_file = File::create(&write_path) + .with_context(|| format!("writing to {}", write_path.display()))?; + new_patches_file.write_all(self.serialize_patches()?.as_bytes())?; + Ok(()) + } + + pub fn serialize_patches(&self) -> Result<String> { + let mut serialization_buffer = Vec::<u8>::new(); + // Four spaces to indent json serialization. + let mut serializer = serde_json::Serializer::with_formatter( + &mut serialization_buffer, + serde_json::ser::PrettyFormatter::with_indent(b" "), + ); + self.patches + .serialize(&mut serializer) + .context("serializing patches to JSON")?; + // Append a newline at the end if not present. This is necessary to get + // past some pre-upload hooks. + if serialization_buffer.last() != Some(&b'\n') { + serialization_buffer.push(b'\n'); + } + Ok(std::str::from_utf8(&serialization_buffer)?.to_string()) + } + + fn hash_from_rel_patch(&self, patch: &PatchDictSchema) -> Result<String> { + hash_from_patch_path(&self.workdir.join(&patch.rel_patch_path)) + } +} + +/// Get the hash from the patch file contents. +/// +/// Not every patch file actually contains its own hash, +/// we must compute the hash ourselves when it's not found. +fn hash_from_patch(patch_contents: impl Read) -> Result<String> { + let mut reader = BufReader::new(patch_contents); + let mut buf = String::new(); + reader.read_line(&mut buf)?; + let mut first_line_iter = buf.trim().split(' ').fuse(); + let (fst_word, snd_word) = (first_line_iter.next(), first_line_iter.next()); + if let (Some("commit" | "From"), Some(hash_str)) = (fst_word, snd_word) { + // If the first line starts with either "commit" or "From", the following + // text is almost certainly a commit hash. + Ok(hash_str.to_string()) + } else { + // This is an annoying case where the patch isn't actually a commit. + // So we'll hash the entire file, and hope that's sufficient. + let mut hasher = Sha256::new(); + hasher.update(&buf); // Have to hash the first line. + reader.read_to_string(&mut buf)?; + hasher.update(buf); // Hash the rest of the file. + let sha = hasher.finalize(); + Ok(format!("{:x}", &sha)) + } +} + +fn hash_from_patch_path(patch: &Path) -> Result<String> { + let f = File::open(patch)?; + hash_from_patch(f) +} + +/// Copy a file from one path to another, and create any parent +/// directories along the way. +fn copy_create_parents(from: &Path, to: &Path) -> Result<()> { + let to_parent = to + .parent() + .with_context(|| format!("getting parent of {}", to.display()))?; + if !to_parent.exists() { + std::fs::create_dir_all(to_parent)?; + } + + copy(&from, &to) + .with_context(|| format!("copying file from {} to {}", &from.display(), &to.display()))?; + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + /// Test we can extract the hash from patch files. + #[test] + fn test_hash_from_patch() { + // Example git patch from Gerrit + let desired_hash = "004be4037e1e9c6092323c5c9268acb3ecf9176c"; + let test_file_contents = "commit 004be4037e1e9c6092323c5c9268acb3ecf9176c\n\ + Author: An Author <some_email>\n\ + Date: Thu Aug 6 12:34:16 2020 -0700"; + assert_eq!( + &hash_from_patch(test_file_contents.as_bytes()).unwrap(), + desired_hash + ); + + // Example git patch from upstream + let desired_hash = "6f85225ef3791357f9b1aa097b575b0a2b0dff48"; + let test_file_contents = "From 6f85225ef3791357f9b1aa097b575b0a2b0dff48\n\ + Mon Sep 17 00:00:00 2001\n\ + From: Another Author <another_email>\n\ + Date: Wed, 18 Aug 2021 15:03:03 -0700"; + assert_eq!( + &hash_from_patch(test_file_contents.as_bytes()).unwrap(), + desired_hash + ); + } + + #[test] + fn test_union() { + let patch1 = PatchDictSchema { + start_version: Some(0), + end_version: Some(1), + rel_patch_path: "a".into(), + metadata: None, + platforms: BTreeSet::from(["x".into()]), + }; + let patch2 = PatchDictSchema { + rel_patch_path: "b".into(), + platforms: BTreeSet::from(["x".into(), "y".into()]), + ..patch1.clone() + }; + let patch3 = PatchDictSchema { + platforms: BTreeSet::from(["z".into(), "x".into()]), + ..patch1.clone() + }; + let collection1 = PatchCollection { + workdir: PathBuf::new(), + patches: vec![patch1, patch2], + }; + let collection2 = PatchCollection { + workdir: PathBuf::new(), + patches: vec![patch3], + }; + let union = collection1 + .union_helper( + &collection2, + |p| Ok(p.rel_patch_path.to_string()), + |p| Ok(p.rel_patch_path.to_string()), + ) + .expect("could not create union"); + assert_eq!(union.patches.len(), 2); + assert_eq!( + union.patches[0].platforms.iter().collect::<Vec<&String>>(), + vec!["x", "z"] + ); + assert_eq!( + union.patches[1].platforms.iter().collect::<Vec<&String>>(), + vec!["x", "y"] + ); + } +} diff --git a/llvm_tools/patch_sync/src/version_control.rs b/llvm_tools/patch_sync/src/version_control.rs new file mode 100644 index 00000000..3dc5aae9 --- /dev/null +++ b/llvm_tools/patch_sync/src/version_control.rs @@ -0,0 +1,238 @@ +use anyhow::{anyhow, bail, ensure, Context, Result}; +use regex::Regex; +use std::ffi::OsStr; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +const CHROMIUMOS_OVERLAY_REL_PATH: &str = "src/third_party/chromiumos-overlay"; +const ANDROID_LLVM_REL_PATH: &str = "toolchain/llvm_android"; + +/// Context struct to keep track of both Chromium OS and Android checkouts. +#[derive(Debug)] +pub struct RepoSetupContext { + pub cros_checkout: PathBuf, + pub android_checkout: PathBuf, + /// Run `repo sync` before doing any comparisons. + pub sync_before: bool, +} + +impl RepoSetupContext { + pub fn setup(&self) -> Result<()> { + if self.sync_before { + repo_cd_cmd(&self.cros_checkout, &["sync", CHROMIUMOS_OVERLAY_REL_PATH])?; + repo_cd_cmd(&self.android_checkout, &["sync", ANDROID_LLVM_REL_PATH])?; + } + Ok(()) + } + + pub fn cros_repo_upload(&self) -> Result<()> { + let llvm_dir = self + .cros_checkout + .join(&CHROMIUMOS_OVERLAY_REL_PATH) + .join("sys-devel/llvm"); + ensure!( + llvm_dir.is_dir(), + "CrOS LLVM dir {} is not a directory", + llvm_dir.display() + ); + Self::rev_bump_llvm(&llvm_dir)?; + Self::repo_upload( + &self.cros_checkout, + CHROMIUMOS_OVERLAY_REL_PATH, + &Self::build_commit_msg("android", "chromiumos", "BUG=None\nTEST=CQ"), + ) + } + + pub fn android_repo_upload(&self) -> Result<()> { + Self::repo_upload( + &self.android_checkout, + ANDROID_LLVM_REL_PATH, + &Self::build_commit_msg("chromiumos", "android", "Test: N/A"), + ) + } + + fn repo_upload(path: &Path, git_wd: &str, commit_msg: &str) -> Result<()> { + // TODO(ajordanr): Need to clean up if there's any failures during upload. + let git_path = &path.join(&git_wd); + ensure!( + git_path.is_dir(), + "git_path {} is not a directory", + git_path.display() + ); + repo_cd_cmd(path, &["start", "patch_sync_branch", git_wd])?; + git_cd_cmd(git_path, &["add", "."])?; + git_cd_cmd(git_path, &["commit", "-m", commit_msg])?; + repo_cd_cmd(path, &["upload", "-y", "--verify", git_wd])?; + Ok(()) + } + + pub fn android_patches_path(&self) -> PathBuf { + self.android_checkout + .join(&ANDROID_LLVM_REL_PATH) + .join("patches/PATCHES.json") + } + + pub fn cros_patches_path(&self) -> PathBuf { + self.cros_checkout + .join(&CHROMIUMOS_OVERLAY_REL_PATH) + .join("sys-devel/llvm/files/PATCHES.json") + } + + /// Increment LLVM's revision number + fn rev_bump_llvm(llvm_dir: &Path) -> Result<PathBuf> { + let ebuild = find_ebuild(llvm_dir) + .with_context(|| format!("finding ebuild in {} to rev bump", llvm_dir.display()))?; + let ebuild_dir = ebuild.parent().unwrap(); + let suffix_matcher = Regex::new(r"-r([0-9]+)\.ebuild").unwrap(); + let ebuild_name = ebuild + .file_name() + .unwrap() + .to_str() + .ok_or_else(|| anyhow!("converting ebuild filename to utf-8"))?; + let new_path = if let Some(captures) = suffix_matcher.captures(ebuild_name) { + let full_suffix = captures.get(0).unwrap().as_str(); + let cur_version = captures.get(1).unwrap().as_str().parse::<u32>().unwrap(); + let new_filename = + ebuild_name.replace(full_suffix, &format!("-r{}.ebuild", cur_version + 1_u32)); + let new_path = ebuild_dir.join(new_filename); + fs::rename(&ebuild, &new_path)?; + new_path + } else { + // File did not end in a revision. We should append -r1 to the end. + let new_filename = ebuild.file_stem().unwrap().to_string_lossy() + "-r1.ebuild"; + let new_path = ebuild_dir.join(new_filename.as_ref()); + fs::rename(&ebuild, &new_path)?; + new_path + }; + Ok(new_path) + } + + /// Return the contents of the old PATCHES.json from Chromium OS + #[allow(dead_code)] + pub fn old_cros_patch_contents(&self, hash: &str) -> Result<String> { + Self::old_file_contents( + hash, + &self.cros_checkout.join(CHROMIUMOS_OVERLAY_REL_PATH), + Path::new("sys-devel/llvm/files/PATCHES.json"), + ) + } + + /// Return the contents of the old PATCHES.json from android + #[allow(dead_code)] + pub fn old_android_patch_contents(&self, hash: &str) -> Result<String> { + Self::old_file_contents( + hash, + &self.android_checkout.join(ANDROID_LLVM_REL_PATH), + Path::new("patches/PATCHES.json"), + ) + } + + /// Return the contents of an old file in git + #[allow(dead_code)] + fn old_file_contents(hash: &str, pwd: &Path, file: &Path) -> Result<String> { + let git_ref = format!( + "{}:{}", + hash, + file.to_str() + .ok_or_else(|| anyhow!("failed to convert filepath to str"))? + ); + let output = git_cd_cmd(pwd, &["show", &git_ref])?; + if !output.status.success() { + bail!("could not get old file contents for {}", &git_ref) + } + String::from_utf8(output.stdout) + .with_context(|| format!("converting {} file contents to UTF-8", &git_ref)) + } + + /// Create the commit message + fn build_commit_msg(from: &str, to: &str, footer: &str) -> String { + format!( + "[patch_sync] Synchronize patches from {}\n\n\ + Copies new PATCHES.json changes from {} to {}\n\n{}", + from, from, to, footer + ) + } +} + +/// Return the path of an ebuild located within the given directory. +fn find_ebuild(dir: &Path) -> Result<PathBuf> { + // TODO(ajordanr): Maybe use OnceCell for this regex? + let ebuild_matcher = Regex::new(r"(-r[0-9]+)?\.ebuild").unwrap(); + for entry in fs::read_dir(dir)? { + let path = entry?.path(); + if let Some(name) = path.file_name() { + if ebuild_matcher.is_match( + name.to_str() + .ok_or_else(|| anyhow!("converting filepath to UTF-8"))?, + ) { + return Ok(path); + } + } + } + bail!("could not find ebuild") +} + +/// Run a given git command from inside a specified git dir. +pub fn git_cd_cmd<I, S>(pwd: &Path, args: I) -> Result<Output> +where + I: IntoIterator<Item = S>, + S: AsRef<OsStr>, +{ + let output = Command::new("git").current_dir(&pwd).args(args).output()?; + if !output.status.success() { + bail!("git command failed") + } + Ok(output) +} + +pub fn repo_cd_cmd<I, S>(pwd: &Path, args: I) -> Result<()> +where + I: IntoIterator<Item = S>, + S: AsRef<OsStr>, +{ + let status = Command::new("repo").current_dir(&pwd).args(args).status()?; + if !status.success() { + bail!("repo command failed") + } + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + use rand::prelude::Rng; + use std::env; + use std::fs::File; + + #[test] + fn test_revbump_ebuild() { + // Random number to append at the end of the test folder to prevent conflicts. + let rng: u32 = rand::thread_rng().gen(); + let llvm_dir = env::temp_dir().join(format!("patch_sync_test_{}", rng)); + fs::create_dir(&llvm_dir).expect("creating llvm dir in temp directory"); + + { + // With revision + let ebuild_name = "llvm-13.0_pre433403_p20211019-r10.ebuild"; + let ebuild_path = llvm_dir.join(ebuild_name); + File::create(&ebuild_path).expect("creating test ebuild file"); + let new_ebuild_path = + RepoSetupContext::rev_bump_llvm(&llvm_dir).expect("rev bumping the ebuild"); + assert!(new_ebuild_path.ends_with("llvm-13.0_pre433403_p20211019-r11.ebuild")); + fs::remove_file(new_ebuild_path).expect("removing renamed ebuild file"); + } + { + // Without revision + let ebuild_name = "llvm-13.0_pre433403_p20211019.ebuild"; + let ebuild_path = llvm_dir.join(ebuild_name); + File::create(&ebuild_path).expect("creating test ebuild file"); + let new_ebuild_path = + RepoSetupContext::rev_bump_llvm(&llvm_dir).expect("rev bumping the ebuild"); + assert!(new_ebuild_path.ends_with("llvm-13.0_pre433403_p20211019-r1.ebuild")); + fs::remove_file(new_ebuild_path).expect("removing renamed ebuild file"); + } + + fs::remove_dir(&llvm_dir).expect("removing temp test dir"); + } +} diff --git a/llvm_tools/update_chromeos_llvm_hash.py b/llvm_tools/update_chromeos_llvm_hash.py index ab83321a..4e9b9104 100755 --- a/llvm_tools/update_chromeos_llvm_hash.py +++ b/llvm_tools/update_chromeos_llvm_hash.py @@ -25,6 +25,15 @@ import get_llvm_hash import git import llvm_patch_management +DEFAULT_PACKAGES = [ + 'dev-util/lldb-server', + 'sys-devel/llvm', + 'sys-libs/compiler-rt', + 'sys-libs/libcxx', + 'sys-libs/libcxxabi', + 'sys-libs/llvm-libunwind', +] + # Specify which LLVM hash to update class LLVMVariant(enum.Enum): @@ -69,26 +78,23 @@ def GetCommandLineArgs(): description="Updates the build's hash for llvm-next.") # Add argument for a specific chroot path. - parser.add_argument( - '--chroot_path', - default=defaultCrosRoot(), - help='the path to the chroot (default: %(default)s)') + parser.add_argument('--chroot_path', + default=defaultCrosRoot(), + help='the path to the chroot (default: %(default)s)') # Add argument for specific builds to uprev and update their llvm-next hash. - parser.add_argument( - '--update_packages', - default=['sys-devel/llvm'], - required=False, - nargs='+', - help='the ebuilds to update their hash for llvm-next ' - '(default: %(default)s)') + parser.add_argument('--update_packages', + default=DEFAULT_PACKAGES, + required=False, + nargs='+', + help='the ebuilds to update their hash for llvm-next ' + '(default: %(default)s)') # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - '--verbose', - action='store_true', - help='display contents of a command to the terminal ' - '(default: %(default)s)') + parser.add_argument('--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') # Add argument for the LLVM hash to update parser.add_argument( @@ -314,9 +320,11 @@ def UprevEbuildToVersion(symlink, svn_version, git_hash): count=1) # any other package else: - new_ebuild, is_changed = re.subn( - r'(\d+)\.(\d+)_pre([0-9]+)', - '%s.\\2_pre%s' % (llvm_major_version, svn_version), ebuild, count=1) + new_ebuild, is_changed = re.subn(r'(\d+)\.(\d+)_pre([0-9]+)', + '%s.\\2_pre%s' % + (llvm_major_version, svn_version), + ebuild, + count=1) if not is_changed: # failed to increment the revision number raise ValueError('Failed to uprev the ebuild.') @@ -397,7 +405,8 @@ def StagePatchMetadataFileForCommit(patch_metadata_file_path): # Cmd to stage the patch metadata file for commit. subprocess.check_output([ 'git', '-C', - os.path.dirname(patch_metadata_file_path), 'add', patch_metadata_file_path + os.path.dirname(patch_metadata_file_path), 'add', + patch_metadata_file_path ]) @@ -419,9 +428,9 @@ def StagePackagesPatchResultsForCommit(package_info_dict, commit_messages): # changed, if so, add which patches have changed to the commit # message. for package_name, patch_info_dict in package_info_dict.items(): - if (patch_info_dict['disabled_patches'] or - patch_info_dict['removed_patches'] or - patch_info_dict['modified_metadata']): + if (patch_info_dict['disabled_patches'] + or patch_info_dict['removed_patches'] + or patch_info_dict['modified_metadata']): cur_package_header = '\nFor the package %s:' % package_name commit_messages.append(cur_package_header) @@ -600,16 +609,16 @@ def main(): git_hash, svn_version = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption( git_hash_source) - change_list = UpdatePackages( - args_output.update_packages, - llvm_variant, - git_hash, - svn_version, - args_output.chroot_path, - args_output.patch_metadata_file, - failure_modes.FailureModes(args_output.failure_mode), - git_hash_source, - extra_commit_msg=None) + change_list = UpdatePackages(args_output.update_packages, + llvm_variant, + git_hash, + svn_version, + args_output.chroot_path, + args_output.patch_metadata_file, + failure_modes.FailureModes( + args_output.failure_mode), + git_hash_source, + extra_commit_msg=None) print('Successfully updated packages to %s (%d)' % (git_hash, svn_version)) print('Gerrit URL: %s' % change_list.url) diff --git a/llvm_tools/update_packages_and_run_tests.py b/llvm_tools/update_packages_and_run_tests.py index dd01253e..2e4a9058 100755 --- a/llvm_tools/update_packages_and_run_tests.py +++ b/llvm_tools/update_packages_and_run_tests.py @@ -51,10 +51,9 @@ def GetCommandLineArgs(): 'of updating the packages') # Add argument for a specific chroot path. - parser.add_argument( - '--chroot_path', - default=cros_root, - help='the path to the chroot (default: %(default)s)') + parser.add_argument('--chroot_path', + default=cros_root, + help='the path to the chroot (default: %(default)s)') # Add argument to choose between llvm and llvm-next. parser.add_argument( @@ -71,65 +70,58 @@ def GetCommandLineArgs(): 'arguments.') # Add argument for the LLVM version to use. - parser.add_argument( - '--llvm_version', - type=get_llvm_hash.IsSvnOption, - required=True, - help='which git hash of LLVM to find ' - '{google3, ToT, <svn_version>} ' - '(default: finds the git hash of the google3 LLVM ' - 'version)') + parser.add_argument('--llvm_version', + type=get_llvm_hash.IsSvnOption, + required=True, + help='which git hash of LLVM to find ' + '{google3, ToT, <svn_version>} ' + '(default: finds the git hash of the google3 LLVM ' + 'version)') # Add argument to add reviewers for the created CL. - parser.add_argument( - '--reviewers', - nargs='+', - default=[], - help='The reviewers for the package update changelist') + parser.add_argument('--reviewers', + nargs='+', + default=[], + help='The reviewers for the package update changelist') # Add argument for whether to display command contents to `stdout`. - parser.add_argument( - '--verbose', - action='store_true', - help='display contents of a command to the terminal ' - '(default: %(default)s)') + parser.add_argument('--verbose', + action='store_true', + help='display contents of a command to the terminal ' + '(default: %(default)s)') subparsers = parser.add_subparsers(dest='subparser_name') subparser_names = [] # Testing with the tryjobs. tryjob_subparser = subparsers.add_parser('tryjobs') subparser_names.append('tryjobs') - tryjob_subparser.add_argument( - '--builders', - required=True, - nargs='+', - default=[], - help='builders to use for the tryjob testing') + tryjob_subparser.add_argument('--builders', + required=True, + nargs='+', + default=[], + help='builders to use for the tryjob testing') # Add argument for custom options for the tryjob. - tryjob_subparser.add_argument( - '--options', - required=False, - nargs='+', - default=[], - help='options to use for the tryjob testing') + tryjob_subparser.add_argument('--options', + required=False, + nargs='+', + default=[], + help='options to use for the tryjob testing') # Testing with the recipe builders recipe_subparser = subparsers.add_parser('recipe') subparser_names.append('recipe') - recipe_subparser.add_argument( - '--options', - required=False, - nargs='+', - default=[], - help='options passed to the recipe builders') - - recipe_subparser.add_argument( - '--builders', - required=True, - nargs='+', - default=[], - help='recipe builders to launch') + recipe_subparser.add_argument('--options', + required=False, + nargs='+', + default=[], + help='options passed to the recipe builders') + + recipe_subparser.add_argument('--builders', + required=True, + nargs='+', + default=[], + help='recipe builders to launch') # Testing with CQ. cq_subparser = subparsers.add_parser('cq') @@ -360,7 +352,8 @@ def GetCQDependString(dependent_cls): return None # Cq-Depend must start a new paragraph prefixed with "Cq-Depend". - return '\nCq-Depend: ' + ', '.join(('chromium:%s' % i) for i in dependent_cls) + return '\nCq-Depend: ' + ', '.join( + ('chromium:%s' % i) for i in dependent_cls) def GetCQIncludeTrybotsString(trybot): @@ -402,11 +395,6 @@ def main(): args_output = GetCommandLineArgs() - update_packages = [ - 'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx', - 'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind' - ] - patch_metadata_file = 'PATCHES.json' svn_option = args_output.llvm_version @@ -420,8 +408,8 @@ def main(): # If --last_tested is specified, check if the current run has the same # arguments last time --last_tested is used. if args_output.last_tested: - chroot_file_paths = chroot.GetChrootEbuildPaths(args_output.chroot_path, - update_packages) + chroot_file_paths = chroot.GetChrootEbuildPaths( + args_output.chroot_path, update_chromeos_llvm_hash.DEFAULT_PACKAGES) arg_dict = { 'svn_version': svn_version, 'ebuilds': chroot_file_paths, @@ -449,7 +437,7 @@ def main(): extra_commit_msg += cq_trybot_msg change_list = update_chromeos_llvm_hash.UpdatePackages( - update_packages, + update_chromeos_llvm_hash.DEFAULT_PACKAGES, llvm_variant, git_hash, svn_version, diff --git a/lock_machine.py b/lock_machine.py index e7020591..03c8c991 100755 --- a/lock_machine.py +++ b/lock_machine.py @@ -125,8 +125,8 @@ class LockManager(object): Returns: A list of names of the toolchain machines in the ChromeOS HW lab. """ - machines_file = os.path.join( - os.path.dirname(__file__), 'crosperf', 'default_remotes') + machines_file = os.path.join(os.path.dirname(__file__), 'crosperf', + 'default_remotes') machine_list = [] with open(machines_file, 'r') as input_file: lines = input_file.readlines() @@ -166,8 +166,8 @@ class LockManager(object): """ if state['locked']: print('%s (%s)\t\t%slocked by %s since %s' % - (m, state['board'], '\t\t' if machine_type == MachineType.LOCAL else - '', state['locked_by'], state['lock_time'])) + (m, state['board'], '\t\t' if machine_type == MachineType.LOCAL + else '', state['locked_by'], state['lock_time'])) else: print('%s (%s)\t\t%sunlocked' % (m, state['board'], @@ -241,8 +241,8 @@ class LockManager(object): """ try: if should_lock_machine: - ret = file_lock_machine.Machine(machine, - self.locks_dir).Lock(True, sys.argv[0]) + ret = file_lock_machine.Machine(machine, self.locks_dir).Lock( + True, sys.argv[0]) else: ret = file_lock_machine.Machine(machine, self.locks_dir).Unlock(True) except Exception: @@ -325,14 +325,14 @@ class LockManager(object): # TODO(zhizhouy): Crosfleet doesn't support host info such as locked_by. # Need to update this when crosfleet supports it. - if (state['locked'] and state['locked_by'] and - state['locked_by'] != self.user): + if (state['locked'] and state['locked_by'] + and state['locked_by'] != self.user): raise DontOwnLock('Attempt to unlock machine (%s) locked by someone ' 'else (%s).' % (k, state['locked_by'])) elif cmd == 'lock': if state['locked']: - self.logger.LogWarning('Attempt to lock already locked machine (%s)' % - k) + self.logger.LogWarning( + 'Attempt to lock already locked machine (%s)' % k) self._InternalRemoveMachine(k) def GetMachineStates(self, cmd=''): @@ -381,12 +381,10 @@ class LockManager(object): swarming = os.path.join(self.chromeos_root, self.SWARMING) # TODO(zhizhouy): Swarming script doesn't support python3 so explicitly # launch it with python2 until migrated. - cmd = (('python2 %s ' \ - 'query --swarming https://chromeos-swarming.appspot.com ' \ - "%s 'bots/list?is_dead=FALSE&dimensions=dut_name:%s'") % \ - (swarming, - credential, - machine.rstrip('.cros'))) + cmd = (('python2 %s ' + 'query --swarming https://chromeos-swarming.appspot.com ' + "%s 'bots/list?is_dead=FALSE&dimensions=dut_name:%s'") % + (swarming, credential, machine.rstrip('.cros'))) exit_code, stdout, stderr = self.ce.RunCommandWOutput(cmd) if exit_code: raise ValueError('Querying bots failed (2); stdout: %r; stderr: %r' % @@ -411,15 +409,12 @@ class LockManager(object): credential = '' if os.path.exists(self.CROSFLEET_CREDENTIAL): credential = '-service-account-json %s' % self.CROSFLEET_CREDENTIAL - cmd = (('%s dut lease -minutes %s %s %s %s') % \ - (self.CROSFLEET_PATH, - self.LEASE_MINS, - credential, - '-host' if '.cros' in machine else '-board', - machine.rstrip('.cros'))) - # Wait 120 seconds for server to start the lease task, if not started, + cmd = (('%s dut lease -minutes %s %s %s %s') % + (self.CROSFLEET_PATH, self.LEASE_MINS, credential, '-host' + if '.cros' in machine else '-board', machine.rstrip('.cros'))) + # Wait 8 minutes for server to start the lease task, if not started, # we will treat it as unavailable. - check_interval_time = 120 + check_interval_time = 480 retval = self.ce.RunCommand(cmd, command_timeout=check_interval_time) return retval == self.SUCCESS @@ -432,10 +427,8 @@ class LockManager(object): credential = '' if os.path.exists(self.CROSFLEET_CREDENTIAL): credential = '-service-account-json %s' % self.CROSFLEET_CREDENTIAL - cmd = (('%s dut abandon %s %s') % \ - (self.CROSFLEET_PATH, - credential, - machine.rstrip('.cros'))) + cmd = (('%s dut abandon %s %s') % + (self.CROSFLEET_PATH, credential, machine.rstrip('.cros'))) retval = self.ce.RunCommand(cmd) return retval == self.SUCCESS @@ -451,44 +444,39 @@ def Main(argv): """ parser = argparse.ArgumentParser() - parser.add_argument( - '--list', - dest='cmd', - action='store_const', - const='status', - help='List current status of all known machines.') - parser.add_argument( - '--lock', - dest='cmd', - action='store_const', - const='lock', - help='Lock given machine(s).') - parser.add_argument( - '--unlock', - dest='cmd', - action='store_const', - const='unlock', - help='Unlock given machine(s).') - parser.add_argument( - '--status', - dest='cmd', - action='store_const', - const='status', - help='List current status of given machine(s).') - parser.add_argument( - '--remote', dest='remote', help='machines on which to operate') - parser.add_argument( - '--chromeos_root', - dest='chromeos_root', - required=True, - help='ChromeOS root to use for autotest scripts.') - parser.add_argument( - '--force', - dest='force', - action='store_true', - default=False, - help='Force lock/unlock of machines, even if not' - ' current lock owner.') + parser.add_argument('--list', + dest='cmd', + action='store_const', + const='status', + help='List current status of all known machines.') + parser.add_argument('--lock', + dest='cmd', + action='store_const', + const='lock', + help='Lock given machine(s).') + parser.add_argument('--unlock', + dest='cmd', + action='store_const', + const='unlock', + help='Unlock given machine(s).') + parser.add_argument('--status', + dest='cmd', + action='store_const', + const='status', + help='List current status of given machine(s).') + parser.add_argument('--remote', + dest='remote', + help='machines on which to operate') + parser.add_argument('--chromeos_root', + dest='chromeos_root', + required=True, + help='ChromeOS root to use for autotest scripts.') + parser.add_argument('--force', + dest='force', + action='store_true', + default=False, + help='Force lock/unlock of machines, even if not' + ' current lock owner.') options = parser.parse_args(argv) @@ -506,7 +494,8 @@ def Main(argv): if options.remote: machine_list = options.remote.split() - lock_manager = LockManager(machine_list, options.force, options.chromeos_root) + lock_manager = LockManager(machine_list, options.force, + options.chromeos_root) machine_states = lock_manager.GetMachineStates(cmd=options.cmd) cmd = options.cmd diff --git a/pgo_tools/monitor_pgo_profiles.py b/pgo_tools/monitor_pgo_profiles.py index be159b94..5c17423b 100755 --- a/pgo_tools/monitor_pgo_profiles.py +++ b/pgo_tools/monitor_pgo_profiles.py @@ -7,14 +7,11 @@ import argparse import datetime -import sys -import subprocess import logging +import subprocess +import sys from typing import List, NamedTuple, Optional, Tuple -from cros_utils import email_sender -from cros_utils import tiny_render - PGO_BUILDBOT_LINK = ('https://ci.chromium.org/p/chromeos/builders/toolchain/' 'pgo-generate-llvm-next-orchestrator') @@ -54,47 +51,35 @@ def fetch_most_recent_profdata(arch: str) -> ProfdataInfo: return max(infos) -def compose_complaint_email( +def compose_complaint( out_of_date_profiles: List[Tuple[datetime.datetime, ProfdataInfo]] -) -> Optional[Tuple[str, tiny_render.Piece]]: +) -> Optional[str]: if not out_of_date_profiles: return None if len(out_of_date_profiles) == 1: - subject = '1 llvm profile is out of date' - body = ['out-of-date profile:'] + body_lines = ['1 profile is out of date:'] else: - subject = f'{len(out_of_date_profiles)} llvm profiles are out of date' - body = ['out-of-date profiles:'] + body_lines = [f'{len(out_of_date_profiles)} profiles are out of date:'] - out_of_date_items = [] for arch, profdata_info in out_of_date_profiles: - out_of_date_items.append( - f'{arch} (most recent profile was from {profdata_info.date} at ' + body_lines.append( + f'- {arch} (most recent profile was from {profdata_info.date} at ' f'{profdata_info.location!r})') - body += [ - tiny_render.UnorderedList(out_of_date_items), - tiny_render.line_break, - tiny_render.line_break, + body_lines.append('\n') + body_lines.append( 'PTAL to see if the llvm-pgo-generate bots are functioning normally. ' - 'Their status can be found at ', - tiny_render.Link(href=PGO_BUILDBOT_LINK, inner=PGO_BUILDBOT_LINK), - '.', - ] - return subject, body + f'Their status can be found at {PGO_BUILDBOT_LINK}.') + return '\n'.join(body_lines) def main() -> None: logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '--dry_run', - action='store_true', - help="Don't actually send an email", - ) + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( '--max_age_days', # These builders run ~weekly. If we fail to generate two in a row, @@ -119,29 +104,12 @@ def main() -> None: if age >= max_age: out_of_date_profiles.append((arch, most_recent)) - email = compose_complaint_email(out_of_date_profiles) - if not email: - logging.info('No email to send; quit') - return - - subject, body = email + complaint = compose_complaint(out_of_date_profiles) + if complaint: + logging.error('%s', complaint) + sys.exit(1) - identifier = 'llvm-pgo-monitor' - subject = f'[{identifier}] {subject}' - - logging.info('Sending email with title %r', subject) - if args.dry_run: - logging.info('Dry run specified\nSubject: %s\nBody:\n%s', subject, - tiny_render.render_text_pieces(body)) - else: - email_sender.EmailSender().SendX20Email( - subject=subject, - identifier=identifier, - well_known_recipients=['mage'], - direct_recipients=['gbiv@google.com'], - text_body=tiny_render.render_text_pieces(body), - html_body=tiny_render.render_html_pieces(body), - ) + logging.info('Nothing seems wrong') if __name__ == '__main__': diff --git a/pgo_tools/monitor_pgo_profiles_unittest.py b/pgo_tools/monitor_pgo_profiles_unittest.py index b4e085ec..eef33887 100755 --- a/pgo_tools/monitor_pgo_profiles_unittest.py +++ b/pgo_tools/monitor_pgo_profiles_unittest.py @@ -11,41 +11,36 @@ import unittest import unittest.mock import monitor_pgo_profiles -from cros_utils import tiny_render class Test(unittest.TestCase): """Tests for monitor_pgo_profiles.""" + def test_compose_complaint_with_zero_out_of_date(self): + self.assertIsNone(monitor_pgo_profiles.compose_complaint([])) - def test_compose_complaint_email_with_zero_out_of_date(self): - self.assertIsNone(monitor_pgo_profiles.compose_complaint_email([])) - - def test_compose_complaint_email_with_one_out_of_date(self): + def test_compose_complaint_with_one_out_of_date(self): profdata_info = monitor_pgo_profiles.ProfdataInfo( date=datetime.datetime(2020, 1, 2, 3, 4, 5), location='gs://somewhere', ) - result = monitor_pgo_profiles.compose_complaint_email([ + result = monitor_pgo_profiles.compose_complaint([ ('some_arch', profdata_info), ]) - self.assertEqual(result, ('1 llvm profile is out of date', [ - 'out-of-date profile:', - tiny_render.UnorderedList([ - f'some_arch (most recent profile was from {profdata_info.date} at ' - f'{profdata_info.location!r})' - ]), - tiny_render.line_break, - tiny_render.line_break, - 'PTAL to see if the llvm-pgo-generate bots are functioning normally. ' - 'Their status can be found at ', - tiny_render.Link( - href=monitor_pgo_profiles.PGO_BUILDBOT_LINK, - inner=monitor_pgo_profiles.PGO_BUILDBOT_LINK, - ), - '.', - ])) + self.assertEqual( + result, + '\n'.join(( + '1 profile is out of date:', + f'- some_arch (most recent profile was from {profdata_info.date} ' + f'at {profdata_info.location!r})', + '', + '', + 'PTAL to see if the llvm-pgo-generate bots are functioning ' + 'normally. Their status can be found at ' + f'{monitor_pgo_profiles.PGO_BUILDBOT_LINK}.', + )), + ) - def test_compose_complaint_email_with_two_out_of_date(self): + def test_compose_complaint_with_two_out_of_date(self): profdata_info_1 = monitor_pgo_profiles.ProfdataInfo( date=datetime.datetime(2020, 1, 2, 3, 4, 5), location='gs://somewhere', @@ -54,28 +49,25 @@ class Test(unittest.TestCase): date=datetime.datetime(2020, 3, 2, 1, 4, 5), location='gs://somewhere-else', ) - result = monitor_pgo_profiles.compose_complaint_email([ + result = monitor_pgo_profiles.compose_complaint([ ('some_arch', profdata_info_1), ('some_other_arch', profdata_info_2), ]) - self.assertEqual(result, ('2 llvm profiles are out of date', [ - 'out-of-date profiles:', - tiny_render.UnorderedList([ - f'some_arch (most recent profile was from {profdata_info_1.date} ' + self.assertEqual( + result, + '\n'.join(( + '2 profiles are out of date:', + f'- some_arch (most recent profile was from {profdata_info_1.date} ' f'at {profdata_info_1.location!r})', - f'some_other_arch (most recent profile was from ' - f'{profdata_info_2.date} at {profdata_info_2.location!r})' - ]), - tiny_render.line_break, - tiny_render.line_break, - 'PTAL to see if the llvm-pgo-generate bots are functioning normally. ' - 'Their status can be found at ', - tiny_render.Link( - href=monitor_pgo_profiles.PGO_BUILDBOT_LINK, - inner=monitor_pgo_profiles.PGO_BUILDBOT_LINK, - ), - '.', - ])) + f'- some_other_arch (most recent profile was from ' + f'{profdata_info_2.date} at {profdata_info_2.location!r})', + '', + '', + 'PTAL to see if the llvm-pgo-generate bots are functioning ' + 'normally. Their status can be found at ' + f'{monitor_pgo_profiles.PGO_BUILDBOT_LINK}.', + )), + ) @unittest.mock.patch.object(subprocess, 'run') def test_fetching_profdata_functions(self, subprocess_run_mock): diff --git a/rust_tools/rust_watch.py b/rust_tools/rust_watch.py index 66df7c8a..db6ae71b 100755 --- a/rust_tools/rust_watch.py +++ b/rust_tools/rust_watch.py @@ -19,10 +19,9 @@ import shutil import subprocess import sys import time -from typing import Any, Dict, Iterable, List, Optional, Tuple, NamedTuple +from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple -from cros_utils import email_sender -from cros_utils import tiny_render +from cros_utils import bugs, email_sender, tiny_render def gentoo_sha_to_link(sha: str) -> str: @@ -162,7 +161,7 @@ def get_new_gentoo_commits(git_dir: pathlib.Path, 'git', 'log', '--format=%H %s', - f'{most_recent_sha}..origin/master', + f'{most_recent_sha}..origin/master', # nocheck '--', 'dev-lang/rust', ], @@ -223,43 +222,63 @@ def atomically_write_state(state_file: pathlib.Path, state: State) -> None: temp_file.rename(state_file) -def maybe_compose_email(old_state: State, newest_release: RustReleaseVersion, - new_gentoo_commits: List[GitCommit] - ) -> Optional[Tuple[str, List[tiny_render.Piece]]]: +def file_bug(title: str, body: str) -> None: + """Files a bug against gbiv@ with the given title/body.""" + bugs.CreateNewBug( + bugs.WellKnownComponents.CrOSToolchainPublic, + title, + body, + # To either take or reassign depending on the rotation. + assignee='gbiv@google.com', + ) + + +def maybe_compose_bug( + old_state: State, + newest_release: RustReleaseVersion, +) -> Optional[Tuple[str, str]]: + """Creates a bug to file about the new release, if doing is desired.""" + if newest_release == old_state.last_seen_release: + return None + + title = f'New rustc release detected: v{newest_release}' + body = ('A new release has been detected; we should probably roll to it. ' + "Please see go/crostc-rust-rotation for who's turn it is.") + return title, body + + +def maybe_compose_email( + new_gentoo_commits: List[GitCommit] +) -> Optional[Tuple[str, List[tiny_render.Piece]]]: """Creates an email given our new state, if doing so is appropriate.""" + if not new_gentoo_commits: + return None + subject_pieces = [] body_pieces = [] - if newest_release > old_state.last_seen_release: - subject_pieces.append('new rustc release detected') - body_pieces.append(f'Rustc tag for v{newest_release} was found.') + # Separate the sections a bit for prettier output. + if body_pieces: + body_pieces += [tiny_render.line_break, tiny_render.line_break] - if new_gentoo_commits: - # Separate the sections a bit for prettier output. - if body_pieces: - body_pieces += [tiny_render.line_break, tiny_render.line_break] + if len(new_gentoo_commits) == 1: + subject_pieces.append('new rust ebuild commit detected') + body_pieces.append('commit:') + else: + subject_pieces.append('new rust ebuild commits detected') + body_pieces.append('commits (newest first):') + + commit_lines = [] + for commit in new_gentoo_commits: + commit_lines.append([ + tiny_render.Link( + gentoo_sha_to_link(commit.sha), + commit.sha[:12], + ), + f': {commit.subject}', + ]) - if len(new_gentoo_commits) == 1: - subject_pieces.append('new rust ebuild commit detected') - body_pieces.append('commit:') - else: - subject_pieces.append('new rust ebuild commits detected') - body_pieces.append('commits (newest first):') - - commit_lines = [] - for commit in new_gentoo_commits: - commit_lines.append([ - tiny_render.Link( - gentoo_sha_to_link(commit.sha), - commit.sha[:12], - ), - f': {commit.subject}', - ]) - - body_pieces.append(tiny_render.UnorderedList(commit_lines)) - - if not subject_pieces: - return None + body_pieces.append(tiny_render.UnorderedList(commit_lines)) subject = '[rust-watch] ' + '; '.join(subject_pieces) return subject, body_pieces @@ -269,11 +288,14 @@ def main(argv: List[str]) -> None: logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '--state_dir', required=True, help='Directory to store state in.') - parser.add_argument( - '--skip_email', action='store_true', help="Don't send an email.") + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--state_dir', + required=True, + help='Directory to store state in.') + parser.add_argument('--skip_side_effects', + action='store_true', + help="Don't send an email or file a bug.") parser.add_argument( '--skip_state_update', action='store_true', @@ -316,14 +338,25 @@ def main(argv: List[str]) -> None: prior_state.last_gentoo_sha) logging.info('New commits: %r', new_commits) - maybe_email = maybe_compose_email(prior_state, most_recent_release, - new_commits) + maybe_bug = maybe_compose_bug(prior_state, most_recent_release) + maybe_email = maybe_compose_email(new_commits) + + if maybe_bug is None: + logging.info('No bug to file') + else: + title, body = maybe_bug + if opts.skip_side_effects: + logging.info('Skipping sending bug with title %r and contents\n%s', + title, body) + else: + logging.info('Writing new bug') + file_bug(title, body) if maybe_email is None: - logging.info('No updates to send') + logging.info('No email to send') else: title, body = maybe_email - if opts.skip_email: + if opts.skip_side_effects: logging.info('Skipping sending email with title %r and contents\n%s', title, tiny_render.render_html_pieces(body)) else: @@ -334,8 +367,8 @@ def main(argv: List[str]) -> None: logging.info('Skipping state update, as requested') return - newest_sha = ( - new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha) + newest_sha = (new_commits[-1].sha + if new_commits else prior_state.last_gentoo_sha) atomically_write_state( state_file, State( diff --git a/rust_tools/rust_watch_test.py b/rust_tools/rust_watch_test.py index a00f3ddc..30bacbb9 100755 --- a/rust_tools/rust_watch_test.py +++ b/rust_tools/rust_watch_test.py @@ -13,16 +13,15 @@ import time import unittest import unittest.mock -import rust_watch from cros_utils import tiny_render +import rust_watch + class Test(unittest.TestCase): """Tests.""" - def _silence_logs(self): """Silences all log output until the end of the current test.""" - def should_log(_record): return 0 @@ -38,8 +37,8 @@ class Test(unittest.TestCase): def test_release_version_json_round_trips(self): ver = rust_watch.RustReleaseVersion(1, 2, 3) - self.assertEqual( - rust_watch.RustReleaseVersion.from_json(ver.to_json()), ver) + self.assertEqual(rust_watch.RustReleaseVersion.from_json(ver.to_json()), + ver) def test_state_json_round_trips(self): state = rust_watch.State( @@ -96,34 +95,14 @@ class Test(unittest.TestCase): rust_watch.GitCommit('abc123', 'newer commit'), ]) - def test_compose_email_on_a_new_release(self): - new_release = rust_watch.maybe_compose_email( - old_state=rust_watch.State( - last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), - last_gentoo_sha='', - ), - newest_release=rust_watch.RustReleaseVersion(1, 1, 0), - new_gentoo_commits=[], - ) - - self.assertEqual(new_release, ('[rust-watch] new rustc release detected', - ['Rustc tag for v1.1.0 was found.'])) - def test_compose_email_on_a_new_gentoo_commit(self): sha_a = 'a' * 40 - new_commit = rust_watch.maybe_compose_email( - old_state=rust_watch.State( - last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), - last_gentoo_sha='', + new_commit = rust_watch.maybe_compose_email(new_gentoo_commits=[ + rust_watch.GitCommit( + sha=sha_a, + subject='summary_a', ), - newest_release=rust_watch.RustReleaseVersion(1, 0, 0), - new_gentoo_commits=[ - rust_watch.GitCommit( - sha=sha_a, - subject='summary_a', - ), - ], - ) + ], ) self.assertEqual(new_commit, ('[rust-watch] new rust ebuild commit detected', [ @@ -139,60 +118,48 @@ class Test(unittest.TestCase): ]) ])) - def test_compose_email_on_multiple_events(self): - sha_a = 'a' * 40 - new_commit_and_release = rust_watch.maybe_compose_email( + def test_compose_email_composes_nothing_when_no_new_updates_exist(self): + self.assertIsNone(rust_watch.maybe_compose_email(new_gentoo_commits=())) + + def test_compose_bug_creates_bugs_on_new_versions(self): + title, body = rust_watch.maybe_compose_bug( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(1, 0, 1), + ) + self.assertEqual(title, 'New rustc release detected: v1.0.1') + self.assertTrue(body.startswith('A new release has been detected;')) + + title, body = rust_watch.maybe_compose_bug( old_state=rust_watch.State( last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), last_gentoo_sha='', ), newest_release=rust_watch.RustReleaseVersion(1, 1, 0), - new_gentoo_commits=[ - rust_watch.GitCommit( - sha=sha_a, - subject='summary_a', - ), - ], ) + self.assertEqual(title, 'New rustc release detected: v1.1.0') + self.assertTrue(body.startswith('A new release has been detected;')) - self.assertEqual( - new_commit_and_release, - ('[rust-watch] new rustc release detected; new rust ebuild commit ' - 'detected', [ - 'Rustc tag for v1.1.0 was found.', - tiny_render.line_break, - tiny_render.line_break, - 'commit:', - tiny_render.UnorderedList([ - [ - tiny_render.Link( - rust_watch.gentoo_sha_to_link(sha_a), - sha_a[:12], - ), - ': summary_a', - ], - ]), - ])) + title, body = rust_watch.maybe_compose_bug( + old_state=rust_watch.State( + last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), + last_gentoo_sha='', + ), + newest_release=rust_watch.RustReleaseVersion(2, 0, 0), + ) + self.assertEqual(title, 'New rustc release detected: v2.0.0') + self.assertTrue(body.startswith('A new release has been detected;')) - def test_compose_email_composes_nothing_when_no_new_updates_exist(self): + def test_compose_bug_does_nothing_when_no_new_updates_exist(self): self.assertIsNone( - rust_watch.maybe_compose_email( + rust_watch.maybe_compose_bug( old_state=rust_watch.State( last_seen_release=rust_watch.RustReleaseVersion(1, 0, 0), last_gentoo_sha='', ), newest_release=rust_watch.RustReleaseVersion(1, 0, 0), - new_gentoo_commits=[], - )) - - self.assertIsNone( - rust_watch.maybe_compose_email( - old_state=rust_watch.State( - last_seen_release=rust_watch.RustReleaseVersion(1, 1, 0), - last_gentoo_sha='', - ), - newest_release=rust_watch.RustReleaseVersion(1, 0, 0), - new_gentoo_commits=[], )) diff --git a/setup_chromeos.py b/setup_chromeos.py deleted file mode 100755 index 07c7530d..00000000 --- a/setup_chromeos.py +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2010 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Script to checkout the ChromeOS source. - -This script sets up the ChromeOS source in the given directory, matching a -particular release of ChromeOS. -""" - -from __future__ import print_function - -__author__ = 'raymes@google.com (Raymes Khoury)' - -from datetime import datetime - -import argparse -import os -import pickle -import sys -import tempfile -import time -from cros_utils import command_executer -from cros_utils import logger -from cros_utils import manifest_versions - -GCLIENT_FILE = """solutions = [ - { "name" : "CHROME_DEPS", - "url" : - "svn://svn.chromium.org/chrome-internal/trunk/tools/buildspec/releases/%s", - "custom_deps" : { - "src/third_party/WebKit/LayoutTests": None, - "src-pdf": None, - "src/pdf": None, - }, - "safesync_url": "", - }, -] -""" - -# List of stable versions used for common team image -# Sheriff must update this list when a new common version becomes available -COMMON_VERSIONS = '/home/mobiletc-prebuild/common_images/common_list.txt' - - -def Usage(parser): - parser.print_help() - sys.exit(0) - - -# Get version spec file, either from "paladin" or "buildspec" directory. -def GetVersionSpecFile(version, versions_git): - temp = tempfile.mkdtemp() - commands = ['cd {0}'.format(temp), \ - 'git clone {0} versions'.format(versions_git)] - cmd_executer = command_executer.GetCommandExecuter() - ret = cmd_executer.RunCommands(commands) - err_msg = None - if ret: - err_msg = 'Failed to checkout versions_git - {0}'.format(versions_git) - ret = None - else: - v, m = version.split('.', 1) - paladin_spec = 'paladin/buildspecs/{0}/{1}.xml'.format(v, m) - generic_spec = 'buildspecs/{0}/{1}.xml'.format(v, m) - paladin_path = '{0}/versions/{1}'.format(temp, paladin_spec) - generic_path = '{0}/versions/{1}'.format(temp, generic_spec) - if os.path.exists(paladin_path): - ret = paladin_spec - elif os.path.exists(generic_path): - ret = generic_spec - else: - err_msg = 'No spec found for version {0}'.format(version) - ret = None - # Fall through to clean up. - - commands = ['rm -rf {0}'.format(temp)] - cmd_executer.RunCommands(commands) - if err_msg: - logger.GetLogger().LogFatal(err_msg) - return ret - - -def TimeToCommonVersion(timestamp): - """Convert timestamp to common image version.""" - tdt = datetime.fromtimestamp(float(timestamp)) - with open(COMMON_VERSIONS, 'r', encoding='utf-8') as f: - common_list = pickle.load(f) - for sv in common_list: - sdt = datetime.strptime(sv['date'], '%Y-%m-%d %H:%M:%S.%f') - if tdt >= sdt: - return '%s.%s' % (sv['chrome_major_version'], sv['chromeos_version']) - # should never reach here - logger.GetLogger().LogFatal('No common version for timestamp') - return None - - -def Main(argv): - """Checkout the ChromeOS source.""" - parser = argparse.ArgumentParser() - parser.add_argument( - '--dir', - dest='directory', - help='Target directory for ChromeOS installation.') - parser.add_argument( - '--version', - dest='version', - default='latest_lkgm', - help="""ChromeOS version. Can be: -(1) A release version in the format: 'X.X.X.X' -(2) 'top' for top of trunk -(3) 'latest_lkgm' for the latest lkgm version -(4) 'lkgm' for the lkgm release before timestamp -(5) 'latest_common' for the latest team common stable version -(6) 'common' for the team common stable version before timestamp -Default is 'latest_lkgm'.""") - parser.add_argument( - '--timestamp', - dest='timestamp', - default=None, - help='Timestamps in epoch format. It will check out the' - 'latest LKGM or the latest COMMON version of ChromeOS' - ' before the timestamp. Use in combination with' - ' --version=latest or --version=common. Use ' - '"date -d <date string> +%s" to find epoch time') - parser.add_argument( - '--minilayout', - dest='minilayout', - default=False, - action='store_true', - help='Whether to checkout the minilayout (smaller ' - 'checkout).') - parser.add_argument( - '--jobs', '-j', dest='jobs', help='Number of repo sync threads to use.') - parser.add_argument( - '--public', - '-p', - dest='public', - default=False, - action='store_true', - help='Use the public checkout instead of the private ' - 'one.') - - options = parser.parse_args(argv) - - if not options.version: - parser.print_help() - logger.GetLogger().LogFatal('No version specified.') - else: - version = options.version.strip() - - if not options.timestamp: - timestamp = '' - else: - timestamp = options.timestamp.strip() - if version not in ('lkgm', 'common'): - parser.print_help() - logger.GetLogger().LogFatal('timestamp option only applies for ' - 'versions "lkgm" or "common"') - - if not options.directory: - parser.print_help() - logger.GetLogger().LogFatal('No directory specified.') - - directory = options.directory.strip() - - if options.public: - manifest_repo = 'https://chromium.googlesource.com/chromiumos/manifest.git' - versions_repo = ('https://chromium.googlesource.com/' - 'chromiumos/manifest-versions.git') - else: - manifest_repo = ('https://chrome-internal.googlesource.com/chromeos/' - 'manifest-internal.git') - versions_repo = ('https://chrome-internal.googlesource.com/chromeos/' - 'manifest-versions.git') - - if version == 'top': - init = 'repo init -u %s' % manifest_repo - elif version == 'latest_lkgm': - manifests = manifest_versions.ManifestVersions() - version = manifests.TimeToVersionChromeOS(time.mktime(time.gmtime())) - version, manifest = version.split('.', 1) - logger.GetLogger().LogOutput( - 'found version %s.%s for latest LKGM' % (version, manifest)) - init = ('repo init -u %s -m paladin/buildspecs/%s/%s.xml' % - (versions_repo, version, manifest)) - del manifests - elif version == 'lkgm': - if not timestamp: - parser.print_help() - logger.GetLogger().LogFatal('No timestamp specified for version=lkgm') - manifests = manifest_versions.ManifestVersions() - version = manifests.TimeToVersion(timestamp) - version, manifest = version.split('.', 1) - logger.GetLogger().LogOutput('found version %s.%s for LKGM at timestamp %s' - % (version, manifest, timestamp)) - init = ('repo init -u %s -m paladin/buildspecs/%s/%s.xml' % - (versions_repo, version, manifest)) - del manifests - elif version == 'latest_common': - version = TimeToCommonVersion(time.mktime(time.gmtime())) - version, manifest = version.split('.', 1) - logger.GetLogger().LogOutput( - 'found version %s.%s for latest Common image' % (version, manifest)) - init = ('repo init -u %s -m buildspecs/%s/%s.xml' % (versions_repo, version, - manifest)) - elif version == 'common': - if not timestamp: - parser.print_help() - logger.GetLogger().LogFatal('No timestamp specified for version=lkgm') - version = TimeToCommonVersion(timestamp) - version, manifest = version.split('.', 1) - logger.GetLogger().LogOutput( - 'found version %s.%s for latest common image ' - 'at timestamp %s' % (version, manifest, timestamp)) - init = ('repo init -u %s -m buildspecs/%s/%s.xml' % (versions_repo, version, - manifest)) - else: - # user specified a specific version number - version_spec_file = GetVersionSpecFile(version, versions_repo) - if not version_spec_file: - return 1 - init = 'repo init -u %s -m %s' % (versions_repo, version_spec_file) - - if options.minilayout: - init += ' -g minilayout' - - init += ' --repo-url=https://chromium.googlesource.com/external/repo.git' - - # crosbug#31837 - "Sources need to be world-readable to properly - # function inside the chroot" - sync = 'umask 022 && repo sync' - if options.jobs: - sync += ' -j %s' % options.jobs - - commands = ['mkdir -p %s' % directory, 'cd %s' % directory, init, sync] - cmd_executer = command_executer.GetCommandExecuter() - ret = cmd_executer.RunCommands(commands) - if ret: - return ret - - return cmd_executer.RunCommand( - 'git ls-remote ' - 'https://chrome-internal.googlesource.com/chrome/src-internal.git ' - 'refs/HEAD > /dev/null') - - -if __name__ == '__main__': - retval = Main(sys.argv[1:]) - sys.exit(retval) diff --git a/unblocked_terms.txt b/unblocked_terms.txt deleted file mode 100644 index c961e667..00000000 --- a/unblocked_terms.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Code uses this in many cases to refer to git branches. -master - -# A few paths in AOSP contain this term. -native |