diff options
author | Fabian Meumertzheim <meumertzheim@code-intelligence.com> | 2023-09-20 10:17:29 +0200 |
---|---|---|
committer | Fabian Meumertzheim <fabian@meumertzhe.im> | 2023-09-28 15:45:58 +0200 |
commit | f90c39d288a98e9e8c60ebdaa2a1bd9a93b96a2c (patch) | |
tree | 0019f04ff4782249a4bef0e3e65937141245bc2e | |
parent | a9f525530d60270bd4f9046da66d7f3eb3c9feb8 (diff) | |
download | jazzer-api-f90c39d288a98e9e8c60ebdaa2a1bd9a93b96a2c.tar.gz |
mutation: Use `hasFixedSize` in `sizeInClosedRange`
The benchmarks in `//tests/benchmarks` show that biasing the size of
subsets of collections of primitives chosen by the mutator to be small
results in much worse performance than a comparable unstructured fuzz
test.
Before this change, 11 out of 15 runs time out with no run limit, the
other ones result in:
```
{
"values": [
11143,
28128,
581194,
4229980
],
"minimum": 11143,
"maximum": 4229980,
"average": 1212611.25,
"median": 304661
}
```
After this change, all runs pass within a limit of 35,000 runs:
```
{
"values": [
887,
1557,
1889,
2557,
3023,
3346,
3517,
6075,
6613,
7991,
9578,
10850,
15583,
23638,
31046
],
"minimum": 887,
"maximum": 31046,
"average": 8543.333333333334,
"median": 6075
}
```
ExperimentalMutatorComplexProtoFuzzer now takes more runs on Linux, but
still less than on other platforms, which seems to indicate that the
Linux seed just happened to be a lucky choice.
-rw-r--r-- | .github/workflows/run-all-tests.yml | 2 | ||||
-rw-r--r-- | src/main/java/com/code_intelligence/jazzer/mutation/engine/SeededPseudoRandom.java | 6 | ||||
-rw-r--r-- | tests/BUILD.bazel | 8 | ||||
-rw-r--r-- | tests/benchmarks/BUILD.bazel | 2 |
4 files changed, 9 insertions, 9 deletions
diff --git a/.github/workflows/run-all-tests.yml b/.github/workflows/run-all-tests.yml index 93054718..07203e67 100644 --- a/.github/workflows/run-all-tests.yml +++ b/.github/workflows/run-all-tests.yml @@ -42,7 +42,7 @@ jobs: extra_bazel_args: "--jvmopt=-Djava.security.manager=allow" - os: ubuntu-20.04 arch: "linux" - bazel_args: "//launcher/android:jazzer_android" + bazel_args: "//launcher/android:jazzer_android //tests/benchmarks" cache: "/home/runner/.cache/bazel-disk" - os: macos-12 arch: "macos-x86_64" diff --git a/src/main/java/com/code_intelligence/jazzer/mutation/engine/SeededPseudoRandom.java b/src/main/java/com/code_intelligence/jazzer/mutation/engine/SeededPseudoRandom.java index fb9388a7..893cb9d3 100644 --- a/src/main/java/com/code_intelligence/jazzer/mutation/engine/SeededPseudoRandom.java +++ b/src/main/java/com/code_intelligence/jazzer/mutation/engine/SeededPseudoRandom.java @@ -252,7 +252,11 @@ public final class SeededPseudoRandom implements PseudoRandom { @Override public int sizeInClosedRange( int lowerInclusive, int upperInclusive, boolean elementsHaveFixedSize) { - return lowerInclusive + closedRangeBiasedTowardsSmall(upperInclusive - lowerInclusive); + if (elementsHaveFixedSize) { + return closedRange(lowerInclusive, upperInclusive); + } else { + return lowerInclusive + closedRangeBiasedTowardsSmall(upperInclusive - lowerInclusive); + } } private static double zipf_h(double x) { diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index c92c0d16..1d65e1a6 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -523,13 +523,9 @@ java_fuzz_target_test( "--experimental_mutator", "--instrumentation_includes=com.example.**", "--custom_hook_includes=com.example.**", - ] + select({ # Limit runs to catch regressions in mutator efficiency and speed up test runs. - "@platforms//os:linux": ["-runs=400000"], - # TODO: Investigate why this test takes far more runs on macOS, with Windows also being - # significantly worse than Linux. - "//conditions:default": ["-runs=1200000"], - }), + "-runs=1200000", + ], target_class = "com.example.ExperimentalMutatorComplexProtoFuzzer", verify_crash_reproducer = False, deps = [ diff --git a/tests/benchmarks/BUILD.bazel b/tests/benchmarks/BUILD.bazel index bcc48d8a..d7b47d84 100644 --- a/tests/benchmarks/BUILD.bazel +++ b/tests/benchmarks/BUILD.bazel @@ -32,7 +32,7 @@ fuzzer_benchmark( "--experimental_mutator", "-use_value_profile=1", ], - max_runs = 55000, + max_runs = 35000, num_seeds = 15, target_class = "com.example.StructuredMutatorMazeFuzzer", deps = [ |