aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFabian Meumertzheim <meumertzheim@code-intelligence.com>2023-09-20 10:17:29 +0200
committerFabian Meumertzheim <fabian@meumertzhe.im>2023-09-28 15:45:58 +0200
commitf90c39d288a98e9e8c60ebdaa2a1bd9a93b96a2c (patch)
tree0019f04ff4782249a4bef0e3e65937141245bc2e
parenta9f525530d60270bd4f9046da66d7f3eb3c9feb8 (diff)
downloadjazzer-api-f90c39d288a98e9e8c60ebdaa2a1bd9a93b96a2c.tar.gz
mutation: Use `hasFixedSize` in `sizeInClosedRange`
The benchmarks in `//tests/benchmarks` show that biasing the size of subsets of collections of primitives chosen by the mutator to be small results in much worse performance than a comparable unstructured fuzz test. Before this change, 11 out of 15 runs time out with no run limit, the other ones result in: ``` { "values": [ 11143, 28128, 581194, 4229980 ], "minimum": 11143, "maximum": 4229980, "average": 1212611.25, "median": 304661 } ``` After this change, all runs pass within a limit of 35,000 runs: ``` { "values": [ 887, 1557, 1889, 2557, 3023, 3346, 3517, 6075, 6613, 7991, 9578, 10850, 15583, 23638, 31046 ], "minimum": 887, "maximum": 31046, "average": 8543.333333333334, "median": 6075 } ``` ExperimentalMutatorComplexProtoFuzzer now takes more runs on Linux, but still less than on other platforms, which seems to indicate that the Linux seed just happened to be a lucky choice.
-rw-r--r--.github/workflows/run-all-tests.yml2
-rw-r--r--src/main/java/com/code_intelligence/jazzer/mutation/engine/SeededPseudoRandom.java6
-rw-r--r--tests/BUILD.bazel8
-rw-r--r--tests/benchmarks/BUILD.bazel2
4 files changed, 9 insertions, 9 deletions
diff --git a/.github/workflows/run-all-tests.yml b/.github/workflows/run-all-tests.yml
index 93054718..07203e67 100644
--- a/.github/workflows/run-all-tests.yml
+++ b/.github/workflows/run-all-tests.yml
@@ -42,7 +42,7 @@ jobs:
extra_bazel_args: "--jvmopt=-Djava.security.manager=allow"
- os: ubuntu-20.04
arch: "linux"
- bazel_args: "//launcher/android:jazzer_android"
+ bazel_args: "//launcher/android:jazzer_android //tests/benchmarks"
cache: "/home/runner/.cache/bazel-disk"
- os: macos-12
arch: "macos-x86_64"
diff --git a/src/main/java/com/code_intelligence/jazzer/mutation/engine/SeededPseudoRandom.java b/src/main/java/com/code_intelligence/jazzer/mutation/engine/SeededPseudoRandom.java
index fb9388a7..893cb9d3 100644
--- a/src/main/java/com/code_intelligence/jazzer/mutation/engine/SeededPseudoRandom.java
+++ b/src/main/java/com/code_intelligence/jazzer/mutation/engine/SeededPseudoRandom.java
@@ -252,7 +252,11 @@ public final class SeededPseudoRandom implements PseudoRandom {
@Override
public int sizeInClosedRange(
int lowerInclusive, int upperInclusive, boolean elementsHaveFixedSize) {
- return lowerInclusive + closedRangeBiasedTowardsSmall(upperInclusive - lowerInclusive);
+ if (elementsHaveFixedSize) {
+ return closedRange(lowerInclusive, upperInclusive);
+ } else {
+ return lowerInclusive + closedRangeBiasedTowardsSmall(upperInclusive - lowerInclusive);
+ }
}
private static double zipf_h(double x) {
diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel
index c92c0d16..1d65e1a6 100644
--- a/tests/BUILD.bazel
+++ b/tests/BUILD.bazel
@@ -523,13 +523,9 @@ java_fuzz_target_test(
"--experimental_mutator",
"--instrumentation_includes=com.example.**",
"--custom_hook_includes=com.example.**",
- ] + select({
# Limit runs to catch regressions in mutator efficiency and speed up test runs.
- "@platforms//os:linux": ["-runs=400000"],
- # TODO: Investigate why this test takes far more runs on macOS, with Windows also being
- # significantly worse than Linux.
- "//conditions:default": ["-runs=1200000"],
- }),
+ "-runs=1200000",
+ ],
target_class = "com.example.ExperimentalMutatorComplexProtoFuzzer",
verify_crash_reproducer = False,
deps = [
diff --git a/tests/benchmarks/BUILD.bazel b/tests/benchmarks/BUILD.bazel
index bcc48d8a..d7b47d84 100644
--- a/tests/benchmarks/BUILD.bazel
+++ b/tests/benchmarks/BUILD.bazel
@@ -32,7 +32,7 @@ fuzzer_benchmark(
"--experimental_mutator",
"-use_value_profile=1",
],
- max_runs = 55000,
+ max_runs = 35000,
num_seeds = 15,
target_class = "com.example.StructuredMutatorMazeFuzzer",
deps = [