junit: Honor instrumentation settings when run from the driver

The `Driver` accesses `Opt` settings and thus locked in their values
before the JUnit integration, specifically `AgentConfigurator`, had a
chance to update the instrumentation filter.

This is worked around by evaluating the instrumentation settings lazily,
with a more conceptual fix coming with the ongoing config overhaul.

7 files changed, 147 insertions, 40 deletions

diff --git a/examples/junit/src/test/java/com/example/BUILD.bazel b/examples/junit/src/test/java/com/example/BUILD.bazel
index 841db001..8bbdcecd 100644
--- a/examples/junit/src/test/java/com/example/BUILD.bazel
+++ b/examples/junit/src/test/java/com/example/BUILD.bazel
@@ -25,8 +25,6 @@ java_fuzz_target_test(
     srcs = ["ValidFuzzTests.java"],
     allowed_findings = ["com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium"],
     fuzzer_args = [
-        "--instrumentation_includes=com.example.**",
-        "--custom_hook_includes=com.example.**",
         "-runs=0",
     ],
     target_class = "com.example.ValidFuzzTests",
@@ -47,8 +45,6 @@ java_fuzz_target_test(
     srcs = ["ByteFuzzTest.java"],
     allowed_findings = ["org.opentest4j.AssertionFailedError"],
     fuzzer_args = [
-        "--instrumentation_includes=com.example.**",
-        "--custom_hook_includes=com.example.**",
         "-runs=0",
     ],
     target_class = "com.example.ByteFuzzTest",
@@ -69,8 +65,6 @@ java_fuzz_target_test(
     srcs = ["LifecycleFuzzTest.java"],
     allowed_findings = ["java.io.IOException"],
     fuzzer_args = [
-        "--instrumentation_includes=com.example.**",
-        "--custom_hook_includes=com.example.**",
         "-runs=0",
     ],
     target_class = "com.example.LifecycleFuzzTest",
@@ -91,8 +85,6 @@ java_fuzz_target_test(
     allowed_findings = ["java.lang.IllegalArgumentException"],
     expect_crash = False,
     fuzzer_args = [
-        "--instrumentation_includes=com.example.**",
-        "--custom_hook_includes=com.example.**",
         "--keep_going=3",
         "-runs=10",
     ],
@@ -112,8 +104,6 @@ java_fuzz_target_test(
     srcs = ["CommandLineFuzzTest.java"],
     allowed_findings = ["java.lang.Error"],
     fuzzer_args = [
-        "--instrumentation_includes=com.example.**",
-        "--custom_hook_includes=com.example.**",
         # Ignore the first two findings.
         "--ignore=d5e250a5298b81e6,d86371e6d41739ec",
     ],
@@ -155,8 +145,6 @@ java_fuzz_target_test(
     name = "AutofuzzLifecycleFuzzTest",
     srcs = ["AutofuzzLifecycleFuzzTest.java"],
     fuzzer_args = [
-        "--instrumentation_includes=com.example.**",
-        "--custom_hook_includes=com.example.**",
         "-runs=0",
     ],
     target_class = "com.example.AutofuzzLifecycleFuzzTest",
@@ -181,8 +169,6 @@ java_fuzz_target_test(
         "JAZZER_FUZZ": "1",
     },
     fuzzer_args = [
-        "--instrumentation_includes=com.example.**",
-        "--custom_hook_includes=com.example.**",
         "--experimental_mutator",
         "$(rlocationpaths //examples/junit/src/test/resources:MutatorFuzzTestInputs)",
     ],
@@ -205,8 +191,6 @@ java_fuzz_target_test(
     allowed_findings = ["java.lang.Error"],
     env = {"JAZZER_FUZZ": "1"},
     fuzzer_args = [
-        "--instrumentation_includes=com.example.**",
-        "--custom_hook_includes=com.example.**",
         "--experimental_mutator",
     ],
     target_class = "com.example.JavaSeedFuzzTest",
@@ -227,10 +211,6 @@ java_fuzz_target_test(
     srcs = ["JavaBinarySeedFuzzTest.java"],
     allowed_findings = ["java.lang.Error"],
     env = {"JAZZER_FUZZ": "1"},
-    fuzzer_args = [
-        "--instrumentation_includes=com.example.**",
-        "--custom_hook_includes=com.example.**",
-    ],
     target_class = "com.example.JavaBinarySeedFuzzTest",
     verify_crash_reproducer = False,
     runtime_deps = [
diff --git a/src/main/java/com/code_intelligence/jazzer/agent/Agent.kt b/src/main/java/com/code_intelligence/jazzer/agent/Agent.kt
index 6e89aafc..9bcd744f 100644
--- a/src/main/java/com/code_intelligence/jazzer/agent/Agent.kt
+++ b/src/main/java/com/code_intelligence/jazzer/agent/Agent.kt
@@ -37,10 +37,10 @@ fun installInternal(
     instrumentation: Instrumentation,
     userHookNames: List<String> = findManifestCustomHookNames() + Opt.customHooks,
     disabledHookNames: List<String> = Opt.disabledHooks,
-    instrumentationIncludes: List<String> = Opt.instrumentationIncludes,
-    instrumentationExcludes: List<String> = Opt.instrumentationExcludes,
-    customHookIncludes: List<String> = Opt.customHookIncludes,
-    customHookExcludes: List<String> = Opt.customHookExcludes,
+    instrumentationIncludes: List<String> = Opt.instrumentationIncludes.get(),
+    instrumentationExcludes: List<String> = Opt.instrumentationExcludes.get(),
+    customHookIncludes: List<String> = Opt.customHookIncludes.get(),
+    customHookExcludes: List<String> = Opt.customHookExcludes.get(),
     trace: List<String> = Opt.trace,
     idSyncFile: String? = Opt.idSyncFile,
     dumpClassesDir: String = Opt.dumpClassesDir,
diff --git a/src/main/java/com/code_intelligence/jazzer/driver/Opt.java b/src/main/java/com/code_intelligence/jazzer/driver/Opt.java
index 0f10d212..f06f81eb 100644
--- a/src/main/java/com/code_intelligence/jazzer/driver/Opt.java
+++ b/src/main/java/com/code_intelligence/jazzer/driver/Opt.java
@@ -19,6 +19,7 @@ package com.code_intelligence.jazzer.driver;
 import static com.code_intelligence.jazzer.Constants.JAZZER_VERSION;
 import static com.code_intelligence.jazzer.driver.OptParser.boolSetting;
 import static com.code_intelligence.jazzer.driver.OptParser.ignoreSetting;
+import static com.code_intelligence.jazzer.driver.OptParser.lazyStringListSetting;
 import static com.code_intelligence.jazzer.driver.OptParser.stringListSetting;
 import static com.code_intelligence.jazzer.driver.OptParser.stringSetting;
 import static com.code_intelligence.jazzer.driver.OptParser.uint64Setting;
@@ -32,6 +33,7 @@ import static java.util.stream.Stream.concat;
 import com.code_intelligence.jazzer.utils.Log;
 import java.util.List;
 import java.util.Set;
+import java.util.function.Supplier;
 import java.util.stream.Stream;
 
 /**
@@ -84,10 +86,6 @@ public final class Opt {
       "Path to write a JaCoCo .exec file to when the fuzzer exits (if non-empty)");
   public static final String coverageReport = stringSetting("coverage_report", "",
       "Path to write a human-readable coverage report to when the fuzzer exits (if non-empty)");
-  public static final List<String> customHookIncludes = stringListSetting("custom_hook_includes",
-      "Glob patterns matching names of classes to instrument with hooks (custom and built-in)");
-  public static final List<String> customHookExcludes = stringListSetting("custom_hook_excludes",
-      "Glob patterns matching names of classes that should not be instrumented with hooks (custom and built-in)");
   public static final List<String> customHooks =
       stringListSetting("custom_hooks", "Names of classes to load custom hooks from");
   public static final List<String> disabledHooks = stringListSetting("disabled_hooks",
@@ -105,12 +103,6 @@ public final class Opt {
   public static final boolean hooks = boolSetting(
       "hooks", true, "Apply fuzzing instrumentation (use 'trace' for finer-grained control)");
   public static final String idSyncFile = stringSetting("id_sync_file", null, null);
-  public static final List<String> instrumentationIncludes =
-      stringListSetting("instrumentation_includes",
-          "Glob patterns matching names of classes to instrument for fuzzing");
-  public static final List<String> instrumentationExcludes =
-      stringListSetting("instrumentation_excludes",
-          "Glob patterns matching names of classes that should not be instrumented for fuzzing");
   public static final List<String> additionalClassesExcludes =
       stringListSetting("additional_classes_excludes",
           "Glob patterns matching names of classes from Java that are not in your jar file, "
@@ -132,6 +124,24 @@ public final class Opt {
   public static final List<String> trace = stringListSetting("trace",
       "Types of instrumentation to apply: cmp, cov, div, gep (disabled by default), indir, native");
 
+  // When Jazzer is executed from the command line, these settings are potentially modified by
+  // JUnit's AgentConfigurator after the Driver has initialized Opt, which would result in stale
+  // values being read if the settings weren't evaluated lazily.
+  // TODO: Look into making all settings lazy, but verify that their value never changes after they
+  //  have been read once.
+  public static final Supplier<List<String>> customHookIncludes =
+      lazyStringListSetting("custom_hook_includes",
+          "Glob patterns matching names of classes to instrument with hooks (custom and built-in)");
+  public static final Supplier<List<String>> customHookExcludes = lazyStringListSetting(
+      "custom_hook_excludes",
+      "Glob patterns matching names of classes that should not be instrumented with hooks (custom and built-in)");
+  public static final Supplier<List<String>> instrumentationIncludes =
+      lazyStringListSetting("instrumentation_includes",
+          "Glob patterns matching names of classes to instrument for fuzzing");
+  public static final Supplier<List<String>> instrumentationExcludes =
+      lazyStringListSetting("instrumentation_excludes",
+          "Glob patterns matching names of classes that should not be instrumented for fuzzing");
+
   // The values of this setting depends on autofuzz.
   public static final List<String> targetArgs = autofuzz.isEmpty()
       ? stringListSetting(
diff --git a/src/main/java/com/code_intelligence/jazzer/driver/OptParser.java b/src/main/java/com/code_intelligence/jazzer/driver/OptParser.java
index 98a291ff..ab096f2b 100644
--- a/src/main/java/com/code_intelligence/jazzer/driver/OptParser.java
+++ b/src/main/java/com/code_intelligence/jazzer/driver/OptParser.java
@@ -27,6 +27,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.TreeMap;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -70,18 +71,29 @@ final class OptParser {
   }
 
   static List<String> stringListSetting(String name, String description) {
-    return stringListSetting(name, File.pathSeparatorChar, description);
+    return lazyStringListSetting(name, description).get();
   }
 
   static List<String> stringListSetting(String name, char separator, String description) {
+    return lazyStringListSetting(name, separator, description).get();
+  }
+
+  static Supplier<List<String>> lazyStringListSetting(String name, String description) {
+    return lazyStringListSetting(name, File.pathSeparatorChar, description);
+  }
+
+  static Supplier<List<String>> lazyStringListSetting(
+      String name, char separator, String description) {
     knownArgs.put(name,
         OptDetails.create(
             name, String.format("list separated by '%c'", separator), "", description));
-    String value = System.getProperty(OPTIONS_PREFIX + name);
-    if (value == null || value.isEmpty()) {
-      return Collections.emptyList();
-    }
-    return splitOnUnescapedSeparator(value, separator);
+    return () -> {
+      String value = System.getProperty(OPTIONS_PREFIX + name);
+      if (value == null || value.isEmpty()) {
+        return Collections.emptyList();
+      }
+      return splitOnUnescapedSeparator(value, separator);
+    };
   }
 
   static boolean boolSetting(String name, boolean defaultValue, String description) {
diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel
index b0e88452..28f9aafa 100644
--- a/tests/BUILD.bazel
+++ b/tests/BUILD.bazel
@@ -257,6 +257,29 @@ java_fuzz_target_test(
     runtime_deps = [":native_value_profile_fuzzer"],
 )
 
+java_binary(
+    name = "JUnitAgentConfigurationFuzzTest",
+    srcs = ["src/test/java/com/example/JUnitAgentConfigurationFuzzTest.java"],
+    main_class = "com.code_intelligence.jazzer.Jazzer",
+    runtime_deps = [
+        "//deploy:jazzer",
+        "@maven//:org_junit_jupiter_junit_jupiter_engine",
+    ],
+    deps = [
+        "//deploy:jazzer-api",
+        "//deploy:jazzer-junit",
+        "@maven//:org_junit_jupiter_junit_jupiter_api",
+    ],
+)
+
+sh_test(
+    name = "junit_agent_configuration_test",
+    srcs = ["src/test/shell/junit_agent_configuration_test.sh"],
+    args = ["$(rlocationpath :JUnitAgentConfigurationFuzzTest)"],
+    data = [":JUnitAgentConfigurationFuzzTest"],
+    deps = ["@bazel_tools//tools/bash/runfiles"],
+)
+
 java_fuzz_target_test(
     name = "JUnitAssertFuzzer",
     timeout = "short",
diff --git a/tests/src/test/java/com/example/JUnitAgentConfigurationFuzzTest.java b/tests/src/test/java/com/example/JUnitAgentConfigurationFuzzTest.java
new file mode 100644
index 00000000..4f8c2a19
--- /dev/null
+++ b/tests/src/test/java/com/example/JUnitAgentConfigurationFuzzTest.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2023 Code Intelligence GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.example;
+
+import static java.util.Collections.singletonList;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.code_intelligence.jazzer.junit.FuzzTest;
+import java.util.function.Supplier;
+
+class JUnitAgentConfigurationFuzzTest {
+  @FuzzTest
+  void testConfiguration(byte[] bytes) {
+    assertEquals(singletonList("com.example.**"), getLazyOptValue("instrumentationIncludes"));
+    assertEquals(singletonList("com.example.**"), getLazyOptValue("customHookIncludes"));
+  }
+
+  private static Object getLazyOptValue(String name) {
+    try {
+      Supplier<Object> supplier =
+          (Supplier<Object>) Class.forName("com.code_intelligence.jazzer.driver.Opt")
+              .getField(name)
+              .get(null);
+      return supplier.get();
+    } catch (NoSuchFieldException | ClassNotFoundException | IllegalAccessException e) {
+      throw new IllegalStateException(e);
+    }
+  }
+}
diff --git a/tests/src/test/shell/junit_agent_configuration_test.sh b/tests/src/test/shell/junit_agent_configuration_test.sh
new file mode 100755
index 00000000..dd029825
--- /dev/null
+++ b/tests/src/test/shell/junit_agent_configuration_test.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright 2022 Code Intelligence GmbH
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Verify that instrumentation filter defaults set by @FuzzTest work.
+
+# --- begin runfiles.bash initialization v2 ---
+# Copy-pasted from the Bazel Bash runfiles library v2.
+set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$0.runfiles/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v2 ---
+
+function fail() {
+  echo "FAILED: $1"
+  exit 1
+}
+
+stderr="$TEST_TMPDIR/stderr"
+
+"$(rlocation "$1")" --target_class=com.example.JUnitAgentConfigurationFuzzTest 2>&1 -runs=1 | tee "$stderr" || fail "Jazzer did not exit with exit code 0"
+
+[[ $(grep -c "INFO: Instrumented " "$stderr") == 1 ]] || fail "Expected exactly one instrumented class"
+[[ $(grep "INFO: Instrumented " "$stderr" | grep -c -v "INFO: Instrumented com.example.") == 0 ]] || fail "Expected all instrumented classes to be in com.example"