junit: Also include seed invocation in fuzzing mode

This improves consistency between regression test and fuzzing mode as
well as between file-provided and Java-provided seeds (added in a
follow-up commit).

11 files changed, 149 insertions, 45 deletions

diff --git a/examples/junit/src/test/java/com/example/DirectoryInputsFuzzTest.java b/examples/junit/src/test/java/com/example/DirectoryInputsFuzzTest.java
index b0b49e56..1d1ce2c4 100644
--- a/examples/junit/src/test/java/com/example/DirectoryInputsFuzzTest.java
+++ b/examples/junit/src/test/java/com/example/DirectoryInputsFuzzTest.java
@@ -27,11 +27,12 @@ public class DirectoryInputsFuzzTest {
     if (data.remainingBytes() == 0) {
       return;
     }
-    if (!firstSeed) {
+    String input = data.consumeRemainingAsString();
+    if (!firstSeed && !input.equals("directory")) {
       throw new IllegalStateException("Should have crashed on the first non-empty input");
     }
     firstSeed = false;
-    if (data.consumeRemainingAsString().equals("directory")) {
+    if (input.equals("directory")) {
       throw new FuzzerSecurityIssueMedium();
     }
   }
diff --git a/examples/junit/src/test/java/com/example/LifecycleFuzzTest.java b/examples/junit/src/test/java/com/example/LifecycleFuzzTest.java
index 34553d7c..0d5dc2c7 100644
--- a/examples/junit/src/test/java/com/example/LifecycleFuzzTest.java
+++ b/examples/junit/src/test/java/com/example/LifecycleFuzzTest.java
@@ -33,6 +33,10 @@ import org.junit.jupiter.api.extension.TestInstancePostProcessor;
 @TestMethodOrder(MethodOrderer.MethodName.class)
 @ExtendWith(LifecycleFuzzTest.LifecycleInstancePostProcessor.class)
 class LifecycleFuzzTest {
+  // In fuzzing mode, the test is invoked once on the empty input and once with Jazzer.
+  private static final int EXPECTED_EACH_COUNT =
+      System.getenv().getOrDefault("JAZZER_FUZZ", "").isEmpty() ? 1 : 2;
+
   private static int beforeAllCount = 0;
   private static int beforeEachGlobalCount = 0;
   private static int afterEachGlobalCount = 0;
@@ -61,8 +65,7 @@ class LifecycleFuzzTest {
   @FuzzTest(maxDuration = "1s")
   void lifecycleFuzz(byte[] data) {
     Assertions.assertEquals(1, beforeAllCount);
-    Assertions.assertEquals(1, beforeEachGlobalCount);
-    Assertions.assertEquals(0, afterEachGlobalCount);
+    Assertions.assertEquals(beforeEachGlobalCount, afterEachGlobalCount + 1);
     Assertions.assertTrue(beforeEachCalledOnInstance);
     Assertions.assertTrue(testInstancePostProcessorCalledOnInstance);
   }
@@ -76,8 +79,8 @@ class LifecycleFuzzTest {
   static void afterAll() throws IOException {
     afterAllCount++;
     Assertions.assertEquals(1, beforeAllCount);
-    Assertions.assertEquals(1, beforeEachGlobalCount);
-    Assertions.assertEquals(1, afterEachGlobalCount);
+    Assertions.assertEquals(EXPECTED_EACH_COUNT, beforeEachGlobalCount);
+    Assertions.assertEquals(EXPECTED_EACH_COUNT, afterEachGlobalCount);
     Assertions.assertEquals(1, afterAllCount);
     throw new IOException();
   }
diff --git a/src/main/java/com/code_intelligence/jazzer/junit/SeedArgumentsProvider.java b/src/main/java/com/code_intelligence/jazzer/junit/SeedArgumentsProvider.java
index 50618252..4b210033 100644
--- a/src/main/java/com/code_intelligence/jazzer/junit/SeedArgumentsProvider.java
+++ b/src/main/java/com/code_intelligence/jazzer/junit/SeedArgumentsProvider.java
@@ -15,6 +15,7 @@
 package com.code_intelligence.jazzer.junit;
 
 import static com.code_intelligence.jazzer.junit.Utils.isFuzzing;
+import static com.code_intelligence.jazzer.junit.Utils.runFromCommandLine;
 import static org.junit.jupiter.api.Named.named;
 import static org.junit.jupiter.params.provider.Arguments.arguments;
 
@@ -55,11 +56,13 @@ class SeedArgumentsProvider implements ArgumentsProvider {
   @Override
   public Stream<? extends Arguments> provideArguments(ExtensionContext extensionContext)
       throws IOException {
-    return isFuzzing(extensionContext) ? Stream.empty() : provideSeedArguments(extensionContext);
-  }
+    if (runFromCommandLine(extensionContext)) {
+      // libFuzzer always runs on the file-based seeds first anyway and the additional visual
+      // indication provided by test invocations for seeds isn't effective on the command line, so
+      // we skip these invocations.
+      return Stream.empty();
+    }
 
-  private Stream<? extends Arguments> provideSeedArguments(ExtensionContext extensionContext)
-      throws IOException {
     Class<?> testClass = extensionContext.getRequiredTestClass();
     Method testMethod = extensionContext.getRequiredTestMethod();
 
@@ -74,11 +77,13 @@ class SeedArgumentsProvider implements ArgumentsProvider {
     }
 
     return adaptInputsForFuzzTest(extensionContext.getRequiredTestMethod(), rawSeeds).onClose(() -> {
-      extensionContext.publishReportEntry(
-          "No fuzzing has been performed, the fuzz test has only been executed on the fixed "
-          + "set of inputs in the seed corpus.\n"
-          + "To start fuzzing, run a test with the environment variable JAZZER_FUZZ set to a "
-          + "non-empty value.");
+      if (!isFuzzing(extensionContext)) {
+        extensionContext.publishReportEntry(
+            "No fuzzing has been performed, the fuzz test has only been executed on the fixed "
+            + "set of inputs in the seed corpus.\n"
+            + "To start fuzzing, run a test with the environment variable JAZZER_FUZZ set to a "
+            + "non-empty value.");
+      }
       if (invalidCorpusFilesPresent) {
         extensionContext.publishReportEntry(
             "Some files in the seed corpus do not match the fuzz target signature.\n"
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/AutofuzzTest.java b/src/test/java/com/code_intelligence/jazzer/junit/AutofuzzTest.java
index 005ce70a..b9abd3fe 100644
--- a/src/test/java/com/code_intelligence/jazzer/junit/AutofuzzTest.java
+++ b/src/test/java/com/code_intelligence/jazzer/junit/AutofuzzTest.java
@@ -20,6 +20,7 @@ import static com.google.common.truth.Truth8.assertThat;
 import static org.junit.Assume.assumeFalse;
 import static org.junit.Assume.assumeTrue;
 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod;
+import static org.junit.platform.testkit.engine.EventConditions.abortedWithReason;
 import static org.junit.platform.testkit.engine.EventConditions.container;
 import static org.junit.platform.testkit.engine.EventConditions.displayName;
 import static org.junit.platform.testkit.engine.EventConditions.event;
@@ -48,6 +49,7 @@ import org.junit.Test;
 import org.junit.platform.testkit.engine.EngineExecutionResults;
 import org.junit.platform.testkit.engine.EngineTestKit;
 import org.junit.rules.TemporaryFolder;
+import org.opentest4j.TestAbortedException;
 
 public class AutofuzzTest {
   @Rule public TemporaryFolder temp = new TemporaryFolder();
@@ -77,7 +79,7 @@ public class AutofuzzTest {
     final String clazz = "class:com.example.AutofuzzFuzzTest";
     final String autofuzz =
         "test-template:autofuzz(java.lang.String, com.example.AutofuzzFuzzTest$IntHolder)";
-    final String invocation = "test-template-invocation:#1";
+    final String invocation = "test-template-invocation:#";
 
     results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(engine)),
         event(type(STARTED), container(uniqueIdSubstrings(engine, clazz))),
@@ -87,11 +89,15 @@ public class AutofuzzTest {
         event(type(FINISHED), container(uniqueIdSubstrings(engine, clazz)), finishedSuccessfully()),
         event(type(FINISHED), container(engine), finishedSuccessfully()));
 
-    results.testEvents().assertEventsMatchExactly(
+    results.testEvents().assertEventsMatchExactly(event(type(DYNAMIC_TEST_REGISTERED)),
+        event(type(STARTED)),
+        event(test(uniqueIdSubstrings(engine, clazz, autofuzz, invocation + 1)),
+            displayName("<empty input>"),
+            abortedWithReason(instanceOf(TestAbortedException.class))),
         event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(engine, clazz, autofuzz))),
-        event(type(STARTED), test(uniqueIdSubstrings(engine, clazz, autofuzz, invocation)),
+        event(type(STARTED), test(uniqueIdSubstrings(engine, clazz, autofuzz, invocation + 2)),
             displayName("Fuzzing...")),
-        event(type(FINISHED), test(uniqueIdSubstrings(engine, clazz, autofuzz, invocation)),
+        event(type(FINISHED), test(uniqueIdSubstrings(engine, clazz, autofuzz, invocation + 2)),
             displayName("Fuzzing..."), finishedWithFailure(instanceOf(RuntimeException.class))));
 
     // Should crash on an input that contains "jazzer", with the crash emitted into the
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel
index 6ed975e0..9492cb57 100644
--- a/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel
+++ b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel
@@ -187,6 +187,7 @@ java_test(
             "@maven//:junit_junit",
             "@maven//:org_junit_platform_junit_platform_engine",
             "@maven//:org_junit_platform_junit_platform_testkit",
+            "@maven//:org_opentest4j_opentest4j",
         ],
     )
     for JAZZER_FUZZ in [
@@ -273,6 +274,7 @@ java_test(
         deps = [
             "//src/main/java/com/code_intelligence/jazzer/api:hooks",
             "@maven//:junit_junit",
+            "@maven//:org_assertj_assertj_core",
             "@maven//:org_junit_platform_junit_platform_engine",
             "@maven//:org_junit_platform_junit_platform_testkit",
         ],
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/DirectoryInputsTest.java b/src/test/java/com/code_intelligence/jazzer/junit/DirectoryInputsTest.java
index 5af06e99..7ef27a37 100644
--- a/src/test/java/com/code_intelligence/jazzer/junit/DirectoryInputsTest.java
+++ b/src/test/java/com/code_intelligence/jazzer/junit/DirectoryInputsTest.java
@@ -88,9 +88,17 @@ public class DirectoryInputsTest {
 
     results.testEvents().assertEventsMatchExactly(
         event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
-        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION)),
+        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1))),
+        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1)),
+            displayName("<empty input>"), finishedSuccessfully()),
+        event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
+        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2))),
+        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2)),
+            displayName("seed"), finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class))),
+        event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
+        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 3)),
             displayName("Fuzzing...")),
-        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION)),
+        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 3)),
             displayName("Fuzzing..."),
             finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class))));
 
@@ -140,7 +148,7 @@ public class DirectoryInputsTest {
         event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
         event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1))),
         event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1)),
-            finishedSuccessfully()),
+            displayName("<empty input>"), finishedSuccessfully()),
         event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
         event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2))),
         event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2)),
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithCrashTest.java b/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithCrashTest.java
index d9282547..5cc2d1c4 100644
--- a/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithCrashTest.java
+++ b/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithCrashTest.java
@@ -55,7 +55,7 @@ public class FuzzingWithCrashTest {
   private static final byte[] CRASHING_SEED_CONTENT = new byte[] {'b', 'a', 'c'};
   private static final String CRASHING_SEED_DIGEST = "5e4dec23c9afa48bd5bee3daa2a0ab66e147012b";
   private static final String ENGINE = "engine:junit-jupiter";
-  private static final String INVOCATION = "test-template-invocation:#1";
+  private static final String INVOCATION = "test-template-invocation:#";
 
   private static final String CLAZZ_NAME = "com.example.ValidFuzzTests";
 
@@ -115,7 +115,7 @@ public class FuzzingWithCrashTest {
         event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()),
         event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
 
-    results.testEvents().assertEventsMatchExactly(
+    results.testEvents().assertEventsMatchLooselyInOrder(
         event(type(DYNAMIC_TEST_REGISTERED),
             test(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId()))),
         event(type(STARTED),
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithoutCrashTest.java b/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithoutCrashTest.java
index cffa286e..01fe6252 100644
--- a/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithoutCrashTest.java
+++ b/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithoutCrashTest.java
@@ -95,7 +95,7 @@ public class FuzzingWithoutCrashTest {
         event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()),
         event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
 
-    results.testEvents().assertEventsMatchExactly(
+    results.testEvents().assertEventsMatchLooselyInOrder(
         event(
             type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))),
         event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ, INVOCATION)),
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/LifecycleTest.java b/src/test/java/com/code_intelligence/jazzer/junit/LifecycleTest.java
index b7c85992..29dfc664 100644
--- a/src/test/java/com/code_intelligence/jazzer/junit/LifecycleTest.java
+++ b/src/test/java/com/code_intelligence/jazzer/junit/LifecycleTest.java
@@ -46,7 +46,7 @@ public class LifecycleTest {
   private static final String CLAZZ = "class:com.example.LifecycleFuzzTest";
   private static final String DISABLED_FUZZ = "test-template:disabledFuzz([B)";
   private static final String LIFECYCLE_FUZZ = "test-template:lifecycleFuzz([B)";
-  private static final String INVOCATION = "test-template-invocation:#1";
+  private static final String INVOCATION = "test-template-invocation:#";
   @Rule public TemporaryFolder temp = new TemporaryFolder();
   Path baseDir;
 
@@ -86,9 +86,19 @@ public class LifecycleTest {
     results.testEvents().assertEventsMatchExactly(
         event(
             type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
-        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION)),
+        event(type(STARTED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)),
+            displayName("<empty input>")),
+        event(type(FINISHED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)),
+            displayName("<empty input>"), finishedSuccessfully()),
+        event(
+            type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
+        event(type(STARTED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)),
             displayName("Fuzzing...")),
-        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION)),
+        event(type(FINISHED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)),
             displayName("Fuzzing..."), finishedSuccessfully()));
   }
 
@@ -112,9 +122,11 @@ public class LifecycleTest {
     results.testEvents().assertEventsMatchExactly(
         event(
             type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
-        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION)),
+        event(type(STARTED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)),
             displayName("<empty input>")),
-        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION)),
+        event(type(FINISHED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)),
             displayName("<empty input>"), finishedSuccessfully()));
   }
 }
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java b/src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java
index 0ebcbed3..3fcc163b 100644
--- a/src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java
+++ b/src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java
@@ -22,6 +22,7 @@ import static org.junit.platform.testkit.engine.EventConditions.displayName;
 import static org.junit.platform.testkit.engine.EventConditions.event;
 import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully;
 import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure;
+import static org.junit.platform.testkit.engine.EventConditions.reportEntry;
 import static org.junit.platform.testkit.engine.EventConditions.test;
 import static org.junit.platform.testkit.engine.EventConditions.type;
 import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings;
@@ -35,11 +36,14 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import org.assertj.core.api.Condition;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.platform.engine.reporting.ReportEntry;
 import org.junit.platform.testkit.engine.EngineExecutionResults;
 import org.junit.platform.testkit.engine.EngineTestKit;
+import org.junit.platform.testkit.engine.Event;
 import org.junit.rules.TemporaryFolder;
 
 public class MutatorTest {
@@ -47,8 +51,10 @@ public class MutatorTest {
   private static final String CLASS_NAME = "com.example.MutatorFuzzTest";
   private static final String CLAZZ = "class:" + CLASS_NAME;
   private static final String LIFECYCLE_FUZZ = "test-template:mutatorFuzz(java.util.List)";
-  private static final String INVOCATION1 = "test-template-invocation:#1";
-  private static final String INVOCATION2 = "test-template-invocation:#2";
+  private static final String INVOCATION = "test-template-invocation:#";
+  private static final String INVALID_SIGNATURE_ENTRY =
+      "Some files in the seed corpus do not match the fuzz target signature.\n"
+      + "This indicates that they were generated with a different signature and may cause issues reproducing previous findings.";
 
   @Rule public TemporaryFolder temp = new TemporaryFolder();
   private Path baseDir;
@@ -80,17 +86,42 @@ public class MutatorTest {
     results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)),
         event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))),
         event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
+        // Invalid corpus input warning
+        event(type(REPORTING_ENTRY_PUBLISHED),
+            container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ)),
+            new Condition<>(
+                Event.byPayload(ReportEntry.class,
+                    (it) -> it.getKeyValuePairs().values().contains(INVALID_SIGNATURE_ENTRY)),
+                "has invalid signature entry reporting entry")),
         event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ)),
             finishedSuccessfully()),
         event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()),
         event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
 
     results.testEvents().assertEventsMatchExactly(
+        event(type(DYNAMIC_TEST_REGISTERED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1))),
+        event(type(STARTED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)),
+            displayName("<empty input>")),
+        event(type(FINISHED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)),
+            displayName("<empty input>"), finishedSuccessfully()),
+        event(type(DYNAMIC_TEST_REGISTERED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2))),
+        event(type(STARTED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)),
+            displayName("invalid")),
+        event(type(FINISHED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)),
+            displayName("invalid"), finishedSuccessfully()),
         event(
             type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
-        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
+        event(type(STARTED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 3)),
             displayName("Fuzzing...")),
-        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
+        event(type(FINISHED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 3)),
             displayName("Fuzzing..."), finishedWithFailure(instanceOf(AssertionError.class))));
   }
 
@@ -115,16 +146,20 @@ public class MutatorTest {
 
     results.testEvents().assertEventsMatchExactly(
         event(type(DYNAMIC_TEST_REGISTERED),
-            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1))),
-        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1))),
+        event(type(STARTED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)),
             displayName("<empty input>")),
-        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
+        event(type(FINISHED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)),
             displayName("<empty input>"), finishedSuccessfully()),
         event(type(DYNAMIC_TEST_REGISTERED),
-            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION2))),
-        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION2)),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2))),
+        event(type(STARTED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)),
             displayName("invalid")),
-        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION2)),
+        event(type(FINISHED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)),
             displayName("invalid"), finishedSuccessfully()));
   }
 }
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/ValueProfileTest.java b/src/test/java/com/code_intelligence/jazzer/junit/ValueProfileTest.java
index 7dcd39f7..a1cc21cf 100644
--- a/src/test/java/com/code_intelligence/jazzer/junit/ValueProfileTest.java
+++ b/src/test/java/com/code_intelligence/jazzer/junit/ValueProfileTest.java
@@ -55,7 +55,7 @@ public class ValueProfileTest {
   private static final String ENGINE = "engine:junit-jupiter";
   private static final String CLAZZ = "class:com.example.ValueProfileFuzzTest";
   private static final String VALUE_PROFILE_FUZZ = "test-template:valueProfileFuzz([B)";
-  private static final String INVOCATION = "test-template-invocation:#1";
+  private static final String INVOCATION = "test-template-invocation:#";
 
   @Rule public TemporaryFolder temp = new TemporaryFolder();
   Path baseDir;
@@ -99,10 +99,26 @@ public class ValueProfileTest {
         event(type(DYNAMIC_TEST_REGISTERED),
             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
         event(type(STARTED),
-            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION)),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 1)),
+            displayName("<empty input>")),
+        event(type(FINISHED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 1)),
+            displayName("<empty input>"), finishedSuccessfully()),
+        event(type(DYNAMIC_TEST_REGISTERED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
+        event(type(STARTED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 2)),
+            displayName("empty_seed")),
+        event(type(FINISHED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 2)),
+            displayName("empty_seed"), finishedSuccessfully()),
+        event(type(DYNAMIC_TEST_REGISTERED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
+        event(type(STARTED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 3)),
             displayName("Fuzzing...")),
         event(type(FINISHED),
-            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION)),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 3)),
             displayName("Fuzzing..."),
             finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class))));
 
@@ -146,10 +162,26 @@ public class ValueProfileTest {
         event(type(DYNAMIC_TEST_REGISTERED),
             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
         event(type(STARTED),
-            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION)),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 1)),
+            displayName("<empty input>")),
+        event(type(FINISHED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 1)),
+            displayName("<empty input>"), finishedSuccessfully()),
+        event(type(DYNAMIC_TEST_REGISTERED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
+        event(type(STARTED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 2)),
+            displayName("empty_seed")),
+        event(type(FINISHED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 2)),
+            displayName("empty_seed"), finishedSuccessfully()),
+        event(type(DYNAMIC_TEST_REGISTERED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
+        event(type(STARTED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 3)),
             displayName("Fuzzing...")),
         event(type(FINISHED),
-            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION)),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 3)),
             displayName("Fuzzing..."), finishedSuccessfully()));
 
     // No crash means no crashing input is emitted anywhere.