Ignore suite()s and don't ignore init errors when using method filters.android-tsl-4.0

JUnit3 TestSuite's do not support filters. There is an android specific work
around in place, but unfortunately it cannot help when tests create their own
suites via a suite() method. This leads to inconsistent behaviour with
InstrumentationTestRunner when a class has a suite() method, but runner was
instructed to only run a single method. This change will instruct the builder
to ignore suite() methods when a method filter has been provided.

Also add a special case to method filter to never filter out 'initializationError'
descriptions, since those are generated on error cases.

Change-Id: I402d43b0e5d2731512fadae7e9003eb46a5402e5

4 files changed, 81 insertions, 3 deletions

diff --git a/runner/src/androidTest/java/android/support/test/internal/runner/TestRequestBuilderTest.java b/runner/src/androidTest/java/android/support/test/internal/runner/TestRequestBuilderTest.java
index 0185731..a92194e 100644
--- a/runner/src/androidTest/java/android/support/test/internal/runner/TestRequestBuilderTest.java
+++ b/runner/src/androidTest/java/android/support/test/internal/runner/TestRequestBuilderTest.java
@@ -299,6 +299,31 @@ public class TestRequestBuilderTest {
         }
     }
 
+    /**
+     * Test fixture for verifying support for suite() methods with tests.
+     */
+    public static class JUnit3SuiteWithTest extends TestCase {
+        public static junit.framework.Test suite() {
+            TestSuite suite = new TestSuite();
+            suite.addTestSuite(SampleJUnit3Test.class);
+            return suite;
+        }
+
+        public void testPass() {}
+    }
+
+    public static class JUnit4TestInitFailure {
+
+        // this is an invalid method - trying to run test will fail with init error
+        @Before
+        protected void setUp() {
+        }
+
+        @Test
+        public void testWillFailOnClassInit()  {
+        }
+    }
+
     @Mock
     private DeviceBuild mMockDeviceBuild;
     @Mock
@@ -1065,4 +1090,33 @@ public class TestRequestBuilderTest {
         Result result = testRunner.run(request);
         Assert.assertEquals(3, result.getRunCount());
     }
+
+
+    /**
+     * Verify suite() methods are ignored when method filter is used
+     */
+    @Test
+    public void testJUnit3Suite_filtered() {
+        Request request = mBuilder
+                .addTestMethod(JUnit3SuiteWithTest.class.getName(), "testPass")
+                .build()
+                .getRequest();
+        JUnitCore testRunner = new JUnitCore();
+        Result result = testRunner.run(request);
+        Assert.assertEquals(1, result.getRunCount());
+    }
+
+    /**
+     * Verify method filter does not filter out initialization errors
+     */
+    @Test
+    public void testJUnit4FilterWithInitError() {
+        Request request = mBuilder
+                .addTestMethod(JUnit4TestInitFailure.class.getName(), "testWillFailOnClassInit")
+                .build()
+                .getRequest();
+        JUnitCore testRunner = new JUnitCore();
+        Result result = testRunner.run(request);
+        Assert.assertEquals(1, result.getRunCount());
+    }
 }
diff --git a/runner/src/main/java/android/support/test/internal/runner/TestRequestBuilder.java b/runner/src/main/java/android/support/test/internal/runner/TestRequestBuilder.java
index 0c15fee..bced661 100644
--- a/runner/src/main/java/android/support/test/internal/runner/TestRequestBuilder.java
+++ b/runner/src/main/java/android/support/test/internal/runner/TestRequestBuilder.java
@@ -99,6 +99,14 @@ public class TestRequestBuilder {
     private ClassLoader mClassLoader;
 
     /**
+     * Instructs the test builder if JUnit3 suite() methods should be executed.
+     * <p/>
+     * Currently set to false if any method filter is set, for consistency with
+     * InstrumentationTestRunner.
+      */
+    private boolean mIgnoreSuiteMethods = false;
+
+    /**
      * Accessor interface for retrieving device build properties.
      * <p/>
      * Used so unit tests can mock calls
@@ -479,7 +487,10 @@ public class TestRequestBuilder {
                 if (mExcludedMethods.contains(methodName)) {
                     return false;
                 }
-                return mIncludedMethods.isEmpty() || mIncludedMethods.contains(methodName);
+                // don't filter out descriptions with method name "initializationError", since
+                // Junit will generate such descriptions in error cases, See ErrorReportingRunner
+                return mIncludedMethods.isEmpty() || mIncludedMethods.contains(methodName)
+                    || methodName.equals("initializationError");
             }
             // At this point, this could only be a description of this filter
             return true;
@@ -572,6 +583,7 @@ public class TestRequestBuilder {
     public TestRequestBuilder addTestMethod(String testClassName, String testMethodName) {
         mIncludedClasses.add(testClassName);
         mClassMethodFilter.addMethod(testClassName, testMethodName);
+        mIgnoreSuiteMethods = true;
         return this;
     }
 
@@ -580,6 +592,7 @@ public class TestRequestBuilder {
      */
     public TestRequestBuilder removeTestMethod(String testClassName, String testMethodName) {
         mClassMethodFilter.removeMethod(testClassName, testMethodName);
+        mIgnoreSuiteMethods = true;
         return this;
     }
 
@@ -739,7 +752,8 @@ public class TestRequestBuilder {
         }
 
         Request request = classes(
-                new AndroidRunnerParams(mInstr, mArgsBundle, mSkipExecution, mPerTestTimeout),
+                new AndroidRunnerParams(mInstr, mArgsBundle, mSkipExecution, mPerTestTimeout,
+                        mIgnoreSuiteMethods),
                 new Computer(),
                 loader.getLoadedClasses().toArray(new Class[0]));
         return new TestRequest(loader.getLoadFailures(),
diff --git a/runner/src/main/java/android/support/test/internal/runner/junit3/AndroidSuiteBuilder.java b/runner/src/main/java/android/support/test/internal/runner/junit3/AndroidSuiteBuilder.java
index f386de4..440f004 100644
--- a/runner/src/main/java/android/support/test/internal/runner/junit3/AndroidSuiteBuilder.java
+++ b/runner/src/main/java/android/support/test/internal/runner/junit3/AndroidSuiteBuilder.java
@@ -44,6 +44,9 @@ public class AndroidSuiteBuilder extends SuiteMethodBuilder {
 
     @Override
     public Runner runnerForClass(Class<?> testClass) throws Throwable {
+        if (mAndroidRunnerParams.isIgnoreSuiteMethods()) {
+            return null;
+        }
         try {
             if (hasSuiteMethod(testClass)) {
                 Test t = SuiteMethod.testFromSuiteMethod(testClass);
diff --git a/runner/src/main/java/android/support/test/internal/util/AndroidRunnerParams.java b/runner/src/main/java/android/support/test/internal/util/AndroidRunnerParams.java
index 11ce954..5fc4056 100644
--- a/runner/src/main/java/android/support/test/internal/util/AndroidRunnerParams.java
+++ b/runner/src/main/java/android/support/test/internal/util/AndroidRunnerParams.java
@@ -27,6 +27,7 @@ public class AndroidRunnerParams {
     private final Bundle mBundle;
     private final boolean mSkipExecution;
     private final long mPerTestTimeout;
+    private final boolean mIgnoreSuiteMethods;
 
     /**
      * @param instrumentation the {@link Instrumentation} to inject into any tests that require it
@@ -37,11 +38,13 @@ public class AndroidRunnerParams {
      *                      timeout
      */
     public AndroidRunnerParams(Instrumentation instrumentation, Bundle bundle,
-                             boolean skipExecution, long perTestTimeout) {
+                               boolean skipExecution, long perTestTimeout,
+                               boolean ignoreSuiteMethods) {
         this.mInstrumentation = instrumentation;
         this.mBundle = bundle;
         this.mSkipExecution = skipExecution;
         this.mPerTestTimeout = perTestTimeout;
+        this.mIgnoreSuiteMethods = ignoreSuiteMethods;
     }
 
     public Instrumentation getInstrumentation() {
@@ -59,4 +62,8 @@ public class AndroidRunnerParams {
     public long getPerTestTimeout() {
         return mPerTestTimeout;
     }
+
+    public boolean isIgnoreSuiteMethods() {
+        return mIgnoreSuiteMethods;
+    }
 }