Merge pull request #326 from bugsnag/flaky-test-experiments

Fix flaky CI tests
bugsnag · Jun 18, 2018 · ba88ac4 · ba88ac4
2 parents 2dd5d83 + b7789f4
commit ba88ac4
Show file tree

Hide file tree

Showing 17 changed files with 102 additions and 130 deletions.
diff --git a/sdk/src/androidTest/java/com/bugsnag/android/BeforeRecordBreadcrumbsTest.java b/sdk/src/androidTest/java/com/bugsnag/android/BeforeRecordBreadcrumbsTest.java
@@ -8,6 +8,7 @@
 import android.support.test.filters.SmallTest;
 import android.support.test.runner.AndroidJUnit4;
 
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -33,6 +34,11 @@ public void setUp() throws Exception {
         assertEquals(0, client.breadcrumbs.store.size());
     }
 
+    @After
+    public void tearDown() throws Exception {
+        Async.cancelTasks();
+    }
+
     @Test
     public void noCallback() throws Exception {
         client.leaveBreadcrumb("Hello");

diff --git a/sdk/src/androidTest/java/com/bugsnag/android/BreadcrumbsTest.java b/sdk/src/androidTest/java/com/bugsnag/android/BreadcrumbsTest.java
@@ -11,6 +11,7 @@
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -32,6 +33,11 @@ public void setUp() throws Exception {
         breadcrumbs = new Breadcrumbs();
     }
 
+    @After
+    public void tearDown() throws Exception {
+        Async.cancelTasks();
+    }
+
     @Test
     public void testSerialization() throws JSONException, IOException {
         breadcrumbs.add(new Breadcrumb("Started app"));

diff --git a/sdk/src/androidTest/java/com/bugsnag/android/ClientConfigTest.java b/sdk/src/androidTest/java/com/bugsnag/android/ClientConfigTest.java
@@ -10,6 +10,7 @@
 import android.support.test.filters.SmallTest;
 import android.support.test.runner.AndroidJUnit4;
 
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -32,6 +33,11 @@ public void setUp() throws Exception {
         client = new Client(context, config);
     }
 
+    @After
+    public void tearDown() throws Exception {
+        Async.cancelTasks();
+    }
+
     @Test
     public void testSetReleaseStage() throws Exception {
         client.setReleaseStage("beta");

diff --git a/sdk/src/androidTest/java/com/bugsnag/android/ClientNotifyAsyncTest.java b/sdk/src/androidTest/java/com/bugsnag/android/ClientNotifyAsyncTest.java
diff --git a/sdk/src/androidTest/java/com/bugsnag/android/ClientNotifyTest.java b/sdk/src/androidTest/java/com/bugsnag/android/ClientNotifyTest.java
@@ -5,6 +5,7 @@
 import android.support.test.filters.SmallTest;
 import android.support.test.runner.AndroidJUnit4;
 
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -32,6 +33,11 @@ public void setUp() throws Exception {
         client.setErrorReportApiClient(apiClient);
     }
 
+    @After
+    public void tearDown() throws Exception {
+        Async.cancelTasks();
+    }
+
     @Test
     public void testNotifyBlockingDefaultSeverity() {
         client.notifyBlocking(new RuntimeException("Testing"));

diff --git a/sdk/src/androidTest/java/com/bugsnag/android/ClientTest.java b/sdk/src/androidTest/java/com/bugsnag/android/ClientTest.java
@@ -50,6 +50,7 @@ public void setUp() throws Exception {
     @After
     public void tearDown() throws Exception {
         clearSharedPrefs();
+        Async.cancelTasks();
     }
 
     private void clearSharedPrefs() {

diff --git a/sdk/src/androidTest/java/com/bugsnag/android/ConcurrentCallbackTest.java b/sdk/src/androidTest/java/com/bugsnag/android/ConcurrentCallbackTest.java
@@ -4,6 +4,7 @@
 import android.support.test.filters.SmallTest;
 import android.support.test.runner.AndroidJUnit4;
 
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -25,6 +26,11 @@ public void setUp() throws Exception {
         client = BugsnagTestUtils.generateClient();
     }
 
+    @After
+    public void tearDown() throws Exception {
+        Async.cancelTasks();
+    }
+
     @Test
     public void testClientNotifyModification() throws Exception {
         final Collection<BeforeNotify> beforeNotifyTasks = client.config.getBeforeNotifyTasks();

diff --git a/sdk/src/androidTest/java/com/bugsnag/android/DeliveryCompatTest.java b/sdk/src/androidTest/java/com/bugsnag/android/DeliveryCompatTest.java
@@ -5,6 +5,7 @@
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -29,6 +30,11 @@ public void setUp() throws Exception {
         deliveryCompat = new DeliveryCompat();
     }
 
+    @After
+    public void tearDown() throws Exception {
+        Async.cancelTasks();
+    }
+
     @Test
     public void deliverReport() throws Exception {
         Report report = null;

diff --git a/sdk/src/androidTest/java/com/bugsnag/android/ErrorReportApiClientTest.java b/sdk/src/androidTest/java/com/bugsnag/android/ErrorReportApiClientTest.java
@@ -7,6 +7,7 @@
 import android.support.test.filters.SmallTest;
 import android.support.test.runner.AndroidJUnit4;
 
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -25,6 +26,11 @@ public void setUp() throws Exception {
         Bugsnag.init(InstrumentationRegistry.getContext(), "123");
     }
 
+    @After
+    public void tearDown() throws Exception {
+        Async.cancelTasks();
+    }
+
     @Test(expected = IllegalArgumentException.class)
     public void testApiClientNullValidation() {
         Bugsnag.setErrorReportApiClient(null);

diff --git a/sdk/src/androidTest/java/com/bugsnag/android/ExceptionHandlerTest.java b/sdk/src/androidTest/java/com/bugsnag/android/ExceptionHandlerTest.java
@@ -9,6 +9,7 @@
 import android.support.test.filters.SmallTest;
 import android.support.test.runner.AndroidJUnit4;
 
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -32,6 +33,11 @@ public void setUp() throws Exception {
         Thread.setDefaultUncaughtExceptionHandler(null);
     }
 
+    @After
+    public void tearDown() throws Exception {
+        Async.cancelTasks();
+    }
+
     @Test
     public void testEnableDisable() {
         Client client = new Client(context, "api-key");

diff --git a/sdk/src/androidTest/java/com/bugsnag/android/MetaDataTest.java b/sdk/src/androidTest/java/com/bugsnag/android/MetaDataTest.java
@@ -12,6 +12,7 @@
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -35,6 +36,11 @@ public void setUp() throws Exception {
         client = BugsnagTestUtils.generateClient();
     }
 
+    @After
+    public void tearDown() throws Exception {
+        Async.cancelTasks();
+    }
+
     @Test
     public void testBasicSerialization() throws JSONException, IOException {
         MetaData metaData = new MetaData();

diff --git a/sdk/src/androidTest/java/com/bugsnag/android/NullMetadataTest.java b/sdk/src/androidTest/java/com/bugsnag/android/NullMetadataTest.java
@@ -5,6 +5,7 @@
 
 import android.support.test.InstrumentationRegistry;
 
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -44,6 +45,11 @@ public void postReport(String urlString,
         throwable = new RuntimeException("Test");
     }
 
+    @After
+    public void tearDown() throws Exception {
+        Async.cancelTasks();
+    }
+
     @Test
     public void testErrorDefaultMetaData() throws Exception {
         Error error = new Error.Builder(config, throwable, null).build();

diff --git a/sdk/src/androidTest/java/com/bugsnag/android/SessionTrackerTest.java b/sdk/src/androidTest/java/com/bugsnag/android/SessionTrackerTest.java
@@ -10,6 +10,7 @@
 import static junit.framework.Assert.assertTrue;
 import static org.junit.Assert.assertNotEquals;
 
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -38,6 +39,11 @@ public void setUp() throws Exception {
         user = new User();
     }
 
+    @After
+    public void tearDown() throws Exception {
+        Async.cancelTasks();
+    }
+
     @Test
     public void startNewSession() throws Exception {
         assertNotNull(sessionTracker);

diff --git a/sdk/src/androidTest/java/com/bugsnag/android/UniqueBeforeNotifyTest.java b/sdk/src/androidTest/java/com/bugsnag/android/UniqueBeforeNotifyTest.java
@@ -44,6 +44,7 @@ public void setUp() throws Exception {
     @After
     public void tearDown() throws Exception {
         callbackCount = 0;
+        Async.cancelTasks();
     }
 
     @Test

diff --git a/sdk/src/main/java/com/bugsnag/android/Async.java b/sdk/src/main/java/com/bugsnag/android/Async.java
@@ -29,12 +29,19 @@ public Thread newThread(@NonNull Runnable runnable) {
             return new Thread(runnable, "Bugsnag Thread #" + count.getAndIncrement());
         }
     };
-    private static final Executor EXECUTOR = new ThreadPoolExecutor(
+    private static final ThreadPoolExecutor EXECUTOR = new ThreadPoolExecutor(
         CORE_POOL_SIZE, MAXIMUM_POOL_SIZE, KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
         POOL_WORK_QUEUE, THREAD_FACTORY);
 
     static void run(@NonNull Runnable task) throws RejectedExecutionException {
         EXECUTOR.execute(task);
     }
 
+    static void cancelTasks() throws InterruptedException {
+        Logger.info("Cancelling tasks");
+        EXECUTOR.shutdown();
+        EXECUTOR.awaitTermination(2000, TimeUnit.MILLISECONDS);
+        Logger.info("Finishing cancelling tasks");
+    }
+
 }
diff --git a/sdk/src/main/java/com/bugsnag/android/Client.java b/sdk/src/main/java/com/bugsnag/android/Client.java
@@ -192,14 +192,19 @@ public Client(@NonNull Context androidContext, @NonNull Configuration configurat
 
         // register a receiver for automatic breadcrumbs
 
-        Async.run(new Runnable() {
-            @Override
-            public void run() {
-                appContext.registerReceiver(eventReceiver, EventReceiver.getIntentFilter());
-                appContext.registerReceiver(new ConnectivityChangeReceiver(),
-                    new IntentFilter(ConnectivityManager.CONNECTIVITY_ACTION));
-            }
-        });
+        try {
+            Async.run(new Runnable() {
+                @Override
+                public void run() {
+                    appContext.registerReceiver(eventReceiver, EventReceiver.getIntentFilter());
+                    appContext.registerReceiver(new ConnectivityChangeReceiver(),
+                        new IntentFilter(ConnectivityManager.CONNECTIVITY_ACTION));
+                }
+            });
+        } catch (RejectedExecutionException ex) {
+            Logger.warn("Failed to register for automatic breadcrumb broadcasts", ex);
+        }
+
 
         config.addObserver(this);