maze(tests): Rework scenarios to rely less on UB

Some Android + arch combinations do mildly unsettling things, like not crash when you start poking around released memory. It seems this hole was eventually closed, but it doesn’t help test the efficacy of the library. Add new scenarios to test raising each signal, for sanity. Replaced UB-dependent tests with cleaner, crashier ones. Marked stack buffer overflow scenario as API 18+, as prior to that, it would kill() without calling handlers.
bugsnag · Sep 17, 2018 · 49f149c · 49f149c
1 parent 4926938
commit 49f149c
Show file tree

Hide file tree

Showing 13 changed files with 491 additions and 34 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -18,7 +18,7 @@ env:
     # other things) is called!
   - PATH=${ANDROID_HOME}:${ANDROID_HOME}/emulator:${TOOLS}:${TOOLS}/bin:${ANDROID_HOME}/platform-tools:${PATH}
   - QEMU_AUDIO_DRV=none # Prevents some fatal audio issues on API 24+
-  - MAZE_WAIT_TIME=15
+  - MAZE_WAIT_TIME=18
 
 android:
   components:
@@ -50,7 +50,7 @@ before_script:
 - adb shell input keyevent 82 &
 
 script:
-- bundle exec bugsnag-maze-runner --color features/native*.feature
+- bundle exec bugsnag-maze-runner --color features/native*.feature $MAZE_ARGS
 
 jobs:
   include:
@@ -69,6 +69,7 @@ jobs:
     env: NDK_VERSION=r16b
   # Emulators for API 22,23,25 just don't boot ¯\_(ツ)_/¯
   # Emulators for API 14,15,26 hang regularly
+  # There are no images for API 20
   # There's no emulators for x86 because we can't run x86 on Travis
   - stage: end-to-end tests
     env:
@@ -79,10 +80,22 @@ jobs:
   - stage: end-to-end tests
     env:
     - ANDROID_TARGET=android-19
+  - stage: end-to-end tests
+    env:
+    - ANDROID_TARGET=android-18
+    - EMULATOR_FLAGS=-no-audio
+    - MAZE_ARGS=--verbose
+  - stage: end-to-end tests
+    env:
+    - ANDROID_TARGET=android-17
+    - EMULATOR_FLAGS=-no-audio
+    - MAZE_ARGS=--tags "not @skip_below_api18" --verbose
+    - MAZE_WAIT_TIME=30
   - stage: end-to-end tests
     env:
     - ANDROID_TARGET=android-16
     - EMULATOR_FLAGS=-no-audio
+    - MAZE_ARGS=--tags "not @skip_below_api18" --verbose
   - stage: deploy
     script: skip
     before_deploy: ./gradlew javadoc

diff --git a/features/fixtures/mazerunner/src/main/cpp/entrypoint.cpp b/features/fixtures/mazerunner/src/main/cpp/entrypoint.cpp
@@ -1,11 +1,12 @@
 #include <jni.h>
 #include <stdlib.h>
 #include <string.h>
+#include <signal.h>
 #include <time.h>
 
 extern "C" {
 
-static char * __attribute__((used)) somefakefunc(void) {}
+static char * __attribute__((used)) somefakefunc(void) {};
 
 typedef struct {
   int field1;
@@ -35,15 +36,33 @@ int crash_null_pointer(bool route) {
   return j;
 }
 
-int crash_released_obj(int counter) {
-    char *text = (char *)malloc(30);
-    reporter_t *report = (reporter_t *) malloc(sizeof(reporter_t));
-    report->field1 = 6;
-    free(report);
-    if (counter > 0)
-      strncpy(text, report->field2, sizeof(text));
+int crash_write_read_only_mem(int counter) {
+  if (counter > 2) {
+    int *pointer = (int *)&somefakefunc;
+    *pointer = counter;
+  }
+  return counter / 14;
+}
+
+char *crash_improper_cast(int counter) {
+  reporter_t *report = (reporter_t *)counter;
+
+  return report->field2;
+}
 
-    return report->field1;
+int crash_double_free(int counter) {
+  for (int i = 0; i < 30; i++) {
+    reporter_t *reporter = (reporter_t *)malloc(sizeof(reporter_t));
+    reporter->field1 = 22 + counter;
+    char *field2 = reporter->field2;
+    strcpy(field2, "Indeed");
+    printf("%d field1 is: %d", i, reporter->field1);
+    printf("%d field2 is: %s", i, field2);
+    free(field2);
+    free(reporter);
+  }
+
+  return counter / -8;
 }
 
 int crash_trap() {
@@ -63,6 +82,78 @@ int crash_stack_overflow(int counter, char *input) {
   return 4 / counter;
 }
 
+JNIEXPORT int JNICALL
+Java_com_bugsnag_android_mazerunner_scenarios_CXXSigtrapScenario_crash(JNIEnv *env,
+                                                                      jobject instance,
+                                                                      jint value) {
+  int x = 38;
+  if (value > 0) {
+    raise(SIGTRAP);
+  }
+  printf("Shields up! Rrrrred alert!.\n");
+  return value / x;
+}
+
+JNIEXPORT int JNICALL
+Java_com_bugsnag_android_mazerunner_scenarios_CXXSigbusScenario_crash(JNIEnv *env,
+                                                                      jobject instance,
+                                                                      jint value) {
+  int x = 38;
+  if (value > 0) {
+    raise(SIGBUS);
+  }
+  printf("A surprise party? Mr. Worf, I hate surprise parties. I would *never* do that to you.\n");
+  return value / x / 8;
+}
+
+JNIEXPORT int JNICALL
+Java_com_bugsnag_android_mazerunner_scenarios_CXXSigabrtScenario_crash(JNIEnv *env,
+                                                                        jobject instance,
+                                                                        jint value) {
+  int x = 38;
+  if (value > 0) {
+    raise(SIGABRT);
+  }
+  printf("Is it my imagination, or have tempers become a little frayed on the ship lately?\n");
+  return value / x / 8;
+}
+
+JNIEXPORT int JNICALL
+Java_com_bugsnag_android_mazerunner_scenarios_CXXSigfpeScenario_crash(JNIEnv *env,
+                                                                      jobject instance,
+                                                                      jint value) {
+  int x = 38;
+  if (value > 0) {
+    raise(SIGFPE);
+  }
+  printf("We know you're dealing in stolen ore.\n");
+  return value / x / 8;
+}
+
+JNIEXPORT int JNICALL
+Java_com_bugsnag_android_mazerunner_scenarios_CXXSigsegvScenario_crash(JNIEnv *env,
+                                                                       jobject instance,
+                                                                       jint value) {
+  int x = 38;
+  if (value > 0) {
+    raise(SIGSEGV);
+  }
+  printf("That might've been one of the shortest assignments in the history of Starfleet.\n");
+  return value / x / 8;
+}
+
+JNIEXPORT int JNICALL
+Java_com_bugsnag_android_mazerunner_scenarios_CXXSigillScenario_crash(JNIEnv *env,
+                                                                      jobject instance,
+                                                                      jint value) {
+  int x = 38;
+  if (value > 0) {
+    raise(SIGILL);
+  }
+  printf("In all trust, there is the possibility for betrayal.\n");
+  return value / x / 8;
+}
+
 JNIEXPORT void JNICALL
 Java_com_bugsnag_android_mazerunner_scenarios_CXXNullPointerScenario_crash(JNIEnv *env,
                                                                            jobject instance) {
@@ -85,18 +176,22 @@ Java_com_bugsnag_android_mazerunner_scenarios_CXXTrapScenario_crash(JNIEnv *env,
   printf("This one here: %ld\n", (long) crash_trap());
 }
 
+JNIEXPORT void JNICALL
+Java_com_bugsnag_android_mazerunner_scenarios_CXXDoubleFreeScenario_crash(JNIEnv *env,
+                                                                                   jobject instance) {
+    printf("This one here: %d\n", crash_double_free(42));
+}
 
 JNIEXPORT void JNICALL
-Java_com_bugsnag_android_mazerunner_scenarios_CXXUseAfterFreeScenario_crash(JNIEnv *env,
-                                                                            jobject instance,
-                                                                            jint counter) {
-    printf("This one here: %ld\n", (long) crash_released_obj((int)counter));
+Java_com_bugsnag_android_mazerunner_scenarios_CXXWriteReadOnlyMemoryScenario_crash(JNIEnv *env,
+                                                                                   jobject instance) {
+    printf("This one here: %d\n", crash_write_read_only_mem(42));
 }
 
 JNIEXPORT void JNICALL
-Java_com_bugsnag_android_mazerunner_scenarios_CXXUndefinedInstructionScenario_crash(JNIEnv *env,
-                                                                            jobject instance) {
-    printf("This one here: %s\n", somefakefunc());
+Java_com_bugsnag_android_mazerunner_scenarios_CXXImproperTypecastScenario_crash(JNIEnv *env,
+                                                                                jobject instance) {
+    printf("This one here: %s\n", crash_improper_cast(39));
 }
 
 JNIEXPORT void JNICALL

diff --git a/...rios/CXXUndefinedInstructionScenario.java → ...nner/scenarios/CXXDoubleFreeScenario.java b/...rios/CXXUndefinedInstructionScenario.java → ...nner/scenarios/CXXDoubleFreeScenario.java
@@ -7,7 +7,7 @@
 
 import android.support.annotation.NonNull;
 
-public class CXXUndefinedInstructionScenario extends Scenario {
+public class CXXDoubleFreeScenario extends Scenario {
 
     static {
         System.loadLibrary("bugsnag-ndk");
@@ -16,7 +16,7 @@ public class CXXUndefinedInstructionScenario extends Scenario {
 
     public native void crash();
 
-    public CXXUndefinedInstructionScenario(@NonNull Configuration config, @NonNull Context context) {
+    public CXXDoubleFreeScenario(@NonNull Configuration config, @NonNull Context context) {
         super(config, context);
         config.setAutoCaptureSessions(false);
     }

diff --git a/...er/scenarios/CXXUseAfterFreeScenario.java → ...cenarios/CXXImproperTypecastScenario.java b/...er/scenarios/CXXUseAfterFreeScenario.java → ...cenarios/CXXImproperTypecastScenario.java
@@ -1,21 +1,22 @@
 package com.bugsnag.android.mazerunner.scenarios;
 
+
 import android.content.Context;
 
 import com.bugsnag.android.Configuration;
 
 import android.support.annotation.NonNull;
 
-public class CXXUseAfterFreeScenario extends Scenario {
+public class CXXImproperTypecastScenario extends Scenario {
 
     static {
         System.loadLibrary("bugsnag-ndk");
         System.loadLibrary("entrypoint");
     }
 
-    public native void crash(int counter);
+    public native void crash();
 
-    public CXXUseAfterFreeScenario(@NonNull Configuration config, @NonNull Context context) {
+    public CXXImproperTypecastScenario(@NonNull Configuration config, @NonNull Context context) {
         super(config, context);
         config.setAutoCaptureSessions(false);
     }
@@ -27,6 +28,6 @@ public void run() {
         if (metadata != null && metadata.equals("non-crashy")) {
             return;
         }
-        crash(23);
+        crash();
     }
 }
diff --git a/...mazerunner/src/main/java/com/bugsnag/android/mazerunner/scenarios/CXXSigabrtScenario.java b/...mazerunner/src/main/java/com/bugsnag/android/mazerunner/scenarios/CXXSigabrtScenario.java
@@ -0,0 +1,32 @@
+package com.bugsnag.android.mazerunner.scenarios;
+
+import android.content.Context;
+
+import com.bugsnag.android.Configuration;
+
+import android.support.annotation.NonNull;
+
+public class CXXSigabrtScenario extends Scenario {
+
+    static {
+        System.loadLibrary("bugsnag-ndk");
+        System.loadLibrary("entrypoint");
+    }
+
+    public native int crash(int value);
+
+    public CXXSigabrtScenario(@NonNull Configuration config, @NonNull Context context) {
+        super(config, context);
+        config.setAutoCaptureSessions(false);
+    }
+
+    @Override
+    public void run() {
+        super.run();
+        String metadata = getEventMetaData();
+        if (metadata != null && metadata.equals("non-crashy")) {
+            return;
+        }
+        crash(2726);
+    }
+}
diff --git a/.../mazerunner/src/main/java/com/bugsnag/android/mazerunner/scenarios/CXXSigbusScenario.java b/.../mazerunner/src/main/java/com/bugsnag/android/mazerunner/scenarios/CXXSigbusScenario.java
@@ -0,0 +1,32 @@
+package com.bugsnag.android.mazerunner.scenarios;
+
+import android.content.Context;
+
+import com.bugsnag.android.Configuration;
+
+import android.support.annotation.NonNull;
+
+public class CXXSigbusScenario extends Scenario {
+
+    static {
+        System.loadLibrary("bugsnag-ndk");
+        System.loadLibrary("entrypoint");
+    }
+
+    public native int crash(int value);
+
+    public CXXSigbusScenario(@NonNull Configuration config, @NonNull Context context) {
+        super(config, context);
+        config.setAutoCaptureSessions(false);
+    }
+
+    @Override
+    public void run() {
+        super.run();
+        String metadata = getEventMetaData();
+        if (metadata != null && metadata.equals("non-crashy")) {
+            return;
+        }
+        crash(2726);
+    }
+}
diff --git a/.../mazerunner/src/main/java/com/bugsnag/android/mazerunner/scenarios/CXXSigfpeScenario.java b/.../mazerunner/src/main/java/com/bugsnag/android/mazerunner/scenarios/CXXSigfpeScenario.java
@@ -0,0 +1,32 @@
+package com.bugsnag.android.mazerunner.scenarios;
+
+import android.content.Context;
+
+import com.bugsnag.android.Configuration;
+
+import android.support.annotation.NonNull;
+
+public class CXXSigfpeScenario extends Scenario {
+
+    static {
+        System.loadLibrary("bugsnag-ndk");
+        System.loadLibrary("entrypoint");
+    }
+
+    public native int crash(int value);
+
+    public CXXSigfpeScenario(@NonNull Configuration config, @NonNull Context context) {
+        super(config, context);
+        config.setAutoCaptureSessions(false);
+    }
+
+    @Override
+    public void run() {
+        super.run();
+        String metadata = getEventMetaData();
+        if (metadata != null && metadata.equals("non-crashy")) {
+            return;
+        }
+        crash(2726);
+    }
+}
diff --git a/.../mazerunner/src/main/java/com/bugsnag/android/mazerunner/scenarios/CXXSigillScenario.java b/.../mazerunner/src/main/java/com/bugsnag/android/mazerunner/scenarios/CXXSigillScenario.java
@@ -0,0 +1,32 @@
+package com.bugsnag.android.mazerunner.scenarios;
+
+import android.content.Context;
+
+import com.bugsnag.android.Configuration;
+
+import android.support.annotation.NonNull;
+
+public class CXXSigillScenario extends Scenario {
+
+    static {
+        System.loadLibrary("bugsnag-ndk");
+        System.loadLibrary("entrypoint");
+    }
+
+    public native int crash(int value);
+
+    public CXXSigillScenario(@NonNull Configuration config, @NonNull Context context) {
+        super(config, context);
+        config.setAutoCaptureSessions(false);
+    }
+
+    @Override
+    public void run() {
+        super.run();
+        String metadata = getEventMetaData();
+        if (metadata != null && metadata.equals("non-crashy")) {
+            return;
+        }
+        crash(2726);
+    }
+}