Add option to prevent uploading results for Smart Flank

README.md

@@ -131,6 +131,11 @@ flank:
   ## Google cloud storage path to store the JUnit XML results from the last run.
   # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_ios.xml
 
+  ## Whether the JUnit XML results should be uploaded for Smart Flank.
+  ## This can be disabled to prevent new results from overriding previous results.
+  ## Default: true
+  # smart-flank-upload-enabled: true
+
   ## Disables sharding. Useful for parameterized tests.
   # disable-sharding: false
 
@@ -274,6 +279,11 @@ flank:
   ## Google cloud storage path to store the JUnit XML results from the last run.
   # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_android.xml
 
+  ## Whether the JUnit XML results should be uploaded for Smart Flank.
+  ## This can be disabled to prevent new results from overriding previous results.
+  ## Default: true
+  # smart-flank-upload-enabled: true
+
   ## Disables sharding. Useful for parameterized tests.
   # disable-sharding: false
 

docs/smart_flank.md

@@ -65,4 +65,3 @@ Example:
 
 * Keep a user configurable rolling number of aggregated xmls (1.xml, 2.xml, 3.xml) and shard based on the average time. Average time is expected to be more reliable than always using the last time in isolation.
 * Identify a way of translating app binary to a default xml name (bundle id/package name) so that smart flank works out of the box for users. Talk with Firebase on how to do this locally and/or expose an API.
-* Provide an option to disable uploading so that local runs don't update time information for CI/pull request runs.

release_notes.md

@@ -1,6 +1,6 @@
 ## next (unreleased)
 
-- 
+- [#537](https://github.com/TestArmada/flank/pull/537) Add `smart-flank-upload-enabled` yml option to prevent new results from overriding previous results. ([elihart](https://github.com/elihart))
 
 ## v5.0.1
 

test_runner/flank.ios.yml

@@ -82,6 +82,11 @@ flank:
   ## Google cloud storage path to store the JUnit XML results from the last run.
   # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_ios.xml
 
+  ## Whether the JUnit XML results should be uploaded for Smart Flank.
+  ## This can be disabled to prevent new results from overriding previous results.
+  ## Default: true
+  # smart-flank-upload-enabled: true
+
   ## Disables sharding. Useful for parameterized tests.
   # disable-sharding: false
 

test_runner/flank.yml

@@ -107,6 +107,11 @@ flank:
   ## Google cloud storage path to store the JUnit XML results from the last run.
   # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_android.xml
 
+  ## Whether the JUnit XML results should be uploaded for Smart Flank.
+  ## This can be disabled to prevent new results from overriding previous results.
+  ## Default: true
+  # smart-flank-upload-enabled: true
+
   ## Disables sharding. Useful for parameterized tests.
   # disable-sharding: false
 

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -69,6 +69,7 @@ class AndroidArgs(
     override val shardTime = cli?.shardTime ?: flank.shardTime
     override val repeatTests = cli?.repeatTests ?: flank.repeatTests
     override val smartFlankGcsPath = flank.smartFlankGcsPath
+    override val smartFlankUploadEnabled = cli?.smartFlankUploadEnabled ?: flank.smartFlankUploadEnabled
     override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun
     override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload
     override val disableSharding = cli?.disableSharding ?: flank.disableSharding
@@ -169,6 +170,7 @@ ${devicesToString(devices)}
       shard-time: $shardTime
       repeat-tests: $repeatTests
       smart-flank-gcs-path: $smartFlankGcsPath
+      smart-flank-upload-enabled: $smartFlankUploadEnabled
       files-to-download:
 ${listToString(filesToDownload)}
       test-targets-always-run:

test_runner/src/main/kotlin/ftl/args/IArgs.kt

@@ -21,6 +21,7 @@ interface IArgs {
     val shardTime: Int
     val repeatTests: Int
     val smartFlankGcsPath: String
+    val smartFlankUploadEnabled: Boolean
     val testTargetsAlwaysRun: List<String>
     val filesToDownload: List<String>
     val disableSharding: Boolean

test_runner/src/main/kotlin/ftl/args/IosArgs.kt

@@ -52,6 +52,7 @@ class IosArgs(
     override val shardTime = cli?.shardTime ?: flank.shardTime
     override val repeatTests = cli?.repeatTests ?: flank.repeatTests
     override val smartFlankGcsPath = flank.smartFlankGcsPath
+    override val smartFlankUploadEnabled = cli?.smartFlankUploadEnabled ?: flank.smartFlankUploadEnabled
     override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun
     override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload
     override val disableSharding = cli?.disableSharding ?: flank.disableSharding
@@ -124,6 +125,7 @@ ${devicesToString(devices)}
       shard-time: $shardTime
       repeat-tests: $repeatTests
       smart-flank-gcs-path: $smartFlankGcsPath
+      smart-flank-upload-enabled: $smartFlankUploadEnabled
       test-targets-always-run:
 ${listToString(testTargetsAlwaysRun)}
       files-to-download:

test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt

@@ -23,6 +23,9 @@ class FlankYmlParams(
     @field:JsonProperty("smart-flank-gcs-path")
     val smartFlankGcsPath: String = "",
 
+    @field:JsonProperty("smart-flank-upload-enabled")
+    val smartFlankUploadEnabled: Boolean = true,
+
     @field:JsonProperty("disable-sharding")
     val disableSharding: Boolean = false,
 
@@ -39,8 +42,8 @@ class FlankYmlParams(
 ) {
     companion object : IYmlKeys {
         override val keys = listOf(
-            "max-test-shards", "shard-time", "repeat-tests", "smart-flank-gcs-path", "disable-sharding",
-            "test-targets-always-run", "files-to-download", "project", "local-result-dir"
+            "max-test-shards", "shard-time", "repeat-tests", "smart-flank-gcs-path", "smart-flank-upload-enabled",
+            "disable-sharding", "test-targets-always-run", "files-to-download", "project", "local-result-dir"
         )
 
         const val defaultLocalResultDir = "results"

test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt

@@ -279,4 +279,10 @@ class AndroidRunCommand : Runnable {
         description = ["Saves test result to this local folder. Deleted before each run."]
     )
     var localResultDir: String? = null
+
+    @Option(
+        names = ["--smart-flank-upload-enabled"],
+        description = ["Whether the JUnit XML results should be uploaded to the Smart Flank cloud storage."]
+    )
+    var smartFlankUploadEnabled: Boolean? = null
 }

test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt

@@ -220,4 +220,10 @@ class IosRunCommand : Runnable {
         description = ["Saves test result to this local folder. Deleted before each run."]
     )
     var localResultsDir: String? = null
+
+    @Option(
+        names = ["--smart-flank-upload-enabled"],
+        description = ["Whether the JUnit XML results should be uploaded to the Smart Flank cloud storage."]
+    )
+    var smartFlankUploadEnabled: Boolean? = null
 }

test_runner/src/main/kotlin/ftl/gc/GcStorage.kt

@@ -54,7 +54,7 @@ object GcStorage {
         )
 
     fun uploadJunitXml(testResult: JUnitTestResult, args: IArgs) {
-        if (args.smartFlankGcsPath.isEmpty()) return
+        if (args.smartFlankGcsPath.isEmpty() || !args.smartFlankUploadEnabled) return
 
         // bucket/path/to/object
         val rawPath = args.smartFlankGcsPath.drop(GCS_PREFIX.length)

test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

@@ -218,6 +218,7 @@ AndroidArgs
       shard-time: 60
       repeat-tests: 8
       smart-flank-gcs-path:${' '}
+      smart-flank-upload-enabled: true
       files-to-download:
         - /sdcard/screenshots
         - /sdcard/screenshots2
@@ -813,4 +814,22 @@ AndroidArgs
         val androidArgs = AndroidArgs.load(yaml, cli)
         assertThat(androidArgs.localResultDir).isEqualTo("b")
     }
+
+    @Test
+    fun `cli smart-flank-upload-enabled`() {
+        val cli = AndroidRunCommand()
+        CommandLine(cli).parse("--smart-flank-upload-enabled=false")
+
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+        flank:
+          smart-flank-upload-enabled: true
+      """
+        assertThat(AndroidArgs.load(yaml).smartFlankUploadEnabled).isEqualTo(true)
+
+        val androidArgs = AndroidArgs.load(yaml, cli)
+        assertThat(androidArgs.smartFlankUploadEnabled).isEqualTo(false)
+    }
 }

test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt

@@ -167,6 +167,7 @@ IosArgs
       shard-time: 60
       repeat-tests: 8
       smart-flank-gcs-path:${' '}
+      smart-flank-upload-enabled: true
       test-targets-always-run:
         - a/testGrantPermissions
         - a/testGrantPermissions2
@@ -634,6 +635,22 @@ IosArgs
         assertThat(androidArgs.localResultDir).isEqualTo("a")
     }
 
+    @Test
+    fun `cli smart-flank-upload-enabled`() {
+        val cli = IosRunCommand()
+        CommandLine(cli).parse("--smart-flank-upload-enabled=false")
+
+        val yaml = """
+        gcloud:
+          test: $testPath
+          xctestrun-file: $testPath
+      """
+        assertThat(IosArgs.load(yaml).smartFlankUploadEnabled).isEqualTo(true)
+
+        val androidArgs = IosArgs.load(yaml, cli)
+        assertThat(androidArgs.smartFlankUploadEnabled).isEqualTo(false)
+    }
+
     private fun getValidTestsSample() = listOf(
         "ClassOneTest/testOne",
         "ClassOneTest/testTwo",

test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt

@@ -88,6 +88,7 @@ class AndroidRunCommandTest {
         assertThat(cmd.flakyTestAttempts).isNull()
         assertThat(cmd.disableSharding).isNull()
         assertThat(cmd.localResultDir).isNull()
+        assertThat(cmd.smartFlankUploadEnabled).isNull()
     }
 
     @Test
@@ -320,4 +321,12 @@ class AndroidRunCommandTest {
 
         assertThat(cmd.localResultDir).isEqualTo("a")
     }
+
+    @Test
+    fun `smartFlankUploadEnabled parse`() {
+        val cmd = AndroidRunCommand()
+        CommandLine(cmd).parse("--smart-flank-upload-enabled=false")
+
+        assertThat(cmd.smartFlankUploadEnabled).isEqualTo(false)
+    }
 }

test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt

@@ -82,6 +82,7 @@ class IosRunCommandTest {
         assertThat(cmd.resultsDir).isNull()
         assertThat(cmd.flakyTestAttempts).isNull()
         assertThat(cmd.localResultsDir).isNull()
+        assertThat(cmd.smartFlankUploadEnabled).isNull()
     }
 
     @Test
@@ -255,4 +256,12 @@ class IosRunCommandTest {
 
         assertThat(cmd.localResultsDir).isEqualTo("a")
     }
+
+    @Test
+    fun `smart-flank-upload-enabled parse`() {
+        val cmd = IosRunCommand()
+        CommandLine(cmd).parse("--smart-flank-upload-enabled=false")
+
+        assertThat(cmd.smartFlankUploadEnabled).isEqualTo(false)
+    }
 }