diff --git a/automation/cloud_build_api/src/main/kotlin/api/bitrise/private/RollingBuilds.kt b/automation/cloud_build_api/src/main/kotlin/api/bitrise/private/RollingBuilds.kt index fac4b518b0..c2c1eab93b 100644 --- a/automation/cloud_build_api/src/main/kotlin/api/bitrise/private/RollingBuilds.kt +++ b/automation/cloud_build_api/src/main/kotlin/api/bitrise/private/RollingBuilds.kt @@ -19,6 +19,7 @@ package api.bitrise.private import api.bitrise.private.CookieRetrofit.privateCookieRetrofit import retrofit2.Call +import retrofit2.Response import retrofit2.http.* object RollingBuilds { @@ -41,30 +42,46 @@ object RollingBuilds { privateCookieRetrofit.create(ApiService::class.java) } + // 404 error - means BITRISE_USER isn't authorized to access that app. + // TODO: move this to an interceptor + private fun Response.checkError(): Response { + val response = this + val request = response.raw().request() + + if (response.isSuccessful.not()) { + throw RuntimeException(""" + ${request.method()} ${request.url()} ${response.code()} ${response.message()} + ${response.errorBody()?.string()} + """.trimIndent()) + } + + return response + } + fun getConfig(appSlug: String): RollingBuildsConfig { - return apiService.getConfig(appSlug).execute().body() ?: throw RuntimeException("getConfig failed") + return apiService.getConfig(appSlug).execute().checkError().body() ?: throw RuntimeException("getConfig failed") } fun setConfigPR(appSlug: String, enabled: Boolean): RollingBuildsConfig { val config = RollingBuildsPatch("pr", enabled) - return apiService.setConfig(appSlug, config).execute().body() ?: throw RuntimeException("setConfigPR failed") + return apiService.setConfig(appSlug, config).execute().checkError().body() ?: throw RuntimeException("setConfigPR failed") } fun setConfigPush(appSlug: String, enabled: Boolean): RollingBuildsConfig { val config = RollingBuildsPatch("push", enabled) - return apiService.setConfig(appSlug, config).execute().body() ?: throw RuntimeException("setConfigPush failed") + return apiService.setConfig(appSlug, config).execute().checkError().body() ?: throw RuntimeException("setConfigPush failed") } fun setConfigRunning(appSlug: String, enabled: Boolean): RollingBuildsConfig { val config = RollingBuildsPatch("running", enabled) - return apiService.setConfig(appSlug, config).execute().body() ?: throw RuntimeException("setConfigRunning failed") + return apiService.setConfig(appSlug, config).execute().checkError().body() ?: throw RuntimeException("setConfigRunning failed") } fun disable(appSlug: String): Status { - return apiService.disable(appSlug).execute().body() ?: Status(null, null) + return apiService.disable(appSlug).execute().checkError().body() ?: Status(null, null) } fun enable(appSlug: String): Status { - return apiService.enable(appSlug).execute().body() ?: Status(null, null) + return apiService.enable(appSlug).execute().checkError().body() ?: Status(null, null) } } diff --git a/automation/cloud_build_metrics/docs/BackupBitriseYamls.md b/automation/cloud_build_metrics/docs/BackupBitriseYamls.md new file mode 100644 index 0000000000..7c723c6802 --- /dev/null +++ b/automation/cloud_build_metrics/docs/BackupBitriseYamls.md @@ -0,0 +1,111 @@ +# BackupBitriseYamls + +## Objective + +All workflows should be backed up to an encrypted git repository. + +![](./png/keybase_git_repo.png) + +## Solution + +The [Keybase client](https://github.com/keybase/client) provides a cloud encrypted git repo. +Using the bitrise API, all YAMLs are fetched and then committed to the keybase git repo. +See [BackupBitriseYamls.kt][1] for the full source. Note that you should create a paper key when using keybase. +Otherwise if the device you installed keybase on is lost, you won't be able to login anymore. + +```kotlin +override fun execute() { + val apps = BitriseApps.getAppsForOrg() + val appsMap = appsMap(apps) + val repo = keybaseRepo() + writeYamls(repo, apps) +} +``` + +[1]: https://github.com/instructure/canvas-android/blob/f455db88520d37be007af2f7b9e36d17e45182f5/automation/cloud_build_metrics/src/main/kotlin/tasks/BackupBitriseYamls.kt + +The update happens in a scheduled nightly job on Bitrise using this workflow: + +```yaml +prepare_keybase: + envs: + - KEYBASE_PAPERKEY: ... + opts: + is_expand: false + - KEYBASE_USERNAME: ... + opts: + is_expand: false + steps: + - script: + inputs: + - content: | + #!/bin/bash + set -e + + if [ ! -d keybase ]; then + echo "Downloading keybase..." + curl --silent --output keybase.deb https://prerelease.keybase.io/keybase_amd64.deb + echo "Installing keybase deb" + mkdir keybase + set +e + dpkg -i keybase.deb + apt-get install -f + dpkg -i keybase.deb + set -e + else + echo "Keybase exists!" + fi + title: Keybase - Download + - script: + inputs: + - content: | + #!/usr/bin/env bash + set -ex + + export KEYBASE_ALLOW_ROOT=1 + + keybase version + keybase oneshot + title: Keybase - Login + - script: + inputs: + - content: |- + #!/usr/bin/env bash + set -ex + + git clone keybase://team/team/secrets ../../../secrets + title: Keybase - git clone repo +BackupBitriseYamls: + after_run: [] + before_run: + - setup_workflow + - prepare_keybase + steps: + - gradle-runner: + inputs: + - gradle_task: clean fatJar + - gradle_file: "$BITRISE_SOURCE_DIR/automation/cloud_build_metrics/build.gradle.kts" + - gradlew_path: "$BITRISE_SOURCE_DIR/automation/cloud_build_metrics/gradlew" + - cache_level: all + - script: + inputs: + - content: | + #!/bin/bash + set -euxo pipefail + java -version + java -jar "$BITRISE_SOURCE_DIR/automation/cloud_build_metrics/build/libs/cloud_build_metrics-all-1.0-SNAPSHOT.jar" BackupBitriseYamls + title: Backup Bitrise Yamls + - script: + inputs: + - content: | + #!/usr/bin/env bash + set -ex + + if [[ -n $(git -C ../../../secrets status -s) ]]; then + DATE=`date '+%Y-%m-%d %H:%M:%S'` + git -C ../../../secrets add . + git -C ../../../secrets commit -m "update bitrise yamls $DATE" + git -C ../../../secrets push + fi + title: Keybase - git commit & push +``` diff --git a/automation/cloud_build_metrics/docs/BitriseCacheRefresh.md b/automation/cloud_build_metrics/docs/BitriseCacheRefresh.md new file mode 100644 index 0000000000..c1d15b1a79 --- /dev/null +++ b/automation/cloud_build_metrics/docs/BitriseCacheRefresh.md @@ -0,0 +1,69 @@ +# BitriseCacheRefresh + +## Objective + +Automatically ensure the Bitrise cache is fresh. + +``` +App title renamed android-teacher -> Android Teacher. Verify and update. +App title renamed android-parent -> Android Parent. Verify and update. +App title renamed android-student -> Android Student. Verify and update. +App title renamed Android Teacher Espresso -> Android Teacher UI Tests. Verify and update. +App title renamed cloud_build_metrics -> Cloud Build Metrics. Verify and update. + +tasks.BitriseCacheRefresh is running... + +Android Teacher build cache deleted + 1 GB - master +Android Parent build cache deleted + 872 MB - master +Android Student build cache deleted + 1 GB - master +android-polling build cache deleted + 782 MB - master +data seeding api build cache deleted + 694 MB - master +Android Teacher UI Tests build cache deleted + 1 GB - master +Cloud Build Metrics build cache deleted + 709 MB - master +Triggering build for Android Teacher using workflow debug + https://app.bitrise.io/build/48d1dc6727b9ac5b +Triggering build for Android Parent using workflow debug + https://app.bitrise.io/build/d5ae8a3d0c29d745 +Triggering build for Android Student using workflow debug + https://app.bitrise.io/build/95b5256a92888729 +Triggering build for android-polling using workflow debug + https://app.bitrise.io/build/217aca865c4872ac +Triggering build for data seeding api using workflow primary + https://app.bitrise.io/build/989f8637c8cf06da +Triggering build for Android Teacher UI Tests using workflow primary + https://app.bitrise.io/build/3a4dbc026d5667f1 +Triggering build for Cloud Build Metrics using workflow RunUnitTests + https://app.bitrise.io/build/85574c05121ad050 + +Process finished with exit code 0 +``` + +## Solution + +Using the Bitrise private API, we’re able to programmatically delete caches for all jobs that run on pull requests. +To populate the cache, we need to figure out the default workflow for each app. There’s no API support for this, +so the YAML file is downloaded and parsed directly. After determining the workflow, we use Bitrise’s new public API +for scheduling a build. The cache refresh task runs nightly. Each business day, we’re starting with a freshly optimized +cache. This has been a great win for optimizing build times. + +```kotlin +override fun execute() { + signIn() + deleteCaches() + populateCaches() +} +``` + +See [BitriseCacheRefresh.kt][1] for the full source. + +[1]: https://github.com/instructure/canvas-android/blob/f455db88520d37be007af2f7b9e36d17e45182f5/automation/cloud_build_metrics/src/main/kotlin/tasks/BitriseCacheRefresh.kt + +There's an open feature request to [Improve cache reliability](https://discuss.bitrise.io/t/rethink-the-cache-system-to-be-more-reliable/3290). + diff --git a/automation/cloud_build_metrics/docs/BitriseCacheReport.md b/automation/cloud_build_metrics/docs/BitriseCacheReport.md new file mode 100644 index 0000000000..b331202c6b --- /dev/null +++ b/automation/cloud_build_metrics/docs/BitriseCacheReport.md @@ -0,0 +1,60 @@ +# BitriseCacheReport + +## Objective + +Print the cache size for every app. Monitoring cache size helps ensure builds stay fast. + +``` +tasks.BitriseCacheReport is running... + +Android Student UI Tests + 1 GB - master +Android Parent UI Tests + 945 MB - master +data seeding api + 694 MB - master +Canvas iOS Danger + 3 GB - master +Cloud Build Metrics + 709 MB - master +Android Teacher UI Tests + 1 GB - master +android-polling + 782 MB - master +Android Parent + 872 MB - master +Android Teacher + 1 GB - master +Android Student + 1 GB - master + +Process finished with exit code 0 +``` + +## Solution + +The Bitrise private API enables generating an organization wide cache report for each job. This has been helpful in +making sure jobs are cached appropriately and ensuring we’re staying under the 2GB limit. + +```kotlin +override fun execute() { + signIn() + val apps = BitriseApps.getOnlyInstructureApps() + + for (app in apps) { + val cache = BuildCache.get(app.slug) + if (cache.isNotEmpty()) { + println(app.title) + for (item in cache) { + val size = item.file_size_bytes.humanReadable() + val branch = item.the_cache_item_key + println(" $size - $branch") + } + } + } +} +``` + +See [BitriseCacheReport.kt][1] for the full source. + +[1]: https://github.com/instructure/canvas-android/blob/f455db88520d37be007af2f7b9e36d17e45182f5/automation/cloud_build_metrics/src/main/kotlin/tasks/BitriseCacheReport.kt diff --git a/automation/cloud_build_metrics/docs/BitriseSetRollingBuilds.md b/automation/cloud_build_metrics/docs/BitriseSetRollingBuilds.md new file mode 100644 index 0000000000..d14e4cd81f --- /dev/null +++ b/automation/cloud_build_metrics/docs/BitriseSetRollingBuilds.md @@ -0,0 +1,55 @@ +# BitriseSetRollingBuilds + +## Objective + +Automatically set rolling builds on all apps. + +``` +tasks.BitriseSetRollingBuilds is running... + +Updating build config for: earlgrey-2-binary-example +Updating build config for: gwiz-bot +Updated 2 of 50 apps + +Process finished with exit code 0 +``` + +## Solution + +The Bitrise private API allows us to set rolling builds for every app. This represents a great time savings for our team, + as previously this was done manually. + +```kotlin +override fun execute() { + signIn() + val apps = BitriseApps.getOnlyInstructureApps() + + var updatedCount = 0 + for (app in apps) { + val appSlug = app.slug + + val config = try { + RollingBuilds.getConfig(appSlug) + } catch (e: Exception) { + RollingBuilds.enable(appSlug) + RollingBuilds.getConfig(appSlug) + } + + if (config.pr && config.push && config.running) continue + + println("Updating build config for: ${app.title}") + updatedCount += 1 + + RollingBuilds.enable(appSlug) + RollingBuilds.setConfigPR(appSlug, true) + RollingBuilds.setConfigPush(appSlug, true) + RollingBuilds.setConfigRunning(appSlug, true) + } + + println("Updated $updatedCount of ${apps.size} apps") +} +``` + +See [BitriseSetRollingBuilds.kt][1] for the full source. + +[1]: https://github.com/instructure/canvas-android/blob/f455db88520d37be007af2f7b9e36d17e45182f5/automation/cloud_build_metrics/src/main/kotlin/tasks/BitriseSetRollingBuilds.kt diff --git a/automation/cloud_build_metrics/docs/BuildActivityReport.md b/automation/cloud_build_metrics/docs/BuildActivityReport.md new file mode 100644 index 0000000000..85653878cc --- /dev/null +++ b/automation/cloud_build_metrics/docs/BuildActivityReport.md @@ -0,0 +1,61 @@ +# BuildActivityReport + +## Objective + +Print all job activity. Enables easy identification of orphaned jobs. + +``` +Bitrise app count 58 +Canvas LMS Docker 4 minutes ago +data seeding api 12 minutes ago +Cloud Build Metrics 12 minutes ago +Android Teacher UI Tests 12 minutes ago +android-polling 12 minutes ago +Android Parent 12 minutes ago +Android Teacher 12 minutes ago +Android Student 12 minutes ago +Android Student UI Tests 2 hours ago +Android Parent UI Tests 2 hours ago +Android Open Source Parent 2 hours ago +Android Open Source Student 2 hours ago +Android Open Source Teacher 2 hours ago +Canvas iOS Translations 3 hours ago +Canvas iOS Lint New Student 14 hours ago +Canvas iOS Student UI Tests 14 hours ago +iOS Open Source Parent 14 hours ago +iOS Open Source Student 14 hours ago +iOS Open Source Teacher 14 hours ago +Canvas iOS Teacher 14 hours ago +Canvas iOS Parent 14 hours ago +Canvas iOS Student 14 hours ago +Canvas iOS Danger 14 hours ago +mobile-shared Espresso 1 months ago +Android Pact 4 months ago +SoSeedy Kubernetes 5 months ago +earlgrey-2-binary-example 5 months ago +mobile android 8 months ago +expo-ui-test 9 months ago +Test Job for Cloud Build Metrics 12 months ago +android-teacher-robo 12 months ago +android-parent-robo 12 months ago +Create React App TypeScript 17 months ago + + +Process finished with exit code 0 +``` + +## Solution + +The build activity report allows us to easily identify jobs that are abandoned based on the last build time. +It’s great being able to see exactly what jobs are active and inactive. Pruning inactive jobs was a bit painful +before this report existed. + +```kotlin +override fun execute() { + jobActivityReport(BitriseApps) +} +``` + +See [BuildActivityReport.kt][1] for the full source. + +[1]: https://github.com/instructure/canvas-android/blob/f455db88520d37be007af2f7b9e36d17e45182f5/automation/cloud_build_metrics/src/main/kotlin/tasks/BuildActivityReport.kt diff --git a/automation/cloud_build_metrics/docs/BuildErrorReport.md b/automation/cloud_build_metrics/docs/BuildErrorReport.md new file mode 100644 index 0000000000..2996f7f518 --- /dev/null +++ b/automation/cloud_build_metrics/docs/BuildErrorReport.md @@ -0,0 +1,68 @@ +# BuildErrorReport + +## Objective + +Identify which step builds are failing on. + +``` +Found 58 apps in org +Building error report for Canvas iOS Student UI Tests +Monday, January 7 - Sunday, January 5 +"Canvas iOS Student UI Tests" has 175 failed builds out of 265 builds +Downloading: 829237e811f23cf7 to: /var/folders/14/h2zk5sx1315b__blzdt7hz1m4k0by8/T/57b22415f35222e3 +Downloading: 79eaa06117b38494 to: /var/folders/14/h2zk5sx1315b__blzdt7hz1m4k0by8/T/57b22415f35222e3 +... +Parsing errors from 175 logs +https://www.bitrise.io/build/829237e811f23cf7, Build Student app (exit code: 65) +https://www.bitrise.io/build/79eaa06117b38494, Build Student app (exit code: 65) +https://www.bitrise.io/build/7eb299b86be1f66f, Build Student app (exit code: 65) +https://www.bitrise.io/build/e1fe31d9a2ea410b, Build Student app (exit code: 65) +https://www.bitrise.io/build/c48267e4bde4b399, flank (exit code: 1) +https://www.bitrise.io/build/9542b9372c434da1, flank (exit code: 1) +https://www.bitrise.io/build/b5da9e738dd073b1, flank (exit code: 1) +https://www.bitrise.io/build/fdf9db2a3a4e7f46, flank (exit code: 1) +https://www.bitrise.io/build/fdf9db2a3a4e7f46, deploy-to-bitrise-io (exit code: 1) +https://www.bitrise.io/build/56cede26cfd592e8, flank (exit code: 1) +https://www.bitrise.io/build/7dd2a1abe783a2e8, Build Student app (exit code: 65) +https://www.bitrise.io/build/8ee15b609d9f3d93, flank (exit code: 2) +https://www.bitrise.io/build/c521fb889e4e0ddb, flank (exit code: 2) +https://www.bitrise.io/build/1f6f0caeefbe4b88, flank (exit code: 2) +https://www.bitrise.io/build/d2331dc187eda13f, flank (exit code: 1) +https://www.bitrise.io/build/eab7e551e9dd9f67, Build Student app (exit code: 65) +https://www.bitrise.io/build/31e0301c85282bbf, Build Student app (exit code: 65) + + +Process finished with exit code 0 +``` + +## Solution + +The Bitrise API allows us to download and parse the build output for each failed job. In addition to pinpointing exactly + which steps failed, we’re also able to get additional details on the error message via a regular expression. + Automating error reporting has saved us a significant amount of time when diagnosing infrastructure failures + identified by the build graphs. Previously, an engineer would have to manually click through each build and + figure out what step failed. + +```kotlin +override fun execute() { + val targetAppSlug = "57b22415f35222e3" + val appsInOrg = BitriseApps.getAppsForOrg() + + println("Found ${appsInOrg.size} apps in org") + val app = appsInOrg.first { it.slug == targetAppSlug } + + println("Building error report for ${app.title}") + val weekStart = LocalDate.parse("Monday, January 7 2019", prettyDateTimeYear) + val range = weekRange(52, weekStart) + + println("${range.after.pretty()} - ${range.before.pretty()}") + buildErrorReport(app, + deleteCache = false, + limitAfter = range.limitAfter, + limitBefore = range.limitBefore) +} +``` + +See [BuildErrorReport.kt][1] for the full source. + +[1]: https://github.com/instructure/canvas-android/blob/f455db88520d37be007af2f7b9e36d17e45182f5/automation/cloud_build_metrics/src/main/kotlin/tasks/BuildErrorReport.kt diff --git a/automation/cloud_build_metrics/docs/LintBitriseYamls.md b/automation/cloud_build_metrics/docs/LintBitriseYamls.md new file mode 100644 index 0000000000..e9e0ab43b6 --- /dev/null +++ b/automation/cloud_build_metrics/docs/LintBitriseYamls.md @@ -0,0 +1,53 @@ +# LintBitriseYamls + +## Objective + +Define global lint rules to identify problems in workflows. + +``` +Linting 50 apps in org +Exception in thread "main" java.lang.RuntimeException: 💥 Bitrise steps with pinned versions found! + +canvas iOS UI Tests (21c95f472d50f580) + brew-install@0.9.0 +canvas-android (c7219e86fcecb15f) + gradle-runner@1.8.4 + sonarqube-scanner@1.0.5 +canvas-ios unit tests (9b66b2e82404d2f7) + brew-install@0.9.0 + codecov@1.1.5 +canvas-ios (94c7e4f1b42e88c7) + brew-install@0.9.0 + slack@3.1.2 + + at tasks.LintBitriseYamls.checkWarnings$cloud_build_metrics_main(LintBitriseYamls.kt:38) + at tasks.LintBitriseYamls.execute(LintBitriseYamls.kt:53) + at tasks.LintBitriseYamls.main(LintBitriseYamls.kt:58) + +``` + +## Solution + +The Bitrise API allows us to fetch all YAML files and assert they’re following best practices. +By default, Bitrise uses old pinned versions of steps (for example git clone). +As new improvements and fixes are made, none of the workflows benefit from the new versions. +The linter proactively identifies workflows using pinned steps and sends a Slack message so we can fix the problem. +In the future, we may expand the linter to find other areas for improvement. + +```kotlin +override fun execute() { + val warnings = mutableMapOf>() + val appsInOrg = BitriseApps.getOnlyInstructureApps() + println("Linting ${appsInOrg.size} apps in org") + appsInOrg.forEach { app -> + val yaml = BitriseYaml.getYaml(app.slug) + warnings.putAll(stepVersionWarnings(app, yaml)) + } + + checkWarnings(warnings) +} +``` + +See [LintBitriseYamls.kt][1] for the full source. + +[1]: https://github.com/instructure/canvas-android/blob/f455db88520d37be007af2f7b9e36d17e45182f5/automation/cloud_build_metrics/src/main/kotlin/tasks/LintBitriseYamls.kt diff --git a/automation/cloud_build_metrics/docs/SprintReport.md b/automation/cloud_build_metrics/docs/SprintReport.md new file mode 100644 index 0000000000..dd252037f8 --- /dev/null +++ b/automation/cloud_build_metrics/docs/SprintReport.md @@ -0,0 +1,26 @@ +# SprintReport + +## Objective + +Provide sprint level insight into build health + +![](./png/sprint_report.png) + +## Solution + +In addition to Data Studio for aggregated summary build health, there’s a need for understanding how every job is +doing on the platform. The SprintReport tasks generates a color coded Google Sheet with statistics for the week about +every job that executed. The sprint report has been instrumental in identifying jobs that need help. A number of bugs +in the infrastructure on bitrise were found due to this reporting. It’s great to bring data into conversations with +vendors to help provide clarity around the best way to resolve a problem. The per job data helps ensure that the fixes +we’re making have a measurable impact on performance and stability. + +```kotlin +override fun execute() { + uploadBuildReport(BitriseQuery) +} +``` + +See [SprintReport.kt][1] for the full source. + +[1]: https://github.com/instructure/canvas-android/blob/f455db88520d37be007af2f7b9e36d17e45182f5/automation/cloud_build_metrics/src/main/kotlin/tasks/SprintReport.kt diff --git a/automation/cloud_build_metrics/docs/StartBitriseJob_ReportBitriseJobs.md b/automation/cloud_build_metrics/docs/StartBitriseJob_ReportBitriseJobs.md new file mode 100644 index 0000000000..ea23432c1c --- /dev/null +++ b/automation/cloud_build_metrics/docs/StartBitriseJob_ReportBitriseJobs.md @@ -0,0 +1,66 @@ +# StartBitriseJob / ReportBitriseJob + +## Objective + +Batch create jobs and report on their status. Primarily used to measure build health. + +``` +Warning! You are about to trigger 1 builds on Bitrise. +Doing this during business hours may consume all available resources in CI and may prevent any other builds from running. + +Are you sure you want to proceed? Please type 'Yes' to confirm: +Yes +Build triggered: https://app.bitrise.io/build/da85df2045c553a0 + +Process finished with exit code 0 +``` + +``` +$ cat bitriseJobs.txt +da85df2045c553a0 +``` + +``` +https://www.bitrise.io/build/da85df2045c553a0 - error + +Total builds: 1 +Bitrise build status +error: 1 + +Log failures +Success: 0 +Failure: 1 + +Process finished with exit code 0 +``` + +## Solution + +Start/Report bitrise job allows us to create batches of jobs and then check the results asynchronously later. +This is useful for stability testing new workflows. We can start a new batch of 100x jobs, and then check +to make sure they all completed successfully. + +```kotlin +// StartBitriseJob +override fun execute() { + warnUser() + + val appSlug = "693f666c209a029b" + val buildRequest = BitriseTriggerBuildRequest( + build_params = BuildParams(workflow_id = "primary", environments = emptyList()) + ) + + repeat(JOBS_TO_TRIGGER) { + val build = BitriseApps.triggerBuild(appSlug, buildRequest) + BUILD_IDS.add(build.build_slug) + println("Build triggered: ${build.build_url}") + } + + writeFile() +} +``` + +See [StartBitriseJob.kt][1] and [ReportBitriseJob.kt][2] for the full source. + +[1]: https://github.com/instructure/canvas-android/blob/f455db88520d37be007af2f7b9e36d17e45182f5/automation/cloud_build_metrics/src/main/kotlin/tasks/StartBitriseJob.kt +[2]: https://github.com/instructure/canvas-android/blob/f455db88520d37be007af2f7b9e36d17e45182f5/automation/cloud_build_metrics/src/main/kotlin/tasks/ReportBitriseJob.kt diff --git a/automation/cloud_build_metrics/docs/UpdateBitriseYamls.md b/automation/cloud_build_metrics/docs/UpdateBitriseYamls.md new file mode 100644 index 0000000000..7d812c933d --- /dev/null +++ b/automation/cloud_build_metrics/docs/UpdateBitriseYamls.md @@ -0,0 +1,51 @@ +# UpdateBitriseYamls + +## Objective + +Automatically fix lint issues discovered by `LintBitriseYamls` + +``` +Warning! You are about to update the bitrise.yml file for every Instructure job running on Bitrise. +You must ensure that the existing yaml files have been properly backed up. + +Are you sure you want to proceed? Please type 'Yes' to confirm: +Yes + +Updating: canvas iOS UI Tests https://www.bitrise.io/app/21c95f472d50f580# ... ✅ +Updating: canvas-android https://www.bitrise.io/app/c7219e86fcecb15f# ... ✅ +Updating: canvas-ios unit tests https://www.bitrise.io/app/9b66b2e82404d2f7# ... ✅ +Updating: canvas-ios https://www.bitrise.io/app/94c7e4f1b42e88c7# ... ✅ + +Process finished with exit code 0 + +``` + +## Solution + +`UpdateBitriseYamls` provides an automated way of fixing lint issues found with `LintBitriseYamls`. +Currently the update step supports rewriting workflows to use the latest versions of steps. +Using the latest steps helps us collaborate better with the Bitrise team as the platform evolves with new features and bug fixes. +We can rewrite all the workflows with confidence because the YAMLs are backed up by `BackupBitriseYamls`. + +```kotlin +override fun execute() { + warnUser() + + for (app in BitriseApps.getOnlyInstructureApps()) { + val yaml = getYaml(app.slug) + val warnings = stepVersionWarnings(app, yaml) + if (warnings.isEmpty()) { + continue + } + + print("Updating: ${app.title} https://www.bitrise.io/app/${app.slug}# ... ") + val newYaml = fixStepVersions(app, yaml, warnings) + postYaml(app.slug, newYaml) + println(" ✅") + } +} +``` + +See [UpdateBitriseYamls.kt][1] for the full source. + +[1]: https://github.com/instructure/canvas-android/blob/f455db88520d37be007af2f7b9e36d17e45182f5/automation/cloud_build_metrics/src/main/kotlin/tasks/UpdateBitriseYamls.kt diff --git a/automation/cloud_build_metrics/docs/readme.md b/automation/cloud_build_metrics/docs/flaky_tests.md similarity index 100% rename from automation/cloud_build_metrics/docs/readme.md rename to automation/cloud_build_metrics/docs/flaky_tests.md diff --git a/automation/cloud_build_metrics/docs/overview.md b/automation/cloud_build_metrics/docs/overview.md new file mode 100644 index 0000000000..103e9f489a --- /dev/null +++ b/automation/cloud_build_metrics/docs/overview.md @@ -0,0 +1,16 @@ +# Cloud Build Metrics + +The cloud build metrics project uses the Bitrise API to gather data related to build health. Here are a few tasks that have been automated: + +Task | Description +--- | --- +[BackupBitriseYamls](./BackupBitriseYamls.md) | Nightly backup of all Bitrise workflows to an encrypted git repo. +[BitriseCacheRefresh](./BitriseCacheRefresh.md)| Nightly refresh of the build cache. +[BitriseCacheReport](./BitriseCacheReport.md)| List build cache size for all apps. +[BitriseSetRollingBuilds](./BitriseSetRollingBuilds.md)| Enforce rolling build settings across all apps in an org. +[BuildActivityReport](./BuildActivityReport.md)| Print build activity from most recent to least. +[BuildErrorReport](./BuildErrorReport.md)| Parse build results and identify failed steps. +[LintBitriseYamls](./LintBitriseYamls.md)| Lint bitrise yamls for pinned steps. +[StartBitriseJob / ReportBitriseJob](./StartBitriseJob_ReportBitriseJobs.md)| Start a batch of jobs and then asynchronously view the status +[SprintReport](./SprintReport.md) | Update Google Sheet build health dashboard. +[UpdateBitriseYamls](./UpdateBitriseYamls.md) | Rewrites all Bitrise YAMLs to use master version of steps diff --git a/automation/cloud_build_metrics/docs/png/keybase_git_repo.png b/automation/cloud_build_metrics/docs/png/keybase_git_repo.png new file mode 100644 index 0000000000..fb096e88bf Binary files /dev/null and b/automation/cloud_build_metrics/docs/png/keybase_git_repo.png differ diff --git a/automation/cloud_build_metrics/docs/png/sprint_report.png b/automation/cloud_build_metrics/docs/png/sprint_report.png new file mode 100644 index 0000000000..3bf76d0335 Binary files /dev/null and b/automation/cloud_build_metrics/docs/png/sprint_report.png differ diff --git a/automation/cloud_build_metrics/src/main/kotlin/tasks/BitriseSetRollingBuilds.kt b/automation/cloud_build_metrics/src/main/kotlin/tasks/BitriseSetRollingBuilds.kt index 28519c6087..2ba9854a44 100644 --- a/automation/cloud_build_metrics/src/main/kotlin/tasks/BitriseSetRollingBuilds.kt +++ b/automation/cloud_build_metrics/src/main/kotlin/tasks/BitriseSetRollingBuilds.kt @@ -19,6 +19,7 @@ package tasks import api.bitrise.BitriseApps import api.bitrise.private.RollingBuilds import util.getOnlyInstructureApps +import java.lang.Thread.sleep object BitriseSetRollingBuilds : BitriseTask { override fun execute() { @@ -33,6 +34,8 @@ object BitriseSetRollingBuilds : BitriseTask { RollingBuilds.getConfig(appSlug) } catch (e: Exception) { RollingBuilds.enable(appSlug) + // bitrise doesn't immediately enable rolling builds + sleep(2000) RollingBuilds.getConfig(appSlug) } diff --git a/automation/cloud_build_metrics/src/main/kotlin/tasks/ReportBitriseJob.kt b/automation/cloud_build_metrics/src/main/kotlin/tasks/ReportBitriseJob.kt index e554bbc033..74bc0e17ae 100644 --- a/automation/cloud_build_metrics/src/main/kotlin/tasks/ReportBitriseJob.kt +++ b/automation/cloud_build_metrics/src/main/kotlin/tasks/ReportBitriseJob.kt @@ -19,7 +19,6 @@ package tasks import api.bitrise.BitriseApps -import util.defaultBitriseAppSlug import util.defaultBitriseTxt import tasks.BuildErrorReport.downloadErrorLog import java.nio.file.Files @@ -31,22 +30,22 @@ object ReportBitriseJob : Task { override fun execute() { val buildSlugs = Files.readAllLines(Paths.get(defaultBitriseTxt)) - // Write failed: Broken pipe // https://www.bitrise.io/build/a260c64aa59835cb // INSTRUMENTATION_ABORTED: System has crashed. // https://www.bitrise.io/build/28a2d3e5bc76a723 // INSTRUMENTATION_STATUS: stack= - val failures = listOf( + val failures = listOf( "Write failed: Broken pipe", "INSTRUMENTATION_ABORTED:", "INSTRUMENTATION_STATUS: stack=" ) + val appSlug = "693f666c209a029b" val total = buildSlugs.size val status = mutableMapOf() var logFailures = 0 buildSlugs.forEach buildSlugsForEach@{ buildSlug -> - val build = BitriseApps.getBuild(defaultBitriseAppSlug, buildSlug).data + val build = BitriseApps.getBuild(appSlug, buildSlug).data println("https://www.bitrise.io/build/${build.slug} - ${build.status_text}") val value: Int = (status[build.status_text] ?: 0) + 1 @@ -54,7 +53,7 @@ object ReportBitriseJob : Task { // Check logs on successful builds to ensure they're really successful if (build.status_text == "success") { - val logPath = downloadErrorLog(defaultBitriseAppSlug, build) + val logPath = downloadErrorLog(appSlug, build) val logData = Files.readAllLines(logPath) logData.forEach { line -> @@ -91,6 +90,6 @@ object ReportBitriseJob : Task { // "9 hrs 30 mins 41 secs" for 100x espresso @ 10 concurrency @JvmStatic fun main(args: Array) { - ReportBitriseJob.execute() + execute() } } diff --git a/automation/cloud_build_metrics/src/main/kotlin/tasks/StartBitriseJob.kt b/automation/cloud_build_metrics/src/main/kotlin/tasks/StartBitriseJob.kt index 2b07abea3c..e3f99f9631 100644 --- a/automation/cloud_build_metrics/src/main/kotlin/tasks/StartBitriseJob.kt +++ b/automation/cloud_build_metrics/src/main/kotlin/tasks/StartBitriseJob.kt @@ -70,14 +70,13 @@ object StartBitriseJob : Task { override fun execute() { warnUser() - val app = getApp() - val envList = environmentVariables.map { apk -> Environment(apk.envKey, apk.fileName) } + val appSlug = "693f666c209a029b" val buildRequest = BitriseTriggerBuildRequest( - build_params = BuildParams(workflow_id = "x86_emulator", environments = envList) + build_params = BuildParams(workflow_id = "primary", environments = emptyList()) ) repeat(JOBS_TO_TRIGGER) { - val build = BitriseApps.triggerBuild(app.slug, buildRequest) + val build = BitriseApps.triggerBuild(appSlug, buildRequest) BUILD_IDS.add(build.build_slug) println("Build triggered: ${build.build_url}") } @@ -87,6 +86,6 @@ object StartBitriseJob : Task { @JvmStatic fun main(args: Array) { - StartBitriseJob.execute() + execute() } }