From bcf9db66478518507c257abdb4b33883eba51c1a Mon Sep 17 00:00:00 2001 From: Keyong Han Date: Mon, 25 Jul 2022 14:57:29 -0700 Subject: [PATCH] remove refresh chromebot status API --- app_dart/bin/server.dart | 6 - app_dart/lib/cocoon_service.dart | 1 - .../refresh_chromebot_status.dart | 164 ------- .../refresh_chromebot_status_test.dart | 464 ------------------ 4 files changed, 635 deletions(-) delete mode 100644 app_dart/lib/src/request_handlers/refresh_chromebot_status.dart delete mode 100644 app_dart/test/request_handlers/refresh_chromebot_status_test.dart diff --git a/app_dart/bin/server.dart b/app_dart/bin/server.dart index 10e566b43..ec2b58cc6 100644 --- a/app_dart/bin/server.dart +++ b/app_dart/bin/server.dart @@ -100,12 +100,6 @@ Future main() async { authProvider, ), '/api/push-gold-status-to-github': PushGoldStatusToGithub(config, authProvider), - '/api/refresh-chromebot-status': RefreshChromebotStatus( - config, - authProvider, - luciBuildService, - scheduler: scheduler, - ), '/api/reset-prod-task': ResetProdTask( config, authProvider, diff --git a/app_dart/lib/cocoon_service.dart b/app_dart/lib/cocoon_service.dart index 4b4be4a56..fdb0c2006 100644 --- a/app_dart/lib/cocoon_service.dart +++ b/app_dart/lib/cocoon_service.dart @@ -22,7 +22,6 @@ export 'src/request_handlers/push_build_status_to_github.dart'; export 'src/request_handlers/push_gold_status_to_github.dart'; export 'src/request_handlers/query_github_graphql.dart'; export 'src/request_handlers/readiness_check.dart'; -export 'src/request_handlers/refresh_chromebot_status.dart'; export 'src/request_handlers/reset_prod_task.dart'; export 'src/request_handlers/reset_try_task.dart'; export 'src/request_handlers/scheduler/batch_backfiller.dart'; diff --git a/app_dart/lib/src/request_handlers/refresh_chromebot_status.dart b/app_dart/lib/src/request_handlers/refresh_chromebot_status.dart deleted file mode 100644 index 2b093854c..000000000 --- a/app_dart/lib/src/request_handlers/refresh_chromebot_status.dart +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2019 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:async'; - -import 'package:github/github.dart'; -import 'package:meta/meta.dart'; - -import '../../ci_yaml.dart'; -import '../foundation/providers.dart'; -import '../foundation/typedefs.dart'; -import '../foundation/utils.dart'; -import '../model/appengine/commit.dart'; -import '../model/appengine/task.dart'; -import '../request_handling/api_request_handler.dart'; -import '../request_handling/authentication.dart'; -import '../request_handling/body.dart'; -import '../service/buildbucket.dart'; -import '../service/config.dart'; -import '../service/datastore.dart'; -import '../service/logging.dart'; -import '../service/luci.dart'; -import '../service/luci_build_service.dart'; -import '../service/scheduler.dart'; - -@immutable -class RefreshChromebotStatus extends ApiRequestHandler { - const RefreshChromebotStatus( - Config config, - AuthenticationProvider authenticationProvider, - this.luciBuildService, { - required this.scheduler, - @visibleForTesting LuciServiceProvider? luciServiceProvider, - @visibleForTesting DatastoreServiceProvider? datastoreProvider, - @visibleForTesting this.branchHttpClientProvider = Providers.freshHttpClient, - @visibleForTesting this.gitHubBackoffCalculator = twoSecondLinearBackoff, - }) : luciServiceProvider = luciServiceProvider ?? _createLuciService, - datastoreProvider = datastoreProvider ?? DatastoreService.defaultProvider, - super(config: config, authenticationProvider: authenticationProvider); - - final LuciBuildService luciBuildService; - final LuciServiceProvider luciServiceProvider; - final DatastoreServiceProvider datastoreProvider; - final HttpClientProvider branchHttpClientProvider; - final GitHubBackoffCalculator gitHubBackoffCalculator; - final Scheduler scheduler; - - static const String kRepoParam = 'repo'; - - static LuciService _createLuciService(ApiRequestHandler handler) { - return LuciService( - buildBucketClient: BuildBucketClient(), - config: handler.config, - clientContext: handler.authContext!.clientContext, - ); - } - - @override - Future get() async { - final String repoName = request!.uri.queryParameters[kRepoParam] ?? Config.flutterSlug.name; - final RepositorySlug slug = RepositorySlug('flutter', repoName); - final LuciService luciService = luciServiceProvider(this); - final DatastoreService datastore = datastoreProvider(config.db); - final Commit latestCommit = await datastore.queryRecentCommits(limit: 1, slug: slug).single; - final CiYaml ciYaml = await scheduler.getCiYaml(latestCommit); - final List postsubmitBuilders = await scheduler.getPostSubmitBuilders(ciYaml); - final Map>> luciTasks = await luciService.getBranchRecentTasks( - builders: postsubmitBuilders, - requireTaskName: true, - ); - log.fine('${luciTasks.keys.length} builders retrieved'); - - for (BranchLuciBuilder branchLuciBuilder in luciTasks.keys) { - await runTransactionWithRetries(() async { - await _updateStatus( - branchLuciBuilder.luciBuilder!, - branchLuciBuilder.branch, - datastore, - luciTasks[branchLuciBuilder], - slug, - ); - }); - } - return Body.empty; - } - - /// Update chromebot tasks statuses in datastore for [builder], - /// based on latest [luciTasks] statuses. - Future _updateStatus( - LuciBuilder builder, - String? branch, - DatastoreService datastore, - Map>? luciTasksMap, - RepositorySlug slug, - ) async { - final List datastoreTasks = await datastore - .queryRecentTasks( - taskName: builder.taskName, - branch: branch, - slug: slug, - ) - .toList(); - - /// Update [devicelabTask] when first [luciTask] run finishes. There may be - /// reruns for the same commit and same builder. Update [devicelabTask] - /// [builderNumberList] when luci rerun happens, and update [devicelabTask] - /// status when the status of latest luci run changes. - for (FullTask datastoreTask in datastoreTasks) { - final String commitSha = datastoreTask.commit.sha!; - if (!luciTasksMap!.containsKey(commitSha)) { - continue; - } - final List luciTasks = luciTasksMap[commitSha]!; - final String buildNumberList = - luciTasks.reversed.map((LuciTask luciTask) => luciTask.buildNumber.toString()).toList().join(','); - final LuciTask latestLuciTask = luciTasks.first; - if (buildNumberList != datastoreTask.task.buildNumberList || latestLuciTask.status != datastoreTask.task.status) { - final Task update = datastoreTask.task; - update.status = latestLuciTask.status; - - final CiYaml ciYaml = await scheduler.getCiYaml(datastoreTask.commit); - final Target target = - ciYaml.postsubmitTargets.singleWhere((Target target) => target.value.name == datastoreTask.task.name); - - await luciBuildService.checkRerunBuilder( - commit: datastoreTask.commit, - target: target, - task: update, - datastore: datastore, - ); - - update.luciBucket = builder.flaky ?? false ? 'luci.flutter.staging' : 'luci.flutter.prod'; - update.buildNumberList = buildNumberList; - - await datastore.insert([update]); - - // Save luci task record to BigQuery only when task finishes. - if (update.status == Task.statusFailed || update.status == Task.statusSucceeded) { - await _insertBigquery(update); - } - } - } - } - - Future _insertBigquery(Task task) async { - const String bigqueryTableName = 'Task'; - final Map bigqueryData = { - 'ID': task.commitKey?.id, - 'CreateTimestamp': task.createTimestamp, - 'StartTimestamp': task.startTimestamp, - 'EndTimestamp': task.endTimestamp, - 'Name': task.name, - 'Attempts': task.attempts, - 'IsFlaky': task.isFlaky, - 'TimeoutInMinutes': task.timeoutInMinutes ?? 0, - 'RequiredCapabilities': task.requiredCapabilities ?? [], - 'ReservedForAgentID': task.reservedForAgentId, - 'StageName': task.stageName ?? 'unknown', - 'Status': task.status, - }; - await insertBigquery(bigqueryTableName, bigqueryData, await config.createTabledataResourceApi()); - } -} diff --git a/app_dart/test/request_handlers/refresh_chromebot_status_test.dart b/app_dart/test/request_handlers/refresh_chromebot_status_test.dart deleted file mode 100644 index fb445fcbb..000000000 --- a/app_dart/test/request_handlers/refresh_chromebot_status_test.dart +++ /dev/null @@ -1,464 +0,0 @@ -// Copyright 2019 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:cocoon_service/src/model/appengine/commit.dart'; -import 'package:cocoon_service/src/model/appengine/task.dart'; -import 'package:cocoon_service/src/model/ci_yaml/target.dart'; -import 'package:cocoon_service/src/request_handlers/refresh_chromebot_status.dart'; -import 'package:cocoon_service/src/service/config.dart'; -import 'package:cocoon_service/src/service/datastore.dart'; -import 'package:cocoon_service/src/service/luci.dart'; -import 'package:gcloud/db.dart'; -import 'package:googleapis/bigquery/v2.dart'; -import 'package:http/http.dart' as http; -import 'package:http/testing.dart'; -import 'package:mockito/mockito.dart'; -import 'package:test/test.dart'; - -import '../src/bigquery/fake_tabledata_resource.dart'; -import '../src/datastore/fake_config.dart'; -import '../src/request_handling/api_request_handler_tester.dart'; -import '../src/request_handling/fake_authentication.dart'; -import '../src/service/fake_scheduler.dart'; -import '../src/utilities/mocks.dart'; - -void main() { - group('RefreshChromebotStatus', () { - late FakeConfig config; - late ApiRequestHandlerTester tester; - late MockLuciService mockLuciService; - late RefreshChromebotStatus handler; - late MockClient branchHttpClient; - late FakeScheduler scheduler; - late FakeTabledataResource tabledataResource; - late MockLuciBuildService mockLuciBuildService; - - late Commit commit; - late List builders; - - setUp(() async { - tabledataResource = FakeTabledataResource(); - config = FakeConfig(tabledataResource: tabledataResource); - config.flutterBranchesValue = [Config.defaultBranch(Config.flutterSlug)]; - tester = ApiRequestHandlerTester(); - mockLuciService = MockLuciService(); - mockLuciBuildService = MockLuciBuildService(); - branchHttpClient = MockClient((_) async => http.Response('', 200)); - scheduler = FakeScheduler( - config: config, - ciYaml: exampleConfig, - ); - handler = RefreshChromebotStatus( - config, - FakeAuthenticationProvider(), - mockLuciBuildService, - luciServiceProvider: (_) => mockLuciService, - datastoreProvider: (DatastoreDB db) => DatastoreService(config.db, 5), - branchHttpClientProvider: () => branchHttpClient, - gitHubBackoffCalculator: (int attempt) => Duration.zero, - scheduler: scheduler, - ); - commit = Commit( - key: config.db.emptyKey.append(Commit, id: 'flutter/flutter/master/abc'), - sha: 'abc', - branch: Config.defaultBranch(Config.flutterSlug), - repository: Config.flutterSlug.fullName, - ); - builders = await scheduler.getPostSubmitBuilders(exampleConfig); - }); - - group('without builder rerun', () { - setUp(() { - when(mockLuciBuildService.checkRerunBuilder( - commit: anyNamed('commit'), - target: anyNamed('target'), - task: anyNamed('task'), - datastore: anyNamed('datastore'), - ignoreChecks: false, - )).thenAnswer((_) => Future.value(false)); - }); - - test('do not update task status when SHA does not match', () async { - final Task task = Task( - key: commit.key.append(Task, id: 123), - commitKey: commit.key, - name: 'Linux A', - status: Task.statusNew, - ); - config.db.values[commit.key] = commit; - config.db.values[task.key] = task; - - final Map>> luciTasks = { - for (LuciBuilder builder in builders) - BranchLuciBuilder(luciBuilder: builder, branch: 'master'): >{ - 'def': [ - const LuciTask( - commitSha: 'def', - ref: 'refs/heads/master', - status: Task.statusSucceeded, - buildNumber: 1, - builderName: 'abc') - ], - } - }; - when(mockLuciService.getBranchRecentTasks(builders: anyNamed('builders'), requireTaskName: true)) - .thenAnswer((Invocation invocation) { - return Future>>>.value(luciTasks); - }); - - expect(task.status, Task.statusNew); - await tester.get(handler); - expect(task.status, Task.statusNew); - }); - - test('do not update task status when commitSha/ref is unknown', () async { - final Task task = Task( - key: commit.key.append(Task, id: 123), - commitKey: commit.key, - name: 'Linux A', - status: Task.statusNew, - ); - config.db.values[commit.key] = commit; - config.db.values[task.key] = task; - - final Map>> luciTasks = { - for (LuciBuilder builder in builders) - BranchLuciBuilder(luciBuilder: builder, branch: 'master'): >{ - 'def': [ - const LuciTask( - commitSha: 'unknown', - ref: 'unknown', - status: Task.statusSucceeded, - buildNumber: 1, - builderName: 'abc') - ], - } - }; - when(mockLuciService.getBranchRecentTasks(builders: anyNamed('builders'), requireTaskName: true)) - .thenAnswer((Invocation invocation) { - return Future>>>.value(luciTasks); - }); - - expect(task.status, Task.statusNew); - await tester.get(handler); - expect(task.status, Task.statusNew); - }); - - test('do not update task status when branch does not match', () async { - final Commit branchCommit = Commit( - key: config.db.emptyKey.append(Commit, id: 'flutter/flutter/test/abc'), - sha: 'abc', - branch: 'test', - repository: Config.flutterSlug.fullName, - ); - final Task task = Task( - key: branchCommit.key.append(Task, id: 123), - commitKey: branchCommit.key, - name: 'Linux A', - status: Task.statusNew, - ); - config.db.values[branchCommit.key] = branchCommit; - config.db.values[commit.key] = commit; - config.db.values[task.key] = task; - final Map>> luciTasks = { - for (LuciBuilder builder in builders) - BranchLuciBuilder(luciBuilder: builder, branch: 'master'): >{ - 'abc': [ - const LuciTask( - commitSha: 'abc', - ref: 'refs/heads/master', - status: Task.statusSucceeded, - buildNumber: 1, - builderName: 'abc') - ], - } - }; - when(mockLuciService.getBranchRecentTasks(builders: anyNamed('builders'), requireTaskName: true)) - .thenAnswer((Invocation invocation) { - return Future>>>.value(luciTasks); - }); - - expect(task.status, Task.statusNew); - await tester.get(handler); - expect(task.status, Task.statusNew); - }); - - test('update task status and buildNumber when buildNumberList does not match', () async { - final Task task = - Task(key: commit.key.append(Task, id: 123), commitKey: commit.key, name: 'Linux A', status: Task.statusNew); - config.db.values[commit.key] = commit; - config.db.values[task.key] = task; - final Map>> luciTasks = { - for (LuciBuilder builder in builders) - BranchLuciBuilder(luciBuilder: builder, branch: 'master'): >{ - 'abc': [ - const LuciTask( - commitSha: 'abc', - ref: 'refs/heads/master', - status: Task.statusSucceeded, - buildNumber: 1, - builderName: 'abc') - ], - } - }; - when(mockLuciService.getBranchRecentTasks(builders: anyNamed('builders'), requireTaskName: true)) - .thenAnswer((Invocation invocation) { - return Future>>>.value(luciTasks); - }); - - expect(task.status, Task.statusNew); - expect(task.buildNumberList, isNull); - await tester.get(handler); - expect(task.status, Task.statusSucceeded); - expect(task.buildNumberList, '1'); - }); - - test('save data to BigQuery when task finishes', () async { - final Task task = - Task(key: commit.key.append(Task, id: 123), commitKey: commit.key, name: 'Linux A', status: Task.statusNew); - config.db.values[commit.key] = commit; - config.db.values[task.key] = task; - final Map>> luciTasks = { - for (LuciBuilder builder in builders) - BranchLuciBuilder(luciBuilder: builder, branch: 'master'): >{ - 'abc': [ - const LuciTask( - commitSha: 'abc', - ref: 'refs/heads/master', - status: Task.statusSucceeded, - buildNumber: 1, - builderName: 'abc') - ], - } - }; - when(mockLuciService.getBranchRecentTasks(builders: anyNamed('builders'), requireTaskName: true)) - .thenAnswer((Invocation invocation) { - return Future>>>.value(luciTasks); - }); - - await tester.get(handler); - final TableDataList tableDataList = await tabledataResource.list('test', 'test', 'test'); - expect(tableDataList.totalRows, '1'); - }); - - test('update task status and buildNumber when status does not match', () async { - final Task task = Task( - key: commit.key.append(Task, id: 123), - commitKey: commit.key, - name: 'Linux A', - status: Task.statusNew, - buildNumberList: '1'); - config.db.values[commit.key] = commit; - config.db.values[task.key] = task; - final Map>> luciTasks = { - for (LuciBuilder builder in builders) - BranchLuciBuilder(luciBuilder: builder, branch: 'master'): >{ - 'abc': [ - const LuciTask( - commitSha: 'abc', - ref: 'refs/heads/master', - status: Task.statusSucceeded, - buildNumber: 1, - builderName: 'abc') - ], - } - }; - when(mockLuciService.getBranchRecentTasks(builders: anyNamed('builders'), requireTaskName: true)) - .thenAnswer((Invocation invocation) { - return Future>>>.value(luciTasks); - }); - - expect(task.status, Task.statusNew); - await tester.get(handler); - expect(task.status, Task.statusSucceeded); - }); - - test('update task status with latest status when multilple reruns exist', () async { - final Task task = Task( - key: commit.key.append(Task, id: 123), - commitKey: commit.key, - name: 'Linux A', - status: Task.statusNew, - buildNumberList: '1'); - config.db.values[commit.key] = commit; - config.db.values[task.key] = task; - final Map>> luciTasks = { - for (LuciBuilder builder in builders) - BranchLuciBuilder(luciBuilder: builder, branch: 'master'): >{ - 'abc': [ - const LuciTask( - commitSha: 'abc', - ref: 'refs/heads/master', - status: Task.statusSucceeded, - buildNumber: 2, - builderName: 'abc'), - const LuciTask( - commitSha: 'abc', - ref: 'refs/heads/master', - status: Task.statusFailed, - buildNumber: 1, - builderName: 'abc') - ], - } - }; - when(mockLuciService.getBranchRecentTasks(builders: anyNamed('builders'), requireTaskName: true)) - .thenAnswer((Invocation invocation) { - return Future>>>.value(luciTasks); - }); - - expect(task.status, Task.statusNew); - await tester.get(handler); - expect(task.status, Task.statusSucceeded); - expect(task.buildNumberList, '1,2'); - }); - - test('update task status with latest status when ci yaml targets exist', () async { - final Task task = Task( - key: commit.key.append(Task, id: 123), - commitKey: commit.key, - name: 'Linux A', - builderName: 'Linux A', - status: Task.statusNew, - buildNumberList: '1'); - config.db.values[commit.key] = commit; - config.db.values[task.key] = task; - scheduler.ciYaml = exampleConfig; - final List builders = - scheduler.ciYaml!.postsubmitTargets.map((Target target) => LuciBuilder.fromTarget(target)).toList(); - final Map>> luciTasks = { - for (LuciBuilder builder in builders) - BranchLuciBuilder(luciBuilder: builder, branch: 'master'): >{ - 'abc': [ - const LuciTask( - commitSha: 'abc', - ref: 'refs/heads/master', - status: Task.statusSucceeded, - buildNumber: 2, - builderName: 'Linux A', - ), - ], - } - }; - when(mockLuciService.getBranchRecentTasks(builders: anyNamed('builders'), requireTaskName: true)) - .thenAnswer((Invocation invocation) { - return Future>>>.value(luciTasks); - }); - - expect(task.status, Task.statusNew); - await tester.get(handler); - expect(task.status, Task.statusSucceeded); - expect(task.buildNumberList, '2'); - }); - - test('update task status for non master branch', () async { - final Commit branchCommit = Commit( - key: config.db.emptyKey.append(Commit, id: 'flutter/flutter/test/def'), - sha: 'def', - branch: 'test', - repository: Config.flutterSlug.fullName, - ); - final Task task = Task( - key: branchCommit.key.append(Task, id: 456), - commitKey: branchCommit.key, - name: 'Linux A', - status: Task.statusNew); - config.flutterBranchesValue = [Config.defaultBranch(Config.flutterSlug), 'test']; - config.db.values[commit.key] = commit; - config.db.values[branchCommit.key] = branchCommit; - config.db.values[task.key] = task; - final Map>> luciTasks = { - for (LuciBuilder builder in builders) - BranchLuciBuilder(luciBuilder: builder, branch: 'master'): >{ - 'def': [ - const LuciTask( - commitSha: 'def', - ref: 'refs/heads/master', - status: Task.statusFailed, - buildNumber: 1, - builderName: 'abc'), - ], - } - }; - final Map>> testLuciTasks = { - for (LuciBuilder builder in builders) - BranchLuciBuilder(luciBuilder: builder, branch: 'test'): >{ - 'def': [ - const LuciTask( - commitSha: 'def', - ref: 'refs/heads/test', - status: Task.statusSucceeded, - buildNumber: 2, - builderName: 'abc') - ], - } - }; - luciTasks.addAll(testLuciTasks); - when(mockLuciService.getBranchRecentTasks(builders: anyNamed('builders'), requireTaskName: true)) - .thenAnswer((Invocation invocation) { - return Future>>>.value(luciTasks); - }); - - expect(task.status, Task.statusNew); - await tester.get(handler); - expect(task.status, Task.statusSucceeded); - expect(task.luciBucket, 'luci.flutter.prod'); - }); - }); - - group('with builder rerun', () { - setUp(() { - when(mockLuciBuildService.checkRerunBuilder( - commit: anyNamed('commit'), - target: anyNamed('target'), - task: anyNamed('task'), - datastore: anyNamed('datastore'), - )).thenAnswer((_) => Future.value(true)); - }); - - test('rerun Mac builder when hiting recipe infra failure', () async { - config.maxTaskRetriesValue = 2; - final Task task = Task( - key: commit.key.append(Task, id: 123), - commitKey: commit.key, - name: 'Linux A', - status: Task.statusInProgress, - buildNumberList: '1', - attempts: 0, - builderName: 'Mac abc'); - config.db.values[commit.key] = commit; - config.db.values[task.key] = task; - final Map>> luciTasks = { - for (LuciBuilder builder in [ - LuciBuilder(name: 'Mac abc', repo: Config.flutterSlug.name, taskName: 'def', flaky: false) - ]) - BranchLuciBuilder(luciBuilder: builder, branch: 'master'): >{ - 'abc': [ - const LuciTask( - commitSha: 'abc', - ref: 'refs/heads/master', - status: Task.statusInfraFailure, - buildNumber: 1, - builderName: 'Mac abc') - ], - } - }; - when(mockLuciService.getBranchRecentTasks(builders: anyNamed('builders'), requireTaskName: true)) - .thenAnswer((Invocation invocation) { - return Future>>>.value(luciTasks); - }); - - expect(task.status, Task.statusInProgress); - await tester.get(handler); - verify(mockLuciBuildService.checkRerunBuilder( - commit: anyNamed('commit'), - target: anyNamed('target'), - task: anyNamed('task'), - datastore: anyNamed('datastore'), - tags: anyNamed('tags'), - ignoreChecks: false)) - .called(1); - }); - }); - }); -}