From 65e03e05927e0d20f8678f1ba5f5e4037835e12e Mon Sep 17 00:00:00 2001 From: Sarah Zakarias Date: Mon, 28 Oct 2024 12:29:46 +0000 Subject: [PATCH 1/5] Download counts: add computations for weekly downloads --- .../fake/backend/fake_download_counts.dart | 2 +- app/lib/service/download_counts/backend.dart | 2 +- ...ys_total_counts.dart => computations.dart} | 26 +++++++++++ app/lib/tool/neat_task/pub_dev_tasks.dart | 2 +- ...ounts_test.dart => computations_test.dart} | 45 ++++++++++++++++++- 5 files changed, 73 insertions(+), 4 deletions(-) rename app/lib/service/download_counts/{compute_30_days_total_counts.dart => computations.dart} (69%) rename app/test/service/download_counts/{compute_total_download_counts_test.dart => computations_test.dart} (77%) diff --git a/app/lib/fake/backend/fake_download_counts.dart b/app/lib/fake/backend/fake_download_counts.dart index 3b38e8aef8..e1df8214e9 100644 --- a/app/lib/fake/backend/fake_download_counts.dart +++ b/app/lib/fake/backend/fake_download_counts.dart @@ -5,7 +5,7 @@ import 'dart:io'; import 'package:gcloud/storage.dart'; -import 'package:pub_dev/service/download_counts/compute_30_days_total_counts.dart'; +import 'package:pub_dev/service/download_counts/computations.dart'; import 'package:pub_dev/shared/configuration.dart'; import 'package:pub_dev/shared/utils.dart'; diff --git a/app/lib/service/download_counts/backend.dart b/app/lib/service/download_counts/backend.dart index a734abdf3d..1ddb97600a 100644 --- a/app/lib/service/download_counts/backend.dart +++ b/app/lib/service/download_counts/backend.dart @@ -7,7 +7,7 @@ import 'dart:convert'; import 'package:gcloud/service_scope.dart' as ss; import 'package:gcloud/storage.dart'; import 'package:googleapis/storage/v1.dart'; -import 'package:pub_dev/service/download_counts/compute_30_days_total_counts.dart'; +import 'package:pub_dev/service/download_counts/computations.dart'; import 'package:pub_dev/service/download_counts/download_counts.dart'; import 'package:pub_dev/service/download_counts/models.dart'; import 'package:pub_dev/service/entrypoint/analyzer.dart'; diff --git a/app/lib/service/download_counts/compute_30_days_total_counts.dart b/app/lib/service/download_counts/computations.dart similarity index 69% rename from app/lib/service/download_counts/compute_30_days_total_counts.dart rename to app/lib/service/download_counts/computations.dart index 5aaaffa65d..a84c423bc3 100644 --- a/app/lib/service/download_counts/compute_30_days_total_counts.dart +++ b/app/lib/service/download_counts/computations.dart @@ -42,3 +42,29 @@ Future upload30DaysTotal(Map counts) async { await uploadBytesWithRetry(reportsBucket, downloadCounts30DaysTotalsFileName, jsonUtf8Encoder.convert(counts)); } + +Future<({List weeklyPoints, DateTime? newestDate})> computeWeeklyDownloads( + String package) async { + final weeklyPoints = List.filled(52, 0); + final countData = + await downloadCountsBackend.lookupDownloadCountData(package); + if (countData == null) { + return (weeklyPoints: [], newestDate: null); + } + + final totals = countData!.totalCounts; + + var sum = 0; + for (int i = 0; i < 52 * 7; i++) { + if (totals[i] < 0) { + // There is no more available data. + break; + } + sum += totals[i]; + if ((i + 1) % 7 == 0) { + weeklyPoints[(i ~/ 7)] = sum; + sum = 0; + } + } + return (weeklyPoints: weeklyPoints, newestDate: countData.newestDate!); +} diff --git a/app/lib/tool/neat_task/pub_dev_tasks.dart b/app/lib/tool/neat_task/pub_dev_tasks.dart index e50f784cb6..881060ba5c 100644 --- a/app/lib/tool/neat_task/pub_dev_tasks.dart +++ b/app/lib/tool/neat_task/pub_dev_tasks.dart @@ -8,7 +8,7 @@ import 'dart:io'; import 'package:gcloud/service_scope.dart' as ss; import 'package:logging/logging.dart'; import 'package:neat_periodic_task/neat_periodic_task.dart'; -import 'package:pub_dev/service/download_counts/compute_30_days_total_counts.dart'; +import 'package:pub_dev/service/download_counts/computations.dart'; import '../../account/backend.dart'; import '../../account/consent_backend.dart'; diff --git a/app/test/service/download_counts/compute_total_download_counts_test.dart b/app/test/service/download_counts/computations_test.dart similarity index 77% rename from app/test/service/download_counts/compute_total_download_counts_test.dart rename to app/test/service/download_counts/computations_test.dart index fd39024661..dcd1ed4892 100644 --- a/app/test/service/download_counts/compute_total_download_counts_test.dart +++ b/app/test/service/download_counts/computations_test.dart @@ -7,7 +7,7 @@ import 'package:basics/basics.dart'; import 'package:gcloud/storage.dart'; import 'package:pub_dev/fake/backend/fake_download_counts.dart'; import 'package:pub_dev/service/download_counts/backend.dart'; -import 'package:pub_dev/service/download_counts/compute_30_days_total_counts.dart'; +import 'package:pub_dev/service/download_counts/computations.dart'; import 'package:pub_dev/shared/configuration.dart'; import 'package:test/test.dart'; @@ -119,5 +119,48 @@ void main() { expect(downloadCountsBackend.lookup30DaysTotalCounts('baz'), 150); expect(downloadCountsBackend.lookup30DaysTotalCounts('bax'), isNull); }); + + testWithProfile('compute weekly', fn: () async { + final pkg = 'foo'; + final date = DateTime.parse('1986-02-16'); + final versionsCounts = { + '1.0.1': 2, + '2.0.0-alpha': 2, + '2.0.0': 2, + '2.1.0': 2, + '3.1.0': 2, + '4.0.0-0': 2, + '6.1.0': 2, + }; + final versionsCounts2 = { + '1.0.1': 3, + '2.0.0-alpha': 3, + '2.0.0': 3, + '2.1.0': 3, + '3.1.0': 3, + '4.0.0-0': 3, + '6.1.0': 3, + }; + + for (var i = 0; i < 7 * 20; i++) { + await downloadCountsBackend.updateDownloadCounts( + pkg, versionsCounts, date.addCalendarDays(i)); + } + + for (var i = 7 * 20; i < 7 * 40; i++) { + await downloadCountsBackend.updateDownloadCounts( + pkg, versionsCounts2, date.addCalendarDays(i)); + } + + final res = await computeWeeklyDownloads(pkg); + + final expectedList = List.from(List.filled(20, 147)) + ..addAll(List.filled(20, 98)) + ..addAll(List.filled(12, 0)); + final expectedNewstDate = date.addCalendarDays(7 * 40 - 1); + + expect(res.weeklyPoints, expectedList); + expect(res.newestDate, expectedNewstDate); + }); }); } From 9c7438774c2bebb1ef71d1e624173c13172774e6 Mon Sep 17 00:00:00 2001 From: Sarah Zakarias Date: Mon, 28 Oct 2024 12:40:20 +0000 Subject: [PATCH 2/5] remove null assertion --- app/lib/service/download_counts/computations.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/service/download_counts/computations.dart b/app/lib/service/download_counts/computations.dart index a84c423bc3..37e31b300b 100644 --- a/app/lib/service/download_counts/computations.dart +++ b/app/lib/service/download_counts/computations.dart @@ -52,7 +52,7 @@ Future<({List weeklyPoints, DateTime? newestDate})> computeWeeklyDownloads( return (weeklyPoints: [], newestDate: null); } - final totals = countData!.totalCounts; + final totals = countData.totalCounts; var sum = 0; for (int i = 0; i < 52 * 7; i++) { From 672423d6c9389fcbf9f93bd570054e01a4205d39 Mon Sep 17 00:00:00 2001 From: Sarah Zakarias Date: Mon, 28 Oct 2024 13:03:06 +0000 Subject: [PATCH 3/5] Add last partial week --- app/lib/service/download_counts/computations.dart | 1 + app/test/service/download_counts/computations_test.dart | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/lib/service/download_counts/computations.dart b/app/lib/service/download_counts/computations.dart index 37e31b300b..3d839d8bde 100644 --- a/app/lib/service/download_counts/computations.dart +++ b/app/lib/service/download_counts/computations.dart @@ -57,6 +57,7 @@ Future<({List weeklyPoints, DateTime? newestDate})> computeWeeklyDownloads( var sum = 0; for (int i = 0; i < 52 * 7; i++) { if (totals[i] < 0) { + weeklyPoints[(i ~/ 7)] = sum; // There is no more available data. break; } diff --git a/app/test/service/download_counts/computations_test.dart b/app/test/service/download_counts/computations_test.dart index dcd1ed4892..74d7c502f2 100644 --- a/app/test/service/download_counts/computations_test.dart +++ b/app/test/service/download_counts/computations_test.dart @@ -142,12 +142,12 @@ void main() { '6.1.0': 3, }; - for (var i = 0; i < 7 * 20; i++) { + for (var i = 0; i <= 7 * 20; i++) { await downloadCountsBackend.updateDownloadCounts( pkg, versionsCounts, date.addCalendarDays(i)); } - for (var i = 7 * 20; i < 7 * 40; i++) { + for (var i = 7 * 20 + 1; i <= 7 * 40; i++) { await downloadCountsBackend.updateDownloadCounts( pkg, versionsCounts2, date.addCalendarDays(i)); } @@ -156,8 +156,9 @@ void main() { final expectedList = List.from(List.filled(20, 147)) ..addAll(List.filled(20, 98)) - ..addAll(List.filled(12, 0)); - final expectedNewstDate = date.addCalendarDays(7 * 40 - 1); + ..add(14) + ..addAll(List.filled(11, 0)); + final expectedNewstDate = date.addCalendarDays(7 * 40); expect(res.weeklyPoints, expectedList); expect(res.newestDate, expectedNewstDate); From 28e5691eab993cf3967539d97027d435ed637ac1 Mon Sep 17 00:00:00 2001 From: Sarah Zakarias Date: Mon, 28 Oct 2024 13:33:01 +0000 Subject: [PATCH 4/5] Update name and documentation --- .../service/download_counts/computations.dart | 19 ++++++++++++------- .../download_counts/computations_test.dart | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/app/lib/service/download_counts/computations.dart b/app/lib/service/download_counts/computations.dart index 3d839d8bde..93e2eb1cb5 100644 --- a/app/lib/service/download_counts/computations.dart +++ b/app/lib/service/download_counts/computations.dart @@ -43,13 +43,18 @@ Future upload30DaysTotal(Map counts) async { jsonUtf8Encoder.convert(counts)); } -Future<({List weeklyPoints, DateTime? newestDate})> computeWeeklyDownloads( - String package) async { - final weeklyPoints = List.filled(52, 0); +/// Computes `weeklyDownloads` starting from `newestDate` for [package]. +/// +/// Each number in `weeklyDownloads` is the total number of downloads for +/// a given 7 day period starting from the newest date with download counts +/// data available. +Future<({List weeklyDownloads, DateTime? newestDate})> + computeWeeklyDownloads(String package) async { + final weeklyDownloads = List.filled(52, 0); final countData = await downloadCountsBackend.lookupDownloadCountData(package); if (countData == null) { - return (weeklyPoints: [], newestDate: null); + return (weeklyDownloads: [], newestDate: null); } final totals = countData.totalCounts; @@ -57,15 +62,15 @@ Future<({List weeklyPoints, DateTime? newestDate})> computeWeeklyDownloads( var sum = 0; for (int i = 0; i < 52 * 7; i++) { if (totals[i] < 0) { - weeklyPoints[(i ~/ 7)] = sum; + weeklyDownloads[(i ~/ 7)] = sum; // There is no more available data. break; } sum += totals[i]; if ((i + 1) % 7 == 0) { - weeklyPoints[(i ~/ 7)] = sum; + weeklyDownloads[(i ~/ 7)] = sum; sum = 0; } } - return (weeklyPoints: weeklyPoints, newestDate: countData.newestDate!); + return (weeklyDownloads: weeklyDownloads, newestDate: countData.newestDate!); } diff --git a/app/test/service/download_counts/computations_test.dart b/app/test/service/download_counts/computations_test.dart index 74d7c502f2..d2b46d2eb7 100644 --- a/app/test/service/download_counts/computations_test.dart +++ b/app/test/service/download_counts/computations_test.dart @@ -160,7 +160,7 @@ void main() { ..addAll(List.filled(11, 0)); final expectedNewstDate = date.addCalendarDays(7 * 40); - expect(res.weeklyPoints, expectedList); + expect(res.weeklyDownloads, expectedList); expect(res.newestDate, expectedNewstDate); }); }); From d862c7d2e16abab1fa3dd0b184f5063c8eeadea0 Mon Sep 17 00:00:00 2001 From: Sarah Zakarias Date: Mon, 28 Oct 2024 13:53:16 +0000 Subject: [PATCH 5/5] use nested loops instead --- .../service/download_counts/computations.dart | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/app/lib/service/download_counts/computations.dart b/app/lib/service/download_counts/computations.dart index 93e2eb1cb5..7a428b52fb 100644 --- a/app/lib/service/download_counts/computations.dart +++ b/app/lib/service/download_counts/computations.dart @@ -59,18 +59,15 @@ Future<({List weeklyDownloads, DateTime? newestDate})> final totals = countData.totalCounts; - var sum = 0; - for (int i = 0; i < 52 * 7; i++) { - if (totals[i] < 0) { - weeklyDownloads[(i ~/ 7)] = sum; - // There is no more available data. - break; - } - sum += totals[i]; - if ((i + 1) % 7 == 0) { - weeklyDownloads[(i ~/ 7)] = sum; - sum = 0; + for (int w = 0; w < 52; w++) { + var sum = 0; + for (int d = 0; d < 7; d++) { + if (totals[w * 7 + d] > 0) { + sum += totals[w * 7 + d]; + } } + weeklyDownloads[w] = sum; } + return (weeklyDownloads: weeklyDownloads, newestDate: countData.newestDate!); }