From f9ba5e1030d8124ec7812d0c0030cd20489cde15 Mon Sep 17 00:00:00 2001 From: Sarah Zakarias Date: Mon, 16 Dec 2024 13:56:31 +0000 Subject: [PATCH 1/2] Add widget for downloads chart with json encoded data --- .../templates/views/pkg/score_tab.dart | 27 ++---- app/lib/scorecard/models.dart | 1 + .../service/download_counts/computations.dart | 1 + .../download_counts/download_counts.dart | 71 +--------------- app/lib/shared/redis_cache.dart | 1 + pkg/_pub_shared/build.yaml | 1 + .../lib/data/download_counts_data.dart | 76 +++++++++++++++++ .../lib/data/download_counts_data.g.dart | 83 +++++++++++++++++++ .../src/widget/downloads_chart/widget.dart | 21 +++++ pkg/web_app/lib/src/widget/widget.dart | 2 + 10 files changed, 193 insertions(+), 91 deletions(-) create mode 100644 pkg/_pub_shared/lib/data/download_counts_data.dart create mode 100644 pkg/_pub_shared/lib/data/download_counts_data.g.dart create mode 100644 pkg/web_app/lib/src/widget/downloads_chart/widget.dart diff --git a/app/lib/frontend/templates/views/pkg/score_tab.dart b/app/lib/frontend/templates/views/pkg/score_tab.dart index 12f8bc2ecf..2beb5ccae5 100644 --- a/app/lib/frontend/templates/views/pkg/score_tab.dart +++ b/app/lib/frontend/templates/views/pkg/score_tab.dart @@ -2,11 +2,15 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:convert'; + +import 'package:_pub_shared/data/download_counts_data.dart'; import 'package:_pub_shared/format/encoding.dart'; import 'package:_pub_shared/format/number_format.dart'; import 'package:pana/models.dart'; import 'package:pub_dev/service/download_counts/download_counts.dart'; import 'package:pub_dev/shared/popularity_storage.dart'; +import 'package:pub_dev/shared/utils.dart'; import '../../../../scorecard/models.dart' hide ReportStatus; import '../../../../shared/urls.dart' as urls; @@ -184,31 +188,12 @@ d.Node _downloadsChart(WeeklyVersionDownloadCounts weeklyVersionDownloads) { id: '-downloads-chart', attributes: { 'data-widget': 'downloads-chart', - 'data-downloads-chart': _encodeForDownloadsChart(weeklyVersionDownloads) + 'data-downloads-chart-points': + base64Encode(jsonUtf8Encoder.convert(weeklyVersionDownloads)) }, ); } -String _encodeForDownloadsChart(WeeklyVersionDownloadCounts wvcd) { - final date = wvcd.newestDate.toUtc().millisecondsSinceEpoch ~/ 1000; - - final allCounts = []; - final allRanges = []; - wvcd.majorRangeWeeklyDownloads.forEach((e) => allCounts.addAll(e.counts)); - wvcd.minorRangeWeeklyDownloads.forEach((e) => allCounts.addAll(e.counts)); - wvcd.patchRangeWeeklyDownloads.forEach((e) => allCounts.addAll(e.counts)); - allCounts.addAll(wvcd.totalWeeklyDownloads); - - wvcd.majorRangeWeeklyDownloads.forEach((e) => allRanges.add(e.versionRange)); - wvcd.minorRangeWeeklyDownloads.forEach((e) => allRanges.add(e.versionRange)); - wvcd.patchRangeWeeklyDownloads.forEach((e) => allRanges.add(e.versionRange)); - - return [ - encodeIntsAsLittleEndianBase64String([date, ...allCounts]), - allRanges - ].join(','); -} - final _statusIconUrls = { ReportStatus.passed: staticUrls.getAssetUrl('/static/img/report-ok-icon-green.svg'), diff --git a/app/lib/scorecard/models.dart b/app/lib/scorecard/models.dart index 311039e80b..4e0caada21 100644 --- a/app/lib/scorecard/models.dart +++ b/app/lib/scorecard/models.dart @@ -4,6 +4,7 @@ import 'dart:io'; +import 'package:_pub_shared/data/download_counts_data.dart'; import 'package:_pub_shared/search/tags.dart'; import 'package:json_annotation/json_annotation.dart'; import 'package:pana/models.dart'; diff --git a/app/lib/service/download_counts/computations.dart b/app/lib/service/download_counts/computations.dart index 98e8b6d3f4..c8504b1279 100644 --- a/app/lib/service/download_counts/computations.dart +++ b/app/lib/service/download_counts/computations.dart @@ -4,6 +4,7 @@ import 'dart:async'; import 'dart:math'; +import 'package:_pub_shared/data/download_counts_data.dart'; import 'package:gcloud/storage.dart'; import 'package:pub_dev/service/download_counts/backend.dart'; import 'package:pub_dev/service/download_counts/download_counts.dart'; diff --git a/app/lib/service/download_counts/download_counts.dart b/app/lib/service/download_counts/download_counts.dart index e4b2b0afd8..1b6358e151 100644 --- a/app/lib/service/download_counts/download_counts.dart +++ b/app/lib/service/download_counts/download_counts.dart @@ -2,39 +2,13 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'package:_pub_shared/data/download_counts_data.dart'; import 'package:basics/basics.dart'; import 'package:collection/collection.dart' show IterableExtension; import 'package:json_annotation/json_annotation.dart'; import 'package:pub_semver/pub_semver.dart'; part 'download_counts.g.dart'; -/// A [VersionRangeCount] is a tuple containing a version range and a list of -/// download counts for periods of same length. -/// -/// The first entry in the tuple is a string describing the `versionRange`, for -/// instance '>=1.0.0-0 <2.0.0'. -/// -/// The second entry in the tuple is an integer list of `counts` with download -/// counts for each period. A period could for instance be a day, or a week etc. -/// The `counts` list contains at most [maxAge] entries. -/// -/// Consider the example of period being one day. The first count represents the -/// number of downloads on `newestDate` followed by the downloads on -/// `newestDate` - 1 and so on. E.g. -/// -/// counts = [ 42, 21, 55 ] -/// ▲ ▲ ▲ -/// │ │ └──────────── Download count on newestDate - 2 days -/// │ │ -/// │ └──────────────── Download count on newestDate - 1 day -/// │ -/// └──────────────────── Download count on newestDate -/// -/// -/// Each entry in the `counts` list is non-negativ. A `0` entry can for a given -/// day mean that the version range has no downloads or that there is no data. -typedef VersionRangeCount = ({String versionRange, List counts}); - /// The maximum number of days for which we store downloads counts for a package. const maxAge = 731; @@ -242,46 +216,3 @@ class WeeklyDownloadCounts { _$WeeklyDownloadCountsFromJson(json); Map toJson() => _$WeeklyDownloadCountsToJson(this); } - -@JsonSerializable(includeIfNull: false) -class WeeklyVersionDownloadCounts { - /// An integer list where each number is the total number of downloads for a - /// given 7 day period starting from [newestDate]. - final List totalWeeklyDownloads; - - /// A list of [VersionRangeCount] with major version ranges and weekly - /// downloads for these ranges. - /// - /// E.g. each number in the `counts` list is the total number of downloads for - /// the range in a 7 day period starting from [newestDate]. - final List majorRangeWeeklyDownloads; - - /// A list of [VersionRangeCount] with minor version ranges and weekly - /// downloads for these ranges. - /// - /// E.g. each number in the `counts` list is the total number of downloads for - /// the range in a 7 day period starting from [newestDate]. - final List minorRangeWeeklyDownloads; - - /// A list of [VersionRangeCount] with patch version ranges and weekly - /// downloads for these ranges. - /// - /// E.g. each number in the `counts` list is the total number of downloads for - /// the range in a 7 day period starting from [newestDate]. - final List patchRangeWeeklyDownloads; - - /// The newest date with download counts data available. - final DateTime newestDate; - - WeeklyVersionDownloadCounts({ - required this.newestDate, - required this.majorRangeWeeklyDownloads, - required this.minorRangeWeeklyDownloads, - required this.patchRangeWeeklyDownloads, - required this.totalWeeklyDownloads, - }); - - factory WeeklyVersionDownloadCounts.fromJson(Map json) => - _$WeeklyVersionDownloadCountsFromJson(json); - Map toJson() => _$WeeklyVersionDownloadCountsToJson(this); -} diff --git a/app/lib/shared/redis_cache.dart b/app/lib/shared/redis_cache.dart index dc00f0aa17..f57f9f2aa9 100644 --- a/app/lib/shared/redis_cache.dart +++ b/app/lib/shared/redis_cache.dart @@ -6,6 +6,7 @@ import 'dart:async'; import 'dart:convert'; import 'dart:math'; +import 'package:_pub_shared/data/download_counts_data.dart'; import 'package:_pub_shared/data/package_api.dart' show VersionScore; import 'package:gcloud/service_scope.dart' as ss; import 'package:googleapis/youtube/v3.dart'; diff --git a/pkg/_pub_shared/build.yaml b/pkg/_pub_shared/build.yaml index d17164e28e..1739d7761b 100644 --- a/pkg/_pub_shared/build.yaml +++ b/pkg/_pub_shared/build.yaml @@ -10,6 +10,7 @@ targets: - 'lib/data/admin_api.dart' - 'lib/data/advisories_api.dart' - 'lib/data/completion.dart' + - 'lib/data/download_counts_data.dart' - 'lib/data/package_api.dart' - 'lib/data/page_data.dart' - 'lib/data/publisher_api.dart' diff --git a/pkg/_pub_shared/lib/data/download_counts_data.dart b/pkg/_pub_shared/lib/data/download_counts_data.dart new file mode 100644 index 0000000000..a94a30d8c2 --- /dev/null +++ b/pkg/_pub_shared/lib/data/download_counts_data.dart @@ -0,0 +1,76 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:json_annotation/json_annotation.dart'; + +part 'download_counts_data.g.dart'; + +/// A [VersionRangeCount] is a tuple containing a version range and a list of +/// download counts for periods of same length. +/// +/// The first entry in the tuple is a string describing the `versionRange`, for +/// instance '>=1.0.0-0 <2.0.0'. +/// +/// The second entry in the tuple is an integer list of `counts` with download +/// counts for each period. A period could for instance be a day, or a week etc. +/// +/// Consider the example of period being one day. The first count represents the +/// number of downloads on `newestDate` followed by the downloads on +/// `newestDate` - 1 and so on. E.g. +/// +/// counts = [ 42, 21, 55 ] +/// ▲ ▲ ▲ +/// │ │ └──────────── Download count on newestDate - 2 days +/// │ │ +/// │ └──────────────── Download count on newestDate - 1 day +/// │ +/// └──────────────────── Download count on newestDate +/// +/// +/// Each entry in the `counts` list is non-negativ. A `0` entry can for a given +/// day mean that the version range has no downloads or that there is no data. +typedef VersionRangeCount = ({String versionRange, List counts}); + +@JsonSerializable(includeIfNull: false) +class WeeklyVersionDownloadCounts { + /// An integer list where each number is the total number of downloads for a + /// given 7 day period starting from [newestDate]. + final List totalWeeklyDownloads; + + /// A list of [VersionRangeCount] with major version ranges and weekly + /// downloads for these ranges. + /// + /// E.g. each number in the `counts` list is the total number of downloads for + /// the range in a 7 day period starting from [newestDate]. + final List majorRangeWeeklyDownloads; + + /// A list of [VersionRangeCount] with minor version ranges and weekly + /// downloads for these ranges. + /// + /// E.g. each number in the `counts` list is the total number of downloads for + /// the range in a 7 day period starting from [newestDate]. + final List minorRangeWeeklyDownloads; + + /// A list of [VersionRangeCount] with patch version ranges and weekly + /// downloads for these ranges. + /// + /// E.g. each number in the `counts` list is the total number of downloads for + /// the range in a 7 day period starting from [newestDate]. + final List patchRangeWeeklyDownloads; + + /// The newest date with download counts data available. + final DateTime newestDate; + + WeeklyVersionDownloadCounts({ + required this.newestDate, + required this.majorRangeWeeklyDownloads, + required this.minorRangeWeeklyDownloads, + required this.patchRangeWeeklyDownloads, + required this.totalWeeklyDownloads, + }); + + factory WeeklyVersionDownloadCounts.fromJson(Map json) => + _$WeeklyVersionDownloadCountsFromJson(json); + Map toJson() => _$WeeklyVersionDownloadCountsToJson(this); +} diff --git a/pkg/_pub_shared/lib/data/download_counts_data.g.dart b/pkg/_pub_shared/lib/data/download_counts_data.g.dart new file mode 100644 index 0000000000..7d50877a34 --- /dev/null +++ b/pkg/_pub_shared/lib/data/download_counts_data.g.dart @@ -0,0 +1,83 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'download_counts_data.dart'; + +// ************************************************************************** +// JsonSerializableGenerator +// ************************************************************************** + +WeeklyVersionDownloadCounts _$WeeklyVersionDownloadCountsFromJson( + Map json) => + WeeklyVersionDownloadCounts( + newestDate: DateTime.parse(json['newestDate'] as String), + majorRangeWeeklyDownloads: + (json['majorRangeWeeklyDownloads'] as List) + .map((e) => _$recordConvert( + e, + ($jsonValue) => ( + counts: ($jsonValue['counts'] as List) + .map((e) => (e as num).toInt()) + .toList(), + versionRange: $jsonValue['versionRange'] as String, + ), + )) + .toList(), + minorRangeWeeklyDownloads: + (json['minorRangeWeeklyDownloads'] as List) + .map((e) => _$recordConvert( + e, + ($jsonValue) => ( + counts: ($jsonValue['counts'] as List) + .map((e) => (e as num).toInt()) + .toList(), + versionRange: $jsonValue['versionRange'] as String, + ), + )) + .toList(), + patchRangeWeeklyDownloads: + (json['patchRangeWeeklyDownloads'] as List) + .map((e) => _$recordConvert( + e, + ($jsonValue) => ( + counts: ($jsonValue['counts'] as List) + .map((e) => (e as num).toInt()) + .toList(), + versionRange: $jsonValue['versionRange'] as String, + ), + )) + .toList(), + totalWeeklyDownloads: (json['totalWeeklyDownloads'] as List) + .map((e) => (e as num).toInt()) + .toList(), + ); + +Map _$WeeklyVersionDownloadCountsToJson( + WeeklyVersionDownloadCounts instance) => + { + 'totalWeeklyDownloads': instance.totalWeeklyDownloads, + 'majorRangeWeeklyDownloads': instance.majorRangeWeeklyDownloads + .map((e) => { + 'counts': e.counts, + 'versionRange': e.versionRange, + }) + .toList(), + 'minorRangeWeeklyDownloads': instance.minorRangeWeeklyDownloads + .map((e) => { + 'counts': e.counts, + 'versionRange': e.versionRange, + }) + .toList(), + 'patchRangeWeeklyDownloads': instance.patchRangeWeeklyDownloads + .map((e) => { + 'counts': e.counts, + 'versionRange': e.versionRange, + }) + .toList(), + 'newestDate': instance.newestDate.toIso8601String(), + }; + +$Rec _$recordConvert<$Rec>( + Object? value, + $Rec Function(Map) convert, +) => + convert(value as Map); diff --git a/pkg/web_app/lib/src/widget/downloads_chart/widget.dart b/pkg/web_app/lib/src/widget/downloads_chart/widget.dart new file mode 100644 index 0000000000..595577579f --- /dev/null +++ b/pkg/web_app/lib/src/widget/downloads_chart/widget.dart @@ -0,0 +1,21 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +import 'package:_pub_shared/data/download_counts_data.dart'; +import 'package:web/web.dart'; + +void create(HTMLElement element, Map options) { + final dataPoints = options['points']; + if (dataPoints == null) { + throw UnsupportedError('data-downloads-chart-points required'); + } + + final svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + element.append(svg); + final data = WeeklyVersionDownloadCounts.fromJson((utf8.decoder + .fuse(json.decoder) + .convert(base64Decode(dataPoints)) as Map)); +} diff --git a/pkg/web_app/lib/src/widget/widget.dart b/pkg/web_app/lib/src/widget/widget.dart index d34fd81fec..62d02dd9d4 100644 --- a/pkg/web_app/lib/src/widget/widget.dart +++ b/pkg/web_app/lib/src/widget/widget.dart @@ -12,6 +12,7 @@ import '../web_util.dart'; import 'completion/widget.dart' deferred as completion; import 'switch/widget.dart' as switch_; import 'weekly_sparkline/widget.dart' as weekly_sparkline; +import 'downloads_chart/widget.dart' as downloads_chart; /// Function to create an instance of the widget given an element and options. /// @@ -35,6 +36,7 @@ final _widgets = { 'completion': () => completion.loadLibrary().then((_) => completion.create), 'switch': () => switch_.create, 'weekly-sparkline': () => weekly_sparkline.create, + 'downloads-chart': () => downloads_chart.create, }; Future<_WidgetFn> _noSuchWidget() async => From f3193b087754e5292d5e951aac17ad6b8b48e602 Mon Sep 17 00:00:00 2001 From: Sarah Zakarias Date: Tue, 17 Dec 2024 09:25:14 +0000 Subject: [PATCH 2/2] cleanup --- .../templates/views/pkg/score_tab.dart | 2 - app/lib/scorecard/models.dart | 1 - .../download_counts/download_counts.g.dart | 70 ------------------- .../src/widget/downloads_chart/widget.dart | 3 + pkg/web_app/lib/src/widget/widget.dart | 2 +- 5 files changed, 4 insertions(+), 74 deletions(-) diff --git a/app/lib/frontend/templates/views/pkg/score_tab.dart b/app/lib/frontend/templates/views/pkg/score_tab.dart index 2beb5ccae5..5a585112f7 100644 --- a/app/lib/frontend/templates/views/pkg/score_tab.dart +++ b/app/lib/frontend/templates/views/pkg/score_tab.dart @@ -5,10 +5,8 @@ import 'dart:convert'; import 'package:_pub_shared/data/download_counts_data.dart'; -import 'package:_pub_shared/format/encoding.dart'; import 'package:_pub_shared/format/number_format.dart'; import 'package:pana/models.dart'; -import 'package:pub_dev/service/download_counts/download_counts.dart'; import 'package:pub_dev/shared/popularity_storage.dart'; import 'package:pub_dev/shared/utils.dart'; diff --git a/app/lib/scorecard/models.dart b/app/lib/scorecard/models.dart index 4e0caada21..86fcd72b86 100644 --- a/app/lib/scorecard/models.dart +++ b/app/lib/scorecard/models.dart @@ -9,7 +9,6 @@ import 'package:_pub_shared/search/tags.dart'; import 'package:json_annotation/json_annotation.dart'; import 'package:pana/models.dart'; import 'package:pub_dev/service/download_counts/backend.dart'; -import 'package:pub_dev/service/download_counts/download_counts.dart'; import 'package:pub_dev/task/models.dart'; import '../scorecard/backend.dart'; diff --git a/app/lib/service/download_counts/download_counts.g.dart b/app/lib/service/download_counts/download_counts.g.dart index fec28f73c0..0872916207 100644 --- a/app/lib/service/download_counts/download_counts.g.dart +++ b/app/lib/service/download_counts/download_counts.g.dart @@ -92,73 +92,3 @@ Map _$WeeklyDownloadCountsToJson( 'weeklyDownloads': instance.weeklyDownloads, 'newestDate': instance.newestDate.toIso8601String(), }; - -WeeklyVersionDownloadCounts _$WeeklyVersionDownloadCountsFromJson( - Map json) => - WeeklyVersionDownloadCounts( - newestDate: DateTime.parse(json['newestDate'] as String), - majorRangeWeeklyDownloads: - (json['majorRangeWeeklyDownloads'] as List) - .map((e) => _$recordConvert( - e, - ($jsonValue) => ( - counts: ($jsonValue['counts'] as List) - .map((e) => (e as num).toInt()) - .toList(), - versionRange: $jsonValue['versionRange'] as String, - ), - )) - .toList(), - minorRangeWeeklyDownloads: - (json['minorRangeWeeklyDownloads'] as List) - .map((e) => _$recordConvert( - e, - ($jsonValue) => ( - counts: ($jsonValue['counts'] as List) - .map((e) => (e as num).toInt()) - .toList(), - versionRange: $jsonValue['versionRange'] as String, - ), - )) - .toList(), - patchRangeWeeklyDownloads: - (json['patchRangeWeeklyDownloads'] as List) - .map((e) => _$recordConvert( - e, - ($jsonValue) => ( - counts: ($jsonValue['counts'] as List) - .map((e) => (e as num).toInt()) - .toList(), - versionRange: $jsonValue['versionRange'] as String, - ), - )) - .toList(), - totalWeeklyDownloads: (json['totalWeeklyDownloads'] as List) - .map((e) => (e as num).toInt()) - .toList(), - ); - -Map _$WeeklyVersionDownloadCountsToJson( - WeeklyVersionDownloadCounts instance) => - { - 'totalWeeklyDownloads': instance.totalWeeklyDownloads, - 'majorRangeWeeklyDownloads': instance.majorRangeWeeklyDownloads - .map((e) => { - 'counts': e.counts, - 'versionRange': e.versionRange, - }) - .toList(), - 'minorRangeWeeklyDownloads': instance.minorRangeWeeklyDownloads - .map((e) => { - 'counts': e.counts, - 'versionRange': e.versionRange, - }) - .toList(), - 'patchRangeWeeklyDownloads': instance.patchRangeWeeklyDownloads - .map((e) => { - 'counts': e.counts, - 'versionRange': e.versionRange, - }) - .toList(), - 'newestDate': instance.newestDate.toIso8601String(), - }; diff --git a/pkg/web_app/lib/src/widget/downloads_chart/widget.dart b/pkg/web_app/lib/src/widget/downloads_chart/widget.dart index 595577579f..009f08d93b 100644 --- a/pkg/web_app/lib/src/widget/downloads_chart/widget.dart +++ b/pkg/web_app/lib/src/widget/downloads_chart/widget.dart @@ -18,4 +18,7 @@ void create(HTMLElement element, Map options) { final data = WeeklyVersionDownloadCounts.fromJson((utf8.decoder .fuse(json.decoder) .convert(base64Decode(dataPoints)) as Map)); + drawChart(svg, data); } + +void drawChart(Element svg, WeeklyVersionDownloadCounts data) {} diff --git a/pkg/web_app/lib/src/widget/widget.dart b/pkg/web_app/lib/src/widget/widget.dart index 62d02dd9d4..cc4e038162 100644 --- a/pkg/web_app/lib/src/widget/widget.dart +++ b/pkg/web_app/lib/src/widget/widget.dart @@ -10,9 +10,9 @@ import 'package:web/web.dart'; import '../web_util.dart'; import 'completion/widget.dart' deferred as completion; +import 'downloads_chart/widget.dart' as downloads_chart; import 'switch/widget.dart' as switch_; import 'weekly_sparkline/widget.dart' as weekly_sparkline; -import 'downloads_chart/widget.dart' as downloads_chart; /// Function to create an instance of the widget given an element and options. ///