From 3d934876d881b35745b1c827e54ae4321815576c Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Fri, 25 Oct 2024 19:54:18 +0200 Subject: [PATCH] Suggest topics based on search phrases (behind experimental). --- app/lib/frontend/handlers/experimental.dart | 4 ++- app/lib/frontend/templates/listing.dart | 25 ++++++++++++++++ .../frontend/templates/views/pkg/index.dart | 12 ++++++-- app/lib/package/search_adapter.dart | 5 ++++ app/lib/search/mem_index.dart | 29 +++++++++++++++++++ app/lib/search/search_service.dart | 6 ++++ app/lib/search/search_service.g.dart | 4 +++ 7 files changed, 82 insertions(+), 3 deletions(-) diff --git a/app/lib/frontend/handlers/experimental.dart b/app/lib/frontend/handlers/experimental.dart index 41d0914ec5..a61d038275 100644 --- a/app/lib/frontend/handlers/experimental.dart +++ b/app/lib/frontend/handlers/experimental.dart @@ -8,8 +8,9 @@ import '../../shared/cookie_utils.dart'; const _publicFlags = { 'dark', - 'search-completion', 'download-counts', + 'search-completion', + 'search-topics', }; const _allFlags = { @@ -86,6 +87,7 @@ class ExperimentalFlags { } bool get isSearchCompletionEnabled => isEnabled('search-completion'); + bool get isSearchTopicsEnabled => isEnabled('search-topics'); bool get isDarkModeEnabled => isEnabled('dark'); bool get isDarkModeDefault => isEnabled('dark-as-default'); diff --git a/app/lib/frontend/templates/listing.dart b/app/lib/frontend/templates/listing.dart index 702cb93fef..9e58df310d 100644 --- a/app/lib/frontend/templates/listing.dart +++ b/app/lib/frontend/templates/listing.dart @@ -6,6 +6,7 @@ import 'dart:math'; import 'package:_pub_shared/search/search_form.dart'; import 'package:collection/collection.dart'; +import 'package:pub_dev/frontend/request_context.dart'; import '../../package/search_adapter.dart'; import '../../search/search_service.dart'; @@ -51,6 +52,9 @@ String renderPkgIndexPage( messageFromBackend: searchResultPage.errorMessage, ), nameMatches: _nameMatches(searchForm, searchResultPage.nameMatches), + topicMatches: requestContext.experimentalFlags.isSearchTopicsEnabled + ? _topicMatches(searchForm, searchResultPage.topicMatches) + : null, packageList: packageList(searchResultPage), pagination: searchResultPage.hasHit ? paginationNode(links) : null, openSections: openSections, @@ -147,3 +151,24 @@ d.Node? _nameMatches(SearchForm form, List? matches) { }), ]); } + +d.Node? _topicMatches(SearchForm form, List? matches) { + if (matches == null || matches.isEmpty) { + return null; + } + final singular = matches.length == 1; + final isExactNameMatch = singular && form.parsedQuery.text == matches.single; + final nameMatchLabel = isExactNameMatch + ? 'Exact topic match: ' + : 'Matching ${singular ? 'topic' : 'topics'}: '; + + return d.p(children: [ + d.text(nameMatchLabel), + ...matches.expandIndexed((i, name) { + return [ + if (i > 0) d.text(', '), + d.a(href: urls.searchUrl(q: 'topic:$name'), text: '#$name'), + ]; + }), + ]); +} diff --git a/app/lib/frontend/templates/views/pkg/index.dart b/app/lib/frontend/templates/views/pkg/index.dart index 4871a409e2..5edde3e8e8 100644 --- a/app/lib/frontend/templates/views/pkg/index.dart +++ b/app/lib/frontend/templates/views/pkg/index.dart @@ -14,14 +14,22 @@ d.Node packageListingNode({ required SearchForm searchForm, required d.Node listingInfo, required d.Node? nameMatches, + required d.Node? topicMatches, required d.Node packageList, required d.Node? pagination, required Set? openSections, }) { + final matchHighlights = [ + if (nameMatches != null) nameMatches, + if (topicMatches != null) topicMatches, + ]; final innerContent = d.fragment([ listingInfo, - if (nameMatches != null) - d.div(classes: ['listing-highlight-block'], child: nameMatches), + if (matchHighlights.isNotEmpty) + d.div( + classes: ['listing-highlight-block'], + children: matchHighlights, + ), packageList, if (pagination != null) pagination, d.markdown('Check our help page for details on ' diff --git a/app/lib/package/search_adapter.dart b/app/lib/package/search_adapter.dart index a4d019dc7b..2ec8762f7f 100644 --- a/app/lib/package/search_adapter.dart +++ b/app/lib/package/search_adapter.dart @@ -43,6 +43,7 @@ class SearchAdapter { form, result.totalCount, nameMatches: result.nameMatches, + topicMatches: result.topicMatches, sdkLibraryHits: result.sdkLibraryHits, packageHits: result.packageHits.map((h) => views[h.package]).nonNulls.toList(), @@ -146,6 +147,9 @@ class SearchResultPage { /// would be considered as blocker for publishing). final List? nameMatches; + /// Topic names that are exact name matches or are close to a known topic. + final List? topicMatches; + /// The hits from the SDK libraries. final List sdkLibraryHits; @@ -163,6 +167,7 @@ class SearchResultPage { this.form, this.totalCount, { this.nameMatches, + this.topicMatches, List? sdkLibraryHits, List? packageHits, this.errorMessage, diff --git a/app/lib/search/mem_index.dart b/app/lib/search/mem_index.dart index a94cdec057..ec4f028c52 100644 --- a/app/lib/search/mem_index.dart +++ b/app/lib/search/mem_index.dart @@ -8,6 +8,7 @@ import 'package:_pub_shared/search/search_form.dart'; import 'package:clock/clock.dart'; import 'package:logging/logging.dart'; import 'package:meta/meta.dart'; +import 'package:pub_dev/service/topics/models.dart'; import '../shared/utils.dart' show boundedList; import 'models.dart'; @@ -37,6 +38,13 @@ class InMemoryPackageIndex { late final List _likesOrderedHits; late final List _pointsOrderedHits; + // Contains all of the topics the index had seen so far. + // TODO: consider moving this into a separate index + // TODO: get the list of topics from the bucket + final _topics = { + ...canonicalTopics.aliasToCanonicalMap.values, + }; + late final DateTime _lastUpdated; InMemoryPackageIndex({ @@ -57,6 +65,12 @@ class InMemoryPackageIndex { } } } + + // Note: we are not removing topics from this set, only adding them, no + // need for tracking the current topic count. + _topics.addAll(doc.tags + .where((t) => t.startsWith('topic:')) + .map((t) => t.split('topic:').last)); } final packageKeys = _documents.map((d) => d.package).toList(); @@ -170,7 +184,21 @@ class InMemoryPackageIndex { } final nameMatches = textResults?.nameMatches; + List? topicMatches; List packageHits; + + if (parsedQueryText != null) { + final parts = parsedQueryText + .split(' ') + .map((t) => canonicalTopics.aliasToCanonicalMap[t] ?? t) + .toSet() + .where(_topics.contains) + .toList(); + if (parts.isNotEmpty) { + topicMatches = parts; + } + } + switch (query.effectiveOrder ?? SearchOrder.top) { case SearchOrder.top: if (textResults == null) { @@ -229,6 +257,7 @@ class InMemoryPackageIndex { timestamp: clock.now().toUtc(), totalCount: totalCount, nameMatches: nameMatches, + topicMatches: topicMatches, packageHits: packageHits, ); } diff --git a/app/lib/search/search_service.dart b/app/lib/search/search_service.dart index 584c725d14..95bda1269d 100644 --- a/app/lib/search/search_service.dart +++ b/app/lib/search/search_service.dart @@ -312,6 +312,9 @@ class PackageSearchResult { /// Package names that are exact name matches or close to (e.g. names that /// would be considered as blocker for publishing). final List? nameMatches; + + /// Topic names that are exact name matches or close to the queried text. + final List? topicMatches; final List sdkLibraryHits; final List packageHits; @@ -325,6 +328,7 @@ class PackageSearchResult { required this.timestamp, required this.totalCount, this.nameMatches, + this.topicMatches, List? sdkLibraryHits, List? packageHits, this.errorMessage, @@ -339,6 +343,7 @@ class PackageSearchResult { }) : timestamp = clock.now().toUtc(), totalCount = 0, nameMatches = null, + topicMatches = null, sdkLibraryHits = [], packageHits = []; @@ -358,6 +363,7 @@ class PackageSearchResult { timestamp: timestamp, totalCount: totalCount, nameMatches: nameMatches, + topicMatches: topicMatches, sdkLibraryHits: sdkLibraryHits ?? this.sdkLibraryHits, packageHits: packageHits, errorMessage: errorMessage, diff --git a/app/lib/search/search_service.g.dart b/app/lib/search/search_service.g.dart index 8925d4efa2..2d99ab8246 100644 --- a/app/lib/search/search_service.g.dart +++ b/app/lib/search/search_service.g.dart @@ -82,6 +82,9 @@ PackageSearchResult _$PackageSearchResultFromJson(Map json) => nameMatches: (json['nameMatches'] as List?) ?.map((e) => e as String) .toList(), + topicMatches: (json['topicMatches'] as List?) + ?.map((e) => e as String) + .toList(), sdkLibraryHits: (json['sdkLibraryHits'] as List?) ?.map((e) => SdkLibraryHit.fromJson(e as Map)) .toList(), @@ -105,6 +108,7 @@ Map _$PackageSearchResultToJson(PackageSearchResult instance) { } writeNotNull('nameMatches', instance.nameMatches); + writeNotNull('topicMatches', instance.topicMatches); val['sdkLibraryHits'] = instance.sdkLibraryHits.map((e) => e.toJson()).toList(); val['packageHits'] = instance.packageHits.map((e) => e.toJson()).toList();