diff --git a/.env.example b/.env.example index 171a774..a1a8dd8 100644 --- a/.env.example +++ b/.env.example @@ -52,3 +52,7 @@ # OPTIONAL: Window for the /data API endpoints, in minutes. # RATE_LIMIT_DATA_API_WINDOW_MINUTES=60 + +# OPTIONAL: The cache duration for the CountryQueryService, in minutes. +# Defaults to 15 minutes if not specified. +# COUNTRY_SERVICE_CACHE_MINUTES=15 diff --git a/analysis_options.yaml b/analysis_options.yaml index 12a262c..803eb63 100644 --- a/analysis_options.yaml +++ b/analysis_options.yaml @@ -10,6 +10,7 @@ analyzer: avoid_catching_errors: ignore document_ignores: ignore one_member_abstracts: ignore + cascade_invocations: ignore exclude: - build/** linter: diff --git a/lib/src/config/app_dependencies.dart b/lib/src/config/app_dependencies.dart index d766b6e..2667118 100644 --- a/lib/src/config/app_dependencies.dart +++ b/lib/src/config/app_dependencies.dart @@ -9,6 +9,7 @@ import 'package:flutter_news_app_api_server_full_source_code/src/config/environm import 'package:flutter_news_app_api_server_full_source_code/src/rbac/permission_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/auth_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/auth_token_service.dart'; +import 'package:flutter_news_app_api_server_full_source_code/src/services/country_query_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/dashboard_summary_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/database_seeding_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/default_user_preference_limit_service.dart'; @@ -69,6 +70,7 @@ class AppDependencies { late final PermissionService permissionService; late final UserPreferenceLimitService userPreferenceLimitService; late final RateLimitService rateLimitService; + late final CountryQueryService countryQueryService; /// Initializes all application dependencies. /// @@ -238,6 +240,11 @@ class AppDependencies { connectionManager: _mongoDbConnectionManager, log: Logger('MongoDbRateLimitService'), ); + countryQueryService = CountryQueryService( + countryRepository: countryRepository, + log: Logger('CountryQueryService'), + cacheDuration: EnvironmentConfig.countryServiceCacheDuration, + ); _isInitialized = true; _log.info('Application dependencies initialized successfully.'); @@ -255,6 +262,7 @@ class AppDependencies { await _mongoDbConnectionManager.close(); tokenBlacklistService.dispose(); rateLimitService.dispose(); + countryQueryService.dispose(); // Dispose the new service _isInitialized = false; _log.info('Application dependencies disposed.'); } diff --git a/lib/src/config/environment_config.dart b/lib/src/config/environment_config.dart index f78e1f6..71babc7 100644 --- a/lib/src/config/environment_config.dart +++ b/lib/src/config/environment_config.dart @@ -172,4 +172,14 @@ abstract final class EnvironmentConfig { int.tryParse(_env['RATE_LIMIT_DATA_API_WINDOW_MINUTES'] ?? '60') ?? 60; return Duration(minutes: minutes); } + + /// Retrieves the cache duration in minutes for the CountryQueryService. + /// + /// The value is read from the `COUNTRY_SERVICE_CACHE_MINUTES` environment + /// variable. Defaults to 15 minutes if not set or if parsing fails. + static Duration get countryServiceCacheDuration { + final minutes = + int.tryParse(_env['COUNTRY_SERVICE_CACHE_MINUTES'] ?? '15') ?? 15; + return Duration(minutes: minutes); + } } diff --git a/lib/src/registry/data_operation_registry.dart b/lib/src/registry/data_operation_registry.dart index caf2371..9f45053 100644 --- a/lib/src/registry/data_operation_registry.dart +++ b/lib/src/registry/data_operation_registry.dart @@ -2,6 +2,7 @@ import 'package:core/core.dart'; import 'package:dart_frog/dart_frog.dart'; import 'package:data_repository/data_repository.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/middlewares/ownership_check_middleware.dart'; +import 'package:flutter_news_app_api_server_full_source_code/src/services/country_query_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/dashboard_summary_service.dart'; // --- Typedefs for Data Operations --- @@ -128,12 +129,27 @@ class DataOperationRegistry { sort: s, pagination: p, ), - 'country': (c, uid, f, s, p) => c.read>().readAll( - userId: uid, - filter: f, - sort: s, - pagination: p, - ), + 'country': (c, uid, f, s, p) async { + // Check for special filters that require aggregation. + if (f != null && + (f.containsKey('hasActiveSources') || + f.containsKey('hasActiveHeadlines'))) { + // Use the injected CountryQueryService for complex queries. + final countryQueryService = c.read(); + return countryQueryService.getFilteredCountries( + filter: f, + pagination: p, + sort: s, + ); + } + // Fallback to standard readAll if no special filters are present. + return c.read>().readAll( + userId: uid, + filter: f, + sort: s, + pagination: p, + ); + }, 'language': (c, uid, f, s, p) => c .read>() .readAll(userId: uid, filter: f, sort: s, pagination: p), diff --git a/lib/src/services/country_query_service.dart b/lib/src/services/country_query_service.dart new file mode 100644 index 0000000..1af081c --- /dev/null +++ b/lib/src/services/country_query_service.dart @@ -0,0 +1,322 @@ +import 'dart:async'; +import 'dart:collection'; // Added for SplayTreeMap +import 'dart:convert'; + +import 'package:core/core.dart'; +import 'package:data_repository/data_repository.dart'; +import 'package:logging/logging.dart'; + +/// {@template country_query_service} +/// A service responsible for executing complex queries on country data, +/// including filtering by active sources and headlines, and supporting +/// compound filters with text search. +/// +/// This service also implements robust in-memory caching with a configurable +/// Time-To-Live (TTL) to optimize performance for frequently requested queries. +/// {@endtemplate} +class CountryQueryService { + /// {@macro country_query_service} + CountryQueryService({ + required DataRepository countryRepository, + required Logger log, + Duration cacheDuration = const Duration(minutes: 15), + }) : _countryRepository = countryRepository, + _log = log, + _cacheDuration = cacheDuration { + _cleanupTimer = Timer.periodic(const Duration(minutes: 5), (_) { + _cleanupCache(); + }); + _log.info( + 'CountryQueryService initialized with cache duration: $cacheDuration', + ); + } + final DataRepository _countryRepository; + final Logger _log; + final Duration _cacheDuration; + + final Map data, DateTime expiry})> + _cache = {}; + Timer? _cleanupTimer; + bool _isDisposed = false; + + /// Retrieves a paginated list of countries based on the provided filters, + /// including special filters for active sources and headlines, and text search. + /// + /// This method supports compound filtering by combining `q` (text search), + /// `hasActiveSources`, `hasActiveHeadlines`, and other standard filters. + /// Results are cached to improve performance. + /// + /// - [filter]: A map containing query conditions. Special keys like + /// `hasActiveSources` and `hasActiveHeadlines` trigger aggregation logic. + /// The `q` key triggers a text search on country names. + /// - [pagination]: Optional pagination parameters. + /// - [sort]: Optional sorting options. + /// + /// Throws [OperationFailedException] for unexpected errors during query + /// execution or cache operations. + Future> getFilteredCountries({ + required Map filter, + PaginationOptions? pagination, + List? sort, + }) async { + if (_isDisposed) { + _log.warning('Attempted to query on disposed service.'); + throw const OperationFailedException('Service is disposed.'); + } + + final cacheKey = _generateCacheKey(filter, pagination, sort); + final cachedEntry = _cache[cacheKey]; + + if (cachedEntry != null && DateTime.now().isBefore(cachedEntry.expiry)) { + _log.finer('Returning cached result for key: $cacheKey'); + return cachedEntry.data; + } + + _log.info('Executing new query for countries with filter: $filter'); + try { + final pipeline = _buildAggregationPipeline(filter, pagination, sort); + final aggregationResult = await _countryRepository.aggregate( + pipeline: pipeline, + ); + + // MongoDB aggregation returns a list of maps. We need to convert these + // back into Country objects. + final countries = aggregationResult.map(Country.fromJson).toList(); + + // For aggregation queries, pagination and hasMore need to be handled + // manually if not directly supported by the aggregation stages. + // For simplicity, we'll assume the aggregation pipeline handles limit/skip + // and we'll determine hasMore based on if we fetched more than the limit. + final limit = pagination?.limit ?? countries.length; + final hasMore = countries.length > limit; + final paginatedCountries = countries.take(limit).toList(); + + final response = PaginatedResponse( + items: paginatedCountries, + cursor: null, // Aggregation doesn't typically return a cursor directly + hasMore: hasMore, + ); + + _cache[cacheKey] = ( + data: response, + expiry: DateTime.now().add(_cacheDuration), + ); + _log.finer('Cached new result for key: $cacheKey'); + + return response; + } on HttpException { + rethrow; // Propagate known HTTP exceptions + } catch (e, s) { + _log.severe('Error fetching filtered countries: $e', e, s); + throw OperationFailedException( + 'Failed to retrieve filtered countries: $e', + ); + } + } + + /// Builds the MongoDB aggregation pipeline based on the provided filters. + List> _buildAggregationPipeline( + Map filter, + PaginationOptions? pagination, + List? sort, + ) { + final pipeline = >[]; + final compoundMatchStages = >[]; + + // --- Stage 1: Initial Match for active status, text search, and other filters --- + // All countries should be active by default for these queries + compoundMatchStages.add({'status': ContentStatus.active.name}); + + // Handle `q` (text search) filter + final qValue = filter['q']; + if (qValue is String && qValue.isNotEmpty) { + compoundMatchStages.add({ + r'$text': {r'$search': qValue}, + }); + } + + // Handle other standard filters + filter.forEach((key, value) { + if (key != 'q' && + key != 'hasActiveSources' && + key != 'hasActiveHeadlines') { + compoundMatchStages.add({key: value}); + } + }); + + // Combine all compound match stages and add to pipeline first for efficiency + if (compoundMatchStages.isNotEmpty) { + pipeline.add({ + r'$match': {r'$and': compoundMatchStages}, + }); + } + + // --- Stage 2: Handle `hasActiveSources` filter --- + if (filter['hasActiveSources'] == true) { + // This lookup uses a sub-pipeline to filter for active sources *before* + // joining, which is more efficient than a post-join match. + pipeline.add({ + r'$lookup': { + 'from': 'sources', + 'let': {'countryId': r'$_id'}, + 'pipeline': [ + { + r'$match': { + r'$expr': { + r'$eq': [r'$headquarters._id', r'$$countryId'], + }, + 'status': ContentStatus.active.name, + }, + }, + ], + 'as': 'matchingSources', + }, + }); + pipeline.add({ + r'$match': { + 'matchingSources': {r'$ne': []}, + }, + }); + } + + // --- Stage 3: Handle `hasActiveHeadlines` filter --- + if (filter['hasActiveHeadlines'] == true) { + // This lookup uses a sub-pipeline to filter for active headlines *before* + // joining, which is more efficient than a post-join match. + pipeline.add({ + r'$lookup': { + 'from': 'headlines', + 'let': {'countryId': r'$_id'}, + 'pipeline': [ + { + r'$match': { + r'$expr': { + r'$eq': [r'$eventCountry._id', r'$$countryId'], + }, + 'status': ContentStatus.active.name, + }, + }, + ], + 'as': 'matchingHeadlines', + }, + }); + pipeline.add({ + r'$match': { + 'matchingHeadlines': {r'$ne': []}, + }, + }); + } + + // --- Stage 4: Sorting --- + if (sort != null && sort.isNotEmpty) { + final sortStage = {}; + for (final option in sort) { + sortStage[option.field] = option.order == SortOrder.asc ? 1 : -1; + } + pipeline.add({r'$sort': sortStage}); + } + + // --- Stage 5: Pagination (Skip and Limit) --- + if (pagination?.cursor != null) { + // For cursor-based pagination, we'd typically need a more complex + // aggregation that sorts by the cursor field and then skips. + // For simplicity, this example assumes offset-based pagination or + // that the client handles cursor logic. + _log.warning( + 'Cursor-based pagination is not fully implemented for aggregation ' + 'queries in CountryQueryService. Only limit/skip is supported.', + ); + } + if (pagination?.limit != null) { + // Fetch one more than the limit to determine 'hasMore' + pipeline.add({r'$limit': pagination!.limit! + 1}); + } + + // --- Stage 6: Final Projection --- + // Project to match the Country model's JSON structure. + // The $lookup stages add fields ('matchingSources', 'matchingHeadlines') + // that are not part of the Country model, so we project only the fields + // that are part of the model to ensure clean deserialization. + pipeline.add({ + r'$project': { + '_id': 0, // Exclude _id + 'id': {r'$toString': r'$_id'}, // Map _id back to id + 'isoCode': r'$isoCode', + 'name': r'$name', + 'flagUrl': r'$flagUrl', + 'createdAt': r'$createdAt', + 'updatedAt': r'$updatedAt', + 'status': r'$status', + }, + }); + + return pipeline; + } + + /// Generates a unique, canonical cache key from the query parameters. + /// + /// A canonical key is essential for effective caching. If two different + /// sets of parameters represent the same logical query (e.g., filters in a + /// different order), they must produce the exact same cache key. + /// + /// This implementation achieves this by: + /// 1. Using a [SplayTreeMap] for the `filter` map, which automatically + /// sorts the filters by their keys. + /// 2. Sorting the `sort` options by their field names. + /// 3. Combining these sorted structures with pagination details into a + /// standard map. + /// 4. Encoding the final map into a JSON string, which serves as the + /// reliable and unique cache key. + String _generateCacheKey( + Map filter, + PaginationOptions? pagination, + List? sort, + ) { + final sortedFilter = SplayTreeMap.from(filter); + final List? sortedSort; + if (sort != null) { + sortedSort = List.from(sort) + ..sort((a, b) => a.field.compareTo(b.field)); + } else { + sortedSort = null; + } + + final keyData = { + 'filter': sortedFilter, + 'pagination': {'cursor': pagination?.cursor, 'limit': pagination?.limit}, + 'sort': sortedSort?.map((s) => '${s.field}:${s.order.name}').toList(), + }; + return json.encode(keyData); + } + + /// Cleans up expired entries from the in-memory cache. + void _cleanupCache() { + if (_isDisposed) return; + + final now = DateTime.now(); + final expiredKeys = []; + + _cache.forEach((key, value) { + if (now.isAfter(value.expiry)) { + expiredKeys.add(key); + } + }); + + if (expiredKeys.isNotEmpty) { + expiredKeys.forEach(_cache.remove); + _log.info('Cleaned up ${expiredKeys.length} expired cache entries.'); + } else { + _log.finer('Cache cleanup ran, no expired entries found.'); + } + } + + /// Disposes of resources, specifically the periodic cache cleanup timer. + void dispose() { + if (!_isDisposed) { + _isDisposed = true; + _cleanupTimer?.cancel(); + _cache.clear(); + _log.info('CountryQueryService disposed.'); + } + } +} diff --git a/pubspec.yaml b/pubspec.yaml index aba528b..6a05a55 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -7,6 +7,7 @@ environment: sdk: ^3.9.0 dependencies: + collection: ^1.19.1 core: git: url: https://github.com/flutter-news-app-full-source-code/core.git diff --git a/routes/_middleware.dart b/routes/_middleware.dart index 38b0e44..51acc21 100644 --- a/routes/_middleware.dart +++ b/routes/_middleware.dart @@ -10,6 +10,7 @@ import 'package:flutter_news_app_api_server_full_source_code/src/registry/data_o import 'package:flutter_news_app_api_server_full_source_code/src/registry/model_registry.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/auth_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/auth_token_service.dart'; +import 'package:flutter_news_app_api_server_full_source_code/src/services/country_query_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/dashboard_summary_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/rate_limit_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/token_blacklist_service.dart'; @@ -151,6 +152,9 @@ Handler middleware(Handler handler) { ), ) .use(provider((_) => deps.rateLimitService)) + .use( + provider((_) => deps.countryQueryService), + ) .call(context); }; });