From c87f614cfd709d11be31d23bcc5ef6a0666dfd8e Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Tue, 4 Nov 2025 12:23:25 +0100 Subject: [PATCH 1/2] Refactor: select task versions based on the cached package data from API endpoint. --- app/lib/package/backend.dart | 32 ++++++--- app/lib/task/backend.dart | 122 +++++++++++++++++------------------ 2 files changed, 84 insertions(+), 70 deletions(-) diff --git a/app/lib/package/backend.dart b/app/lib/package/backend.dart index 421115320..6a7734c62 100644 --- a/app/lib/package/backend.dart +++ b/app/lib/package/backend.dart @@ -970,21 +970,34 @@ class PackageBackend { /// getting it from cache if available. /// /// This returns gzipped UTF-8 encoded JSON. - Future> listVersionsGzCachedBytes(String package) async { - final body = await cache.packageDataGz(package).get(() async { - final data = await listVersions(package); - final raw = jsonUtf8Encoder.convert(data.toJson()); - return gzip.encode(raw); - }); - return body!; + Future> listVersionsGzCachedBytes( + String package, { + bool refreshVersionsCache = false, + }) async { + final entry = cache.packageDataGz(package); + final cached = refreshVersionsCache ? null : await entry.get(); + if (cached != null) { + return cached; + } + final data = await listVersions(package); + final raw = jsonUtf8Encoder.convert(data.toJson()); + final zipped = gzip.encode(raw); + await entry.set(zipped); + return zipped; } /// Returns the known versions of [package] (via [listVersions]), /// getting it from the cache if available. /// /// The available versions are sorted by their semantic version number (ascending). - Future listVersionsCached(String package) async { - final data = await listVersionsGzCachedBytes(package); + Future listVersionsCached( + String package, { + bool refreshVersionsCache = false, + }) async { + final data = await listVersionsGzCachedBytes( + package, + refreshVersionsCache: refreshVersionsCache, + ); return api.PackageData.fromJson( utf8JsonDecoder.convert(gzip.decode(data)) as Map, ); @@ -2455,6 +2468,7 @@ class _VersionTransactionDataAcccess { () => taskBackend.trackPackage( package, updateDependents: taskUpdateDependents, + refreshVersionsCache: true, ), ), if (!skipExport) diff --git a/app/lib/task/backend.dart b/app/lib/task/backend.dart index 68193affa..b2bb54533 100644 --- a/app/lib/task/backend.dart +++ b/app/lib/task/backend.dart @@ -3,6 +3,7 @@ import 'dart:convert'; import 'dart:io'; import 'dart:typed_data'; +import 'package:_pub_shared/data/package_api.dart' as package_api; import 'package:_pub_shared/data/task_api.dart' as api; import 'package:_pub_shared/data/task_payload.dart'; import 'package:_pub_shared/worker/limits.dart'; @@ -363,44 +364,31 @@ class TaskBackend { Future trackPackage( String packageName, { bool updateDependents = false, + bool refreshVersionsCache = false, }) async { var lastVersionCreated = initialTimestamp; String? latestVersion; - final changed = await withRetryTransaction(_db, (tx) async { - // Lookup Package and PackageVersion in the same transaction. - final packageFuture = tx.packages.lookupOrNull(packageName); - final packageVersionsFuture = tx.versions.listVersionsOfPackage( + late package_api.PackageData data; + try { + data = await packageBackend.listVersionsCached( packageName, + refreshVersionsCache: refreshVersionsCache, ); - final stateFuture = tx.tasks.lookupOrNull(packageName); - // Ensure we await all futures! - await Future.wait([packageFuture, packageVersionsFuture, stateFuture]); - final state = await stateFuture; - final package = await packageFuture; - final packageVersions = await packageVersionsFuture; - - if (package == null) { - return false; // assume package was deleted! - } - latestVersion = package.latestVersion; + } on NotFoundException catch (_) { + // If package is not visible, we should remove it! + await _db.tasks.deleteAllStates(packageName); + return; + } + final versions = _versionsToTrack( + data, + ).map((v) => v.canonicalizedVersion).toList(); + final changed = await withRetryTransaction(_db, (tx) async { + final state = await tx.tasks.lookupOrNull(packageName); + latestVersion = data.latest.version; // Update the timestamp for when the last version was published. // This is used if we need to update dependents. - lastVersionCreated = packageVersions.map((pv) => pv.created!).max; - - // If package is not visible, we should remove it! - if (package.isNotVisible) { - await tx.tasks.deleteAllStates( - packageName, - currentRuntimeKey: state?.key, - ); - return true; - } - - // Determined the set of versions to track - final versions = _versionsToTrack(package, packageVersions).map( - (v) => v.canonicalizedVersion, // add extra sanity! - ); + lastVersionCreated = data.versions.map((pv) => pv.published!).max; // Ensure we have PackageState entity if (state == null) { @@ -1202,16 +1190,11 @@ PackageVersionStateInfo _authorizeWorkerCallback( /// * Latest preview release (if newer than latest stable release); /// * Latest prerelease (if newer than latest preview release); /// * 5 latest major versions (if any). -List _versionsToTrack( - Package package, - List packageVersions, -) { - final visibleVersions = packageVersions +List _versionsToTrack(package_api.PackageData data) { + final visibleVersions = data.versions // Ignore retracted versions - .where((pv) => !pv.isRetracted) - // Ignore moderated versions - .where((pv) => pv.isVisible) - .map((pv) => pv.semanticVersion) + .where((pv) => !(pv.retracted ?? false)) + .map((pv) => Version.parse(pv.version)) .toSet(); final visibleStableVersions = visibleVersions @@ -1219,17 +1202,37 @@ List _versionsToTrack( .where((v) => !v.isPreRelease) .toList() ..sort((a, b) => -a.compareTo(b)); + + final latestVersion = Version.parse(data.latest.version); + + // consider preview version if it is newer than current latest release + final stableAfterLatest = visibleStableVersions + .where((a) => latestVersion.compareTo(a) < 0) + .toList(); + final latestPreview = stableAfterLatest.isEmpty + ? null + : stableAfterLatest.reduce((a, b) => a.compareTo(b) < 0 ? b : a); + + // consider prerelease version if it is newer than current latest release + final prereleaseAfterLatest = visibleVersions + .where((a) => a.isPreRelease) + .where((a) => latestVersion.compareTo(a) < 0) + .toList(); + final latestPrerelease = prereleaseAfterLatest.isEmpty + ? null + : prereleaseAfterLatest.reduce((a, b) => a.compareTo(b) < 0 ? b : a); return { // Always analyze latest version (may be non-stable if package has only prerelease versions). - package.latestSemanticVersion, + latestVersion, - // Consider latest two stable versions to keep previously analyzed results on new package publishing. - ...visibleStableVersions.take(2), + // Consider latest two older stable versions to keep previously analyzed results on new package publishing. + ...visibleStableVersions + .where((a) => a.compareTo(latestVersion) <= 0) + .take(2), - // Only consider prerelease and preview versions, if they are newer than - // the current stable release. - if (package.showPrereleaseVersion) package.latestPrereleaseSemanticVersion, - if (package.showPreviewVersion) package.latestPreviewSemanticVersion, + // Consider the latest prerelease and preview version. + if (latestPrerelease != null) latestPrerelease, + if (latestPreview != null) latestPreview, // Consider 5 latest major versions, if any: ...visibleStableVersions @@ -1374,6 +1377,18 @@ final class _TaskDataAccess { } } + Future deleteAllStates(String name) async { + await withRetryTransaction(_db, (tx) async { + // also delete earlier runtime versions + for (final rv in acceptedRuntimeVersions) { + final s = await lookupOrNull(name, runtimeVersion: rv); + if (s != null) { + tx.delete(s.key); + } + } + }); + } + /// Returns whether the entry has been updated. Future updateDependencyChanged( String package, @@ -1435,21 +1450,6 @@ class _TaskTransactionDataAcccess { return await _tx.lookupOrNull(key); } - Future deleteAllStates(String name, {Key? currentRuntimeKey}) async { - if (currentRuntimeKey != null) { - _tx.delete(currentRuntimeKey); - } - // also delete earlier runtime versions - for (final rv in acceptedRuntimeVersions.where( - (rv) => rv != runtimeVersion, - )) { - final s = await lookupOrNull(name, runtimeVersion: rv); - if (s != null) { - _tx.delete(s.key); - } - } - } - Future insert(PackageState state) async { _tx.insert(state); } From 3a407861e6815614398a2a51a4b0c98d0be941a9 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Wed, 5 Nov 2025 09:24:22 +0100 Subject: [PATCH 2/2] fix test by purging cache --- app/test/service/security_advisory/security_advisory_test.dart | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/test/service/security_advisory/security_advisory_test.dart b/app/test/service/security_advisory/security_advisory_test.dart index 456c8bcd1..41208de90 100644 --- a/app/test/service/security_advisory/security_advisory_test.dart +++ b/app/test/service/security_advisory/security_advisory_test.dart @@ -506,6 +506,9 @@ void main() { expect(oxygenPkg!.latestAdvisory, syncTime); expect(neonPkg!.latestAdvisory, isNull); + // TODO(https://github.com/dart-lang/pub-dev/issues/9056): consider purging the package cache in `ingestSecurityAdvisory`. + await purgePackageCache('oxygen'); + final client = await createFakeAuthPubApiClient( email: adminAtPubDevEmail, );