diff --git a/app/lib/admin/actions/moderate_package.dart b/app/lib/admin/actions/moderate_package.dart index 7cf10f8669..f04abee2dc 100644 --- a/app/lib/admin/actions/moderate_package.dart +++ b/app/lib/admin/actions/moderate_package.dart @@ -82,7 +82,7 @@ Note: the action may take a longer time to complete as the public archive bucket }); // retract or re-populate public archive files - await packageBackend.packageStorage.updatePublicArchiveBucket( + await packageBackend.tarballStorage.updatePublicArchiveBucket( package: package, ageCheckThreshold: Duration.zero, deleteIfOlder: Duration.zero, diff --git a/app/lib/admin/actions/moderate_package_versions.dart b/app/lib/admin/actions/moderate_package_versions.dart index 630eb5cb1e..9866847aeb 100644 --- a/app/lib/admin/actions/moderate_package_versions.dart +++ b/app/lib/admin/actions/moderate_package_versions.dart @@ -115,7 +115,7 @@ Set the moderated flag on a package version (updating the flag and the timestamp }); // retract or re-populate public archive files - await packageBackend.packageStorage.updatePublicArchiveBucket( + await packageBackend.tarballStorage.updatePublicArchiveBucket( package: package, ageCheckThreshold: Duration.zero, deleteIfOlder: Duration.zero, diff --git a/app/lib/admin/backend.dart b/app/lib/admin/backend.dart index 3747d2c134..058769157e 100644 --- a/app/lib/admin/backend.dart +++ b/app/lib/admin/backend.dart @@ -820,7 +820,7 @@ class AdminBackend { 'Deleting moderated package version: ${version.qualifiedVersionKey}'); // deleting from canonical bucket - await packageBackend.packageStorage + await packageBackend.tarballStorage .deleteArchiveFromCanonicalBucket(version.package, version.version!); // deleting from datastore diff --git a/app/lib/package/backend.dart b/app/lib/package/backend.dart index 78343b2e53..6d1c0bf5df 100644 --- a/app/lib/package/backend.dart +++ b/app/lib/package/backend.dart @@ -17,7 +17,7 @@ import 'package:logging/logging.dart'; import 'package:meta/meta.dart'; import 'package:pool/pool.dart'; import 'package:pub_dev/package/api_export/export_api_to_bucket.dart'; -import 'package:pub_dev/package/package_storage.dart'; +import 'package:pub_dev/package/tarball_storage.dart'; import 'package:pub_dev/service/async_queue/async_queue.dart'; import 'package:pub_dev/service/rate_limit/rate_limit.dart'; import 'package:pub_dev/shared/versions.dart'; @@ -81,8 +81,8 @@ class PackageBackend { /// - `tmp/$guid` (incoming package archive that was uploaded, but not yet processed) final Bucket _incomingBucket; - /// The storage handling for the archive files. - final PackageStorage packageStorage; + /// The storage handling for the package archive files. + final TarballStorage tarballStorage; @visibleForTesting int maxVersionsPerPackage = _defaultMaxVersionsPerPackage; @@ -93,8 +93,8 @@ class PackageBackend { this._incomingBucket, Bucket canonicalBucket, Bucket publicBucket, - ) : packageStorage = - PackageStorage(db, storage, canonicalBucket, publicBucket); + ) : tarballStorage = + TarballStorage(db, storage, canonicalBucket, publicBucket); /// Whether the package exists and is not blocked or deleted. Future isPackageVisible(String package) async { @@ -329,7 +329,7 @@ class PackageBackend { // NOTE: We should maybe check for existence first? // return storage.bucket(bucket).info(object) // .then((info) => info.downloadLink); - return packageStorage.getPublicDownloadUrl(package, cv!); + return tarballStorage.getPublicDownloadUrl(package, cv!); } /// Updates the stable, prerelease and preview versions of [package]. @@ -943,7 +943,7 @@ class PackageBackend { // Check canonical archive. final canonicalContentMatch = - await packageStorage.matchArchiveContentInCanonical( + await tarballStorage.matchArchiveContentInCanonical( pubspec.name, versionString, fileBytes, @@ -1184,14 +1184,14 @@ class PackageBackend { ); if (!hasCanonicalArchiveObject) { // Copy archive to canonical bucket. - await packageStorage.copyFromTempToCanonicalBucket( + await tarballStorage.copyFromTempToCanonicalBucket( sourceAbsoluteObjectName: _incomingBucket.absoluteObjectName(tmpObjectName(guid)), package: newVersion.package, version: newVersion.version!, ); } - await packageStorage.copyArchiveFromCanonicalToPublicBucket( + await tarballStorage.copyArchiveFromCanonicalToPublicBucket( newVersion.package, newVersion.version!); final inserts = [ @@ -1256,7 +1256,7 @@ class PackageBackend { apiExporter! .updatePackageVersion(newVersion.package, newVersion.version!), ]); - await packageStorage.updateContentDispositionOnPublicBucket( + await tarballStorage.updateContentDispositionOnPublicBucket( newVersion.package, newVersion.version!); } catch (e, st) { final v = newVersion.qualifiedVersionKey; @@ -1634,12 +1634,12 @@ class PackageBackend { /// Deletes the tarball of a [package] in the given [version] permanently. Future removePackageTarball(String package, String version) async { - await packageStorage.deleteArchiveFromAllBuckets(package, version); + await tarballStorage.deleteArchiveFromAllBuckets(package, version); } /// Gets the file info of a [package] in the given [version]. Future packageTarballInfo(String package, String version) async { - return await packageStorage.getPublicBucketArchiveInfo(package, version); + return await tarballStorage.getPublicBucketArchiveInfo(package, version); } void _updatePackageAutomatedPublishingLock( diff --git a/app/lib/package/package_storage.dart b/app/lib/package/tarball_storage.dart similarity index 99% rename from app/lib/package/package_storage.dart rename to app/lib/package/tarball_storage.dart index 22a9c3e1c5..46b91b73ad 100644 --- a/app/lib/package/package_storage.dart +++ b/app/lib/package/tarball_storage.dart @@ -7,6 +7,7 @@ import 'dart:typed_data'; import 'package:crypto/crypto.dart'; import 'package:gcloud/storage.dart'; import 'package:logging/logging.dart'; +import 'package:meta/meta.dart'; import '../shared/datastore.dart'; import '../shared/storage.dart'; import '../shared/utils.dart'; @@ -16,13 +17,14 @@ import 'models.dart'; final _logger = Logger('package_storage'); /// The GCS object name of a tarball object - excluding leading '/'. +@visibleForTesting String tarballObjectName(String package, String version) => '${_tarballObjectNamePackagePrefix(package)}$version.tar.gz'; /// The GCS object prefix of a tarball object for a given [package] - excluding leading '/'. String _tarballObjectNamePackagePrefix(String package) => 'packages/$package-'; -class PackageStorage { +class TarballStorage { final DatastoreDB _dbService; final Storage _storage; @@ -36,7 +38,7 @@ class PackageStorage { /// - `packages/$package-$version.tar.gz` (package archive) final Bucket _publicBucket; - PackageStorage( + TarballStorage( this._dbService, this._storage, this._canonicalBucket, diff --git a/app/lib/shared/integrity.dart b/app/lib/shared/integrity.dart index 3b496b5123..01569849ed 100644 --- a/app/lib/shared/integrity.dart +++ b/app/lib/shared/integrity.dart @@ -603,7 +603,7 @@ class IntegrityChecker { Uri archiveDownloadUri, { required bool shouldBeInPublicBucket, }) async* { - final canonicalInfo = await packageBackend.packageStorage + final canonicalInfo = await packageBackend.tarballStorage .getCanonicalBucketArchiveInfo(pv.package, pv.version!); if (canonicalInfo == null) { yield 'PackageVersion "${pv.qualifiedVersionKey}" has no matching canonical archive file.'; @@ -628,7 +628,7 @@ class IntegrityChecker { yield 'Canonical archive for PackageVersion "${pv.qualifiedVersionKey}" differs from public bucket.'; } - final publicInfo = await packageBackend.packageStorage + final publicInfo = await packageBackend.tarballStorage .getPublicBucketArchiveInfo(pv.package, pv.version!); if (!canonicalInfo.hasSameSignatureAs(publicInfo)) { yield 'Canonical archive for PackageVersion "${pv.qualifiedVersionKey}" differs in the public bucket.'; diff --git a/app/lib/tool/neat_task/pub_dev_tasks.dart b/app/lib/tool/neat_task/pub_dev_tasks.dart index 1e5d3b0b71..ba3ec03667 100644 --- a/app/lib/tool/neat_task/pub_dev_tasks.dart +++ b/app/lib/tool/neat_task/pub_dev_tasks.dart @@ -114,7 +114,7 @@ void _setupGenericPeriodicTasks() { name: 'sync-public-bucket-from-canonical-bucket', isRuntimeVersioned: false, task: () async => - await packageBackend.packageStorage.updatePublicArchiveBucket(), + await packageBackend.tarballStorage.updatePublicArchiveBucket(), ); // Exports the package name completion data to a bucket. diff --git a/app/test/admin/moderate_package_test.dart b/app/test/admin/moderate_package_test.dart index b4840d2d8f..5959e8a040 100644 --- a/app/test/admin/moderate_package_test.dart +++ b/app/test/admin/moderate_package_test.dart @@ -312,13 +312,13 @@ void main() { await expectStatusCode(404); // another check after background tasks are running - await packageBackend.packageStorage.updatePublicArchiveBucket(); + await packageBackend.tarballStorage.updatePublicArchiveBucket(); await expectStatusCode(404); await _moderate('oxygen', state: false, caseId: mc.caseId); await expectStatusCode(200); // another check after background tasks are running - await packageBackend.packageStorage.updatePublicArchiveBucket(); + await packageBackend.tarballStorage.updatePublicArchiveBucket(); final restoredBytes = await expectStatusCode(200); expect(restoredBytes, bytes); }); @@ -381,7 +381,7 @@ void main() { // canonical file is present expect( - await packageBackend.packageStorage + await packageBackend.tarballStorage .getCanonicalBucketArchiveInfo('oxygen', '1.2.0'), isNotNull, ); @@ -397,7 +397,7 @@ void main() { isNull, ); expect( - await packageBackend.packageStorage + await packageBackend.tarballStorage .getCanonicalBucketArchiveInfo('oxygen', '1.2.0'), isNull, ); diff --git a/app/test/admin/moderate_package_version_test.dart b/app/test/admin/moderate_package_version_test.dart index babeefb2b4..7a27751439 100644 --- a/app/test/admin/moderate_package_version_test.dart +++ b/app/test/admin/moderate_package_version_test.dart @@ -204,13 +204,13 @@ void main() { await expectStatusCode(200, version: '1.2.0'); // another check after background tasks are running - await packageBackend.packageStorage.updatePublicArchiveBucket(); + await packageBackend.tarballStorage.updatePublicArchiveBucket(); await expectStatusCode(404); await expectStatusCode(200, version: '1.2.0'); await _moderate('oxygen', '1.0.0', state: false); await expectStatusCode(200); - await packageBackend.packageStorage.updatePublicArchiveBucket(); + await packageBackend.tarballStorage.updatePublicArchiveBucket(); final restoredBytes = await expectStatusCode(200); expect(restoredBytes, bytes); }); @@ -429,7 +429,7 @@ void main() { fn: () async { // canonical file is present expect( - await packageBackend.packageStorage + await packageBackend.tarballStorage .getCanonicalBucketArchiveInfo('oxygen', '1.0.0'), isNotNull, ); @@ -459,7 +459,7 @@ void main() { // canonical file is not present expect( - await packageBackend.packageStorage + await packageBackend.tarballStorage .getCanonicalBucketArchiveInfo('oxygen', '1.0.0'), isNull, ); @@ -470,7 +470,7 @@ void main() { isNotNull, ); expect( - await packageBackend.packageStorage + await packageBackend.tarballStorage .getCanonicalBucketArchiveInfo('oxygen', '1.2.0'), isNotNull, ); diff --git a/app/test/package/tarball_storage_namer_test.dart b/app/test/package/tarball_storage_namer_test.dart index 23675516e7..16cbbb5052 100644 --- a/app/test/package/tarball_storage_namer_test.dart +++ b/app/test/package/tarball_storage_namer_test.dart @@ -4,7 +4,7 @@ import 'package:gcloud/storage.dart'; import 'package:pub_dev/package/backend.dart'; -import 'package:pub_dev/package/package_storage.dart'; +import 'package:pub_dev/package/tarball_storage.dart'; import 'package:pub_dev/shared/configuration.dart'; import 'package:pub_dev/shared/storage.dart'; import 'package:test/test.dart'; diff --git a/app/test/package/update_public_bucket_test.dart b/app/test/package/update_public_bucket_test.dart index 89626a5518..79147131ba 100644 --- a/app/test/package/update_public_bucket_test.dart +++ b/app/test/package/update_public_bucket_test.dart @@ -4,7 +4,7 @@ import 'package:gcloud/storage.dart'; import 'package:pub_dev/package/backend.dart'; -import 'package:pub_dev/package/package_storage.dart'; +import 'package:pub_dev/package/tarball_storage.dart'; import 'package:pub_dev/shared/configuration.dart'; import 'package:test/test.dart'; @@ -17,7 +17,7 @@ void main() { Duration? ageCheckThreshold, Duration? deleteIfOlder, }) async { - return await packageBackend.packageStorage.updatePublicArchiveBucket( + return await packageBackend.tarballStorage.updatePublicArchiveBucket( package: package, ageCheckThreshold: ageCheckThreshold ?? Duration.zero, deleteIfOlder: deleteIfOlder ?? const Duration(days: 7),