diff --git a/app/lib/package/backend.dart b/app/lib/package/backend.dart index 0ff39c8ad1..78343b2e53 100644 --- a/app/lib/package/backend.dart +++ b/app/lib/package/backend.dart @@ -75,41 +75,26 @@ PackageBackend get packageBackend => /// Represents the backend for the pub site. class PackageBackend { final DatastoreDB db; - final Storage _storage; /// The Cloud Storage bucket to use for incoming package archives. /// The following files are present: /// - `tmp/$guid` (incoming package archive that was uploaded, but not yet processed) final Bucket _incomingBucket; - /// The Cloud Storage bucket to use for canonical package archives. - /// The following files are present: - /// - `packages/$package-$version.tar.gz` (package archive) - final Bucket _canonicalBucket; - - /// The Cloud Storage bucket to use for public package archives. - /// The following files are present: - /// - `packages/$package-$version.tar.gz` (package archive) - final Bucket _publicBucket; - /// The storage handling for the archive files. - late final packageStorage = PackageStorage( - db, - _storage, - _canonicalBucket, - _publicBucket, - ); + final PackageStorage packageStorage; @visibleForTesting int maxVersionsPerPackage = _defaultMaxVersionsPerPackage; PackageBackend( this.db, - this._storage, + Storage storage, this._incomingBucket, - this._canonicalBucket, - this._publicBucket, - ); + Bucket canonicalBucket, + Bucket publicBucket, + ) : packageStorage = + PackageStorage(db, storage, canonicalBucket, publicBucket); /// Whether the package exists and is not blocked or deleted. Future isPackageVisible(String package) async { @@ -344,8 +329,7 @@ class PackageBackend { // NOTE: We should maybe check for existence first? // return storage.bucket(bucket).info(object) // .then((info) => info.downloadLink); - final object = tarballObjectName(package, Uri.encodeComponent(cv!)); - return Uri.parse(_publicBucket.objectUrl(object)); + return packageStorage.getPublicDownloadUrl(package, cv!); } /// Updates the stable, prerelease and preview versions of [package]. @@ -958,18 +942,15 @@ class PackageBackend { } // Check canonical archive. - final canonicalArchivePath = - tarballObjectName(pubspec.name, versionString); - final canonicalArchiveInfo = - await _canonicalBucket.tryInfo(canonicalArchivePath); - if (canonicalArchiveInfo != null) { - // Actually fetch the archive bytes and do full comparison. - final objectBytes = - await _canonicalBucket.readAsBytes(canonicalArchivePath); - if (!fileBytes.byteToByteEquals(objectBytes)) { - throw PackageRejectedException.versionExists( - pubspec.name, versionString); - } + final canonicalContentMatch = + await packageStorage.matchArchiveContentInCanonical( + pubspec.name, + versionString, + fileBytes, + ); + if (canonicalContentMatch == ContentMatchStatus.different) { + throw PackageRejectedException.versionExists( + pubspec.name, versionString); } // check existences of referenced packages @@ -1006,7 +987,8 @@ class PackageBackend { agent: agent, archive: archive, guid: guid, - hasCanonicalArchiveObject: canonicalArchiveInfo != null, + hasCanonicalArchiveObject: + canonicalContentMatch == ContentMatchStatus.same, ); _logger.info('Tarball uploaded in ${sw.elapsed}.'); _logger.info('Removing temporary object $guid.'); @@ -1202,18 +1184,15 @@ class PackageBackend { ); if (!hasCanonicalArchiveObject) { // Copy archive to canonical bucket. - await _storage.copyObject( - _incomingBucket.absoluteObjectName(tmpObjectName(guid)), - _canonicalBucket.absoluteObjectName( - tarballObjectName(newVersion.package, newVersion.version!)), + await packageStorage.copyFromTempToCanonicalBucket( + sourceAbsoluteObjectName: + _incomingBucket.absoluteObjectName(tmpObjectName(guid)), + package: newVersion.package, + version: newVersion.version!, ); } - await _storage.copyObject( - _canonicalBucket.absoluteObjectName( - tarballObjectName(newVersion.package, newVersion.version!)), - _publicBucket.absoluteObjectName( - tarballObjectName(newVersion.package, newVersion.version!)), - ); + await packageStorage.copyArchiveFromCanonicalToPublicBucket( + newVersion.package, newVersion.version!); final inserts = [ package!, @@ -1277,12 +1256,8 @@ class PackageBackend { apiExporter! .updatePackageVersion(newVersion.package, newVersion.version!), ]); - final objectName = - tarballObjectName(newVersion.package, newVersion.version!); - final info = await _publicBucket.tryInfo(objectName); - if (info != null) { - await updateContentDispositionToAttachment(info, _publicBucket); - } + await packageStorage.updateContentDispositionOnPublicBucket( + newVersion.package, newVersion.version!); } catch (e, st) { final v = newVersion.qualifiedVersionKey; _logger.severe('Error post-processing package upload $v', e, st); @@ -1482,12 +1457,6 @@ class PackageBackend { return existingEmails; } - /// Read the archive bytes from the canonical bucket. - Future> readArchiveBytes(String package, String version) async { - final objectName = tarballObjectName(package, version); - return await _canonicalBucket.readAsBytes(objectName); - } - // Uploaders support. Future inviteUploader( @@ -1665,14 +1634,12 @@ class PackageBackend { /// Deletes the tarball of a [package] in the given [version] permanently. Future removePackageTarball(String package, String version) async { - final object = tarballObjectName(package, version); - await deleteFromBucket(_publicBucket, object); - await deleteFromBucket(_canonicalBucket, object); + await packageStorage.deleteArchiveFromAllBuckets(package, version); } /// Gets the file info of a [package] in the given [version]. Future packageTarballInfo(String package, String version) async { - return await _publicBucket.tryInfo(tarballObjectName(package, version)); + return await packageStorage.getPublicBucketArchiveInfo(package, version); } void _updatePackageAutomatedPublishingLock( diff --git a/app/lib/package/package_storage.dart b/app/lib/package/package_storage.dart index 573ab5d2bb..22a9c3e1c5 100644 --- a/app/lib/package/package_storage.dart +++ b/app/lib/package/package_storage.dart @@ -2,12 +2,16 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:typed_data'; + +import 'package:crypto/crypto.dart'; import 'package:gcloud/storage.dart'; import 'package:logging/logging.dart'; -import 'package:pub_dev/package/backend.dart'; -import 'package:pub_dev/package/models.dart'; -import 'package:pub_dev/shared/datastore.dart'; -import 'package:pub_dev/shared/storage.dart'; +import '../shared/datastore.dart'; +import '../shared/storage.dart'; +import '../shared/utils.dart'; +import 'backend.dart'; +import 'models.dart'; final _logger = Logger('package_storage'); @@ -39,7 +43,7 @@ class PackageStorage { this._publicBucket, ); - /// Gets the object info of the archive file from the public bucket. + /// Gets the object info of the archive file from the canonical bucket. Future getCanonicalBucketArchiveInfo( String package, String version) async { final objectName = tarballObjectName(package, version); @@ -53,6 +57,77 @@ class PackageStorage { return await _publicBucket.tryInfo(objectName); } + /// Returns the publicly available download URL from the storage bucket. + Future getPublicDownloadUrl(String package, String version) async { + final object = tarballObjectName(package, Uri.encodeComponent(version)); + return Uri.parse(_publicBucket.objectUrl(object)); + } + + /// Verifies the content of an archive in the canonical bucket. + Future matchArchiveContentInCanonical( + String package, + String version, + Uint8List bytes, + ) async { + final objectName = tarballObjectName(package, version); + final info = await _canonicalBucket.tryInfo(objectName); + if (info == null) { + return ContentMatchStatus.missing; + } + if (info.length != bytes.length) { + return ContentMatchStatus.different; + } + final md5hash = md5.convert(bytes).bytes; + if (!md5hash.byteToByteEquals(info.md5Hash)) { + return ContentMatchStatus.different; + } + final objectBytes = await _canonicalBucket.readAsBytes(objectName); + if (bytes.byteToByteEquals(objectBytes)) { + return ContentMatchStatus.same; + } else { + return ContentMatchStatus.different; + } + } + + /// Copies the uploaded object from the temp bucket to the canonical bucket. + Future copyFromTempToCanonicalBucket({ + required String sourceAbsoluteObjectName, + required String package, + required String version, + }) async { + await _storage.copyObject( + sourceAbsoluteObjectName, + _canonicalBucket.absoluteObjectName(tarballObjectName(package, version)), + ); + } + + /// Copies archive bytes from canonical bucket to public bucket. + Future copyArchiveFromCanonicalToPublicBucket( + String package, String version) async { + final objectName = tarballObjectName(package, version); + await _storage.copyObject( + _canonicalBucket.absoluteObjectName(objectName), + _publicBucket.absoluteObjectName(objectName), + ); + } + + /// Updates the `content-disposition` header to `attachment` on the public archive file. + Future updateContentDispositionOnPublicBucket( + String package, String version) async { + final info = await getPublicBucketArchiveInfo(package, version); + if (info != null) { + await updateContentDispositionToAttachment(info, _publicBucket); + } + } + + /// Deletes package archive from all buckets. + Future deleteArchiveFromAllBuckets( + String package, String version) async { + final objectName = tarballObjectName(package, version); + await deleteFromBucket(_canonicalBucket, objectName); + await deleteFromBucket(_publicBucket, objectName); + } + /// Deletes the package archive file from the canonical bucket. Future deleteArchiveFromCanonicalBucket( String package, String version) async { @@ -205,3 +280,9 @@ class PublicBucketUpdateStat { bool get isAllZero => archivesUpdated == 0 && archivesToBeDeleted == 0 && archivesDeleted == 0; } + +enum ContentMatchStatus { + missing, + different, + same; +}