Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 3 additions & 15 deletions app/lib/admin/backend.dart
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import 'package:clock/clock.dart';
import 'package:collection/collection.dart';
import 'package:convert/convert.dart';
import 'package:gcloud/service_scope.dart' as ss;
import 'package:gcloud/storage.dart';
import 'package:logging/logging.dart';
import 'package:meta/meta.dart';
import 'package:pool/pool.dart';
Expand All @@ -24,19 +23,14 @@ import '../account/models.dart';
import '../admin/models.dart';
import '../audit/models.dart';
import '../package/backend.dart'
show
checkPackageVersionParams,
packageBackend,
purgePackageCache,
tarballObjectName;
show checkPackageVersionParams, packageBackend, purgePackageCache;
import '../package/models.dart';
import '../publisher/models.dart';
import '../scorecard/backend.dart';
import '../service/email/email_templates.dart';
import '../shared/configuration.dart';
import '../shared/datastore.dart';
import '../shared/exceptions.dart';
import '../shared/storage.dart';
import '../shared/versions.dart';
import '../task/backend.dart';
import 'actions/actions.dart' show AdminAction;
Expand Down Expand Up @@ -794,9 +788,6 @@ class AdminBackend {
@visibleForTesting DateTime? before,
}) async {
before ??= clock.ago(days: 3 * 366).toUtc(); // extra buffer days
final canonicalBucket =
storageService.bucket(activeConfiguration.canonicalPackagesBucketName!);

// delete packages
final pQuery = _db.query<Package>()
..filter('moderatedAt <', before)
Expand Down Expand Up @@ -829,11 +820,8 @@ class AdminBackend {
'Deleting moderated package version: ${version.qualifiedVersionKey}');

// deleting from canonical bucket
final objectName = tarballObjectName(version.package, version.version!);
final info = await canonicalBucket.tryInfo(objectName);
if (info != null) {
await canonicalBucket.delete(objectName);
}
await packageBackend.packageStorage
.deleteArchiveFromCanonicalBucket(version.package, version.version!);

// deleting from datastore
await withRetryTransaction(_db, (tx) async {
Expand Down
7 changes: 0 additions & 7 deletions app/lib/package/backend.dart
Original file line number Diff line number Diff line change
Expand Up @@ -1938,13 +1938,6 @@ DerivedPackageVersionEntities derivePackageVersionEntities({
return DerivedPackageVersionEntities(versionInfo, assets);
}

/// The GCS object name of a tarball object - excluding leading '/'.
String tarballObjectName(String package, String version) =>
'${tarballObjectNamePackagePrefix(package)}$version.tar.gz';

/// The GCS object prefix of a tarball object for a given [package] - excluding leading '/'.
String tarballObjectNamePackagePrefix(String package) => 'packages/$package-';

/// The GCS object name of an temporary object [guid] - excluding leading '/'.
@visibleForTesting
String tmpObjectName(String guid) => 'tmp/$guid';
Expand Down
36 changes: 34 additions & 2 deletions app/lib/package/package_storage.dart
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,13 @@ import 'package:pub_dev/shared/storage.dart';

final _logger = Logger('package_storage');

/// The GCS object name of a tarball object - excluding leading '/'.
String tarballObjectName(String package, String version) =>
'${_tarballObjectNamePackagePrefix(package)}$version.tar.gz';

/// The GCS object prefix of a tarball object for a given [package] - excluding leading '/'.
String _tarballObjectNamePackagePrefix(String package) => 'packages/$package-';

class PackageStorage {
final DatastoreDB _dbService;
final Storage _storage;
Expand All @@ -32,6 +39,30 @@ class PackageStorage {
this._publicBucket,
);

/// Gets the object info of the archive file from the public bucket.
Future<ObjectInfo?> getCanonicalBucketArchiveInfo(
String package, String version) async {
final objectName = tarballObjectName(package, version);
return await _canonicalBucket.tryInfo(objectName);
}

/// Gets the object info of the archive file from the public bucket.
Future<ObjectInfo?> getPublicBucketArchiveInfo(
String package, String version) async {
final objectName = tarballObjectName(package, version);
return await _publicBucket.tryInfo(objectName);
}

/// Deletes the package archive file from the canonical bucket.
Future<void> deleteArchiveFromCanonicalBucket(
String package, String version) async {
final objectName = tarballObjectName(package, version);
final info = await _canonicalBucket.tryInfo(objectName);
if (info != null) {
await _canonicalBucket.delete(objectName);
}
}

/// Updates the public package archive:
/// - copies missing archive objects from canonical to public bucket,
/// - deletes leftover objects from public bucket
Expand Down Expand Up @@ -92,8 +123,9 @@ class PackageStorage {
objectNamesInPublicBucket.add(objectName);
}

final filterForNamePrefix =
package == null ? 'packages/' : tarballObjectNamePackagePrefix(package);
final filterForNamePrefix = package == null
? 'packages/'
: _tarballObjectNamePackagePrefix(package);
await for (final entry in _publicBucket.list(prefix: filterForNamePrefix)) {
// Skip non-objects.
if (!entry.isObject) {
Expand Down
13 changes: 4 additions & 9 deletions app/lib/shared/integrity.dart
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import 'package:_pub_shared/search/tags.dart';
import 'package:_pub_shared/utils/http.dart';
import 'package:clock/clock.dart';
import 'package:crypto/crypto.dart';
import 'package:gcloud/storage.dart';
import 'package:http/http.dart' as http;
import 'package:logging/logging.dart';
import 'package:pool/pool.dart';
Expand Down Expand Up @@ -604,10 +603,8 @@ class IntegrityChecker {
Uri archiveDownloadUri, {
required bool shouldBeInPublicBucket,
}) async* {
final canonicalInfo = await storageService
.bucket(activeConfiguration.canonicalPackagesBucketName!)
// ignore: invalid_use_of_visible_for_testing_member
.tryInfo(tarballObjectName(pv.package, pv.version!));
final canonicalInfo = await packageBackend.packageStorage
.getCanonicalBucketArchiveInfo(pv.package, pv.version!);
if (canonicalInfo == null) {
yield 'PackageVersion "${pv.qualifiedVersionKey}" has no matching canonical archive file.';
return;
Expand All @@ -631,10 +628,8 @@ class IntegrityChecker {
yield 'Canonical archive for PackageVersion "${pv.qualifiedVersionKey}" differs from public bucket.';
}

final publicInfo = await storageService
.bucket(activeConfiguration.publicPackagesBucketName!)
// ignore: invalid_use_of_visible_for_testing_member
.tryInfo(tarballObjectName(pv.package, pv.version!));
final publicInfo = await packageBackend.packageStorage
.getPublicBucketArchiveInfo(pv.package, pv.version!);
if (!canonicalInfo.hasSameSignatureAs(publicInfo)) {
yield 'Canonical archive for PackageVersion "${pv.qualifiedVersionKey}" differs in the public bucket.';
}
Expand Down
10 changes: 4 additions & 6 deletions app/test/admin/moderate_package_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import 'package:_pub_shared/data/account_api.dart';
import 'package:_pub_shared/data/admin_api.dart';
import 'package:_pub_shared/data/package_api.dart';
import 'package:clock/clock.dart';
import 'package:gcloud/storage.dart';
import 'package:http/http.dart' as http;
import 'package:pub_dev/account/backend.dart';
import 'package:pub_dev/admin/actions/actions.dart';
Expand All @@ -21,7 +20,6 @@ import 'package:pub_dev/scorecard/backend.dart';
import 'package:pub_dev/search/backend.dart';
import 'package:pub_dev/shared/configuration.dart';
import 'package:pub_dev/shared/datastore.dart';
import 'package:pub_dev/shared/storage.dart';
import 'package:test/test.dart';

import '../admin/models_test.dart';
Expand Down Expand Up @@ -382,10 +380,9 @@ void main() {
});

// canonical file is present
final bucket = storageService
.bucket(activeConfiguration.canonicalPackagesBucketName!);
expect(
await bucket.tryInfo(tarballObjectName('oxygen', '1.2.0')),
await packageBackend.packageStorage
.getCanonicalBucketArchiveInfo('oxygen', '1.2.0'),
isNotNull,
);

Expand All @@ -400,7 +397,8 @@ void main() {
isNull,
);
expect(
await bucket.tryInfo(tarballObjectName('oxygen', '1.2.0')),
await packageBackend.packageStorage
.getCanonicalBucketArchiveInfo('oxygen', '1.2.0'),
isNull,
);

Expand Down
13 changes: 6 additions & 7 deletions app/test/admin/moderate_package_version_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import 'package:_pub_shared/data/account_api.dart';
import 'package:_pub_shared/data/admin_api.dart';
import 'package:_pub_shared/data/package_api.dart';
import 'package:clock/clock.dart';
import 'package:gcloud/storage.dart';
import 'package:http/http.dart' as http;
import 'package:pub_dev/admin/backend.dart';
import 'package:pub_dev/admin/models.dart';
Expand All @@ -20,7 +19,6 @@ import 'package:pub_dev/search/backend.dart';
import 'package:pub_dev/shared/configuration.dart';
import 'package:pub_dev/shared/datastore.dart';
import 'package:pub_dev/shared/exceptions.dart';
import 'package:pub_dev/shared/storage.dart';
import 'package:pub_dev/task/backend.dart';
import 'package:test/test.dart';

Expand Down Expand Up @@ -430,10 +428,9 @@ void main() {
'cleanup deletes datastore entities and canonical archive file',
fn: () async {
// canonical file is present
final bucket = storageService
.bucket(activeConfiguration.canonicalPackagesBucketName!);
expect(
await bucket.tryInfo(tarballObjectName('oxygen', '1.0.0')),
await packageBackend.packageStorage
.getCanonicalBucketArchiveInfo('oxygen', '1.0.0'),
isNotNull,
);

Expand Down Expand Up @@ -462,7 +459,8 @@ void main() {

// canonical file is not present
expect(
await bucket.tryInfo(tarballObjectName('oxygen', '1.0.0')),
await packageBackend.packageStorage
.getCanonicalBucketArchiveInfo('oxygen', '1.0.0'),
isNull,
);

Expand All @@ -472,7 +470,8 @@ void main() {
isNotNull,
);
expect(
await bucket.tryInfo(tarballObjectName('oxygen', '1.2.0')),
await packageBackend.packageStorage
.getCanonicalBucketArchiveInfo('oxygen', '1.2.0'),
isNotNull,
);
});
Expand Down
1 change: 1 addition & 0 deletions app/test/package/tarball_storage_namer_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import 'package:gcloud/storage.dart';
import 'package:pub_dev/package/backend.dart';
import 'package:pub_dev/package/package_storage.dart';
import 'package:pub_dev/shared/configuration.dart';
import 'package:pub_dev/shared/storage.dart';
import 'package:test/test.dart';
Expand Down
Loading