Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 14 additions & 6 deletions app/lib/package/api_export/exported_api.dart
Original file line number Diff line number Diff line change
@@ -1,7 +1,13 @@
// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'package:_pub_shared/data/advisories_api.dart';
import 'package:_pub_shared/data/package_api.dart';
import 'package:clock/clock.dart';
import 'package:gcloud/storage.dart';
import 'package:logging/logging.dart';
Expand Down Expand Up @@ -43,7 +49,8 @@ final class ExportedApi {
ExportedPackage._(this, packageName);

/// Interface for writing `/api/package-name-completion-data`
ExportedJsonFile get packageNameCompletionData => ExportedJsonFile._(
ExportedJsonFile<Map<String, Object?>> get packageNameCompletionData =>
ExportedJsonFile<Map<String, Object?>>._(
this,
'/api/package-name-completion-data',
Duration(hours: 8),
Expand Down Expand Up @@ -178,7 +185,7 @@ final class ExportedPackage {

ExportedPackage._(this._owner, this._package);

ExportedJsonFile _suffix(String suffix) => ExportedJsonFile._(
ExportedJsonFile<T> _suffix<T>(String suffix) => ExportedJsonFile<T>._(
_owner,
'/api/packages/$_package$suffix',
Duration(minutes: 10),
Expand All @@ -187,10 +194,11 @@ final class ExportedPackage {
/// Interface for writing `/api/packages/<package>`.
///
/// Which contains version listing information.
ExportedJsonFile get versions => _suffix('');
ExportedJsonFile<PackageData> get versions => _suffix<PackageData>('');

/// Interface for writing `/api/packages/<package>/advisories`.
ExportedJsonFile get advisories => _suffix('/advisories');
ExportedJsonFile<ListAdvisoriesResponse> get advisories =>
_suffix<ListAdvisoriesResponse>('/advisories');

/// Interace for writing `/api/archives/<package>-<version>.tar.gz`.
ExportedBlob tarball(String version) => ExportedBlob._(
Expand Down Expand Up @@ -239,7 +247,7 @@ sealed class ExportedObject {
/// * `Content-Type`,
/// * `Content-Encoding`, and,
/// * `Cache-Control`.
final class ExportedJsonFile extends ExportedObject {
final class ExportedJsonFile<T> extends ExportedObject {
static final _jsonGzip = json.fuse(utf8).fuse(gzip);
final Duration _maxAge;

Expand All @@ -256,7 +264,7 @@ final class ExportedJsonFile extends ExportedObject {
);

/// Write [data] as gzipped JSON in UTF-8 format.
Future<void> write(Map<String, Object?> data) async {
Future<void> write(T data) async {
final gzipped = _jsonGzip.encode(data);
await Future.wait(_owner._prefixes.map((prefix) async {
await _owner._pool.withResource(() async {
Expand Down
94 changes: 94 additions & 0 deletions app/test/package/api_export/exported_api_test.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';

import 'package:_pub_shared/data/package_api.dart';
import 'package:clock/clock.dart';
import 'package:gcloud/storage.dart';
import 'package:googleapis/storage/v1.dart' show DetailedApiRequestError;
import 'package:pub_dev/package/api_export/exported_api.dart';
import 'package:pub_dev/shared/storage.dart';
import 'package:pub_dev/shared/utils.dart';
import 'package:test/test.dart';
import '../../shared/test_services.dart';

void main() {
testWithFakeTime('ExportedApi', (fakeTime) async {
await storageService.createBucket('exported-api');
final bucket = storageService.bucket('exported-api');

/// Read bytes from bucket
Future<Uint8List?> readBytes(String path) async {
try {
return await bucket.readAsBytes(path);
} on DetailedApiRequestError catch (e) {
if (e.status == 404) return null;
rethrow;
}
}

/// Read gzipped JSON from bucket
Future<Object?> readGzippedJson(String path) async {
final bytes = await readBytes(path);
if (bytes == null) {
return null;
}
return utf8JsonDecoder.convert(gzip.decode(bytes));
}

final exportedApi = ExportedApi(storageService, bucket);

// Test that deletion works when bucket is empty
await exportedApi.package('retry').delete();

// Test that GC works when bucket is empty
await exportedApi.garbageCollect({});

final retryPkgData1 = PackageData(
name: 'retry',
latest: VersionInfo(
version: '1.2.3',
retracted: false,
pubspec: {},
archiveUrl: '-',
archiveSha256: '-',
published: clock.now(),
),
versions: [],
);

await exportedApi.package('retry').versions.write(retryPkgData1);

expect(
await readGzippedJson('latest/api/packages/retry'),
json.decode(json.encode(retryPkgData1.toJson())),
);

// Check that GC after 10 mins won't delete a package we don't recognize
fakeTime.elapseSync(minutes: 10);
await exportedApi.garbageCollect({});
expect(
await readGzippedJson('latest/api/packages/retry'),
isNotNull,
);

// Check that GC after 2 days won't delete a package we know
fakeTime.elapseSync(days: 2);
await exportedApi.garbageCollect({'retry'});
expect(
await readGzippedJson('latest/api/packages/retry'),
isNotNull,
);

// Check retry after 2 days will delete a package we don't know.
await exportedApi.garbageCollect({});
expect(
await readGzippedJson('latest/api/packages/retry'),
isNull,
);
});
}
Loading