From 372b6ec71f18ce4b962b8fa81e122eb809bc354c Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Fri, 18 Oct 2024 15:36:43 +0200 Subject: [PATCH 1/3] Typing for ExportedApi and minimal test --- app/lib/package/api_export/exported_api.dart | 16 ++-- .../package/api_export/exported_api_test.dart | 93 +++++++++++++++++++ 2 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 app/test/package/api_export/exported_api_test.dart diff --git a/app/lib/package/api_export/exported_api.dart b/app/lib/package/api_export/exported_api.dart index 8261291bb3..e6bd54dfe8 100644 --- a/app/lib/package/api_export/exported_api.dart +++ b/app/lib/package/api_export/exported_api.dart @@ -2,6 +2,8 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'package:_pub_shared/data/advisories_api.dart'; +import 'package:_pub_shared/data/package_api.dart'; import 'package:clock/clock.dart'; import 'package:gcloud/storage.dart'; import 'package:logging/logging.dart'; @@ -43,7 +45,8 @@ final class ExportedApi { ExportedPackage._(this, packageName); /// Interface for writing `/api/package-name-completion-data` - ExportedJsonFile get packageNameCompletionData => ExportedJsonFile._( + ExportedJsonFile> get packageNameCompletionData => + ExportedJsonFile>._( this, '/api/package-name-completion-data', Duration(hours: 8), @@ -178,7 +181,7 @@ final class ExportedPackage { ExportedPackage._(this._owner, this._package); - ExportedJsonFile _suffix(String suffix) => ExportedJsonFile._( + ExportedJsonFile _suffix(String suffix) => ExportedJsonFile._( _owner, '/api/packages/$_package$suffix', Duration(minutes: 10), @@ -187,10 +190,11 @@ final class ExportedPackage { /// Interface for writing `/api/packages/`. /// /// Which contains version listing information. - ExportedJsonFile get versions => _suffix(''); + ExportedJsonFile get versions => _suffix(''); /// Interface for writing `/api/packages//advisories`. - ExportedJsonFile get advisories => _suffix('/advisories'); + ExportedJsonFile get advisories => + _suffix('/advisories'); /// Interace for writing `/api/archives/-.tar.gz`. ExportedBlob tarball(String version) => ExportedBlob._( @@ -239,7 +243,7 @@ sealed class ExportedObject { /// * `Content-Type`, /// * `Content-Encoding`, and, /// * `Cache-Control`. -final class ExportedJsonFile extends ExportedObject { +final class ExportedJsonFile extends ExportedObject { static final _jsonGzip = json.fuse(utf8).fuse(gzip); final Duration _maxAge; @@ -256,7 +260,7 @@ final class ExportedJsonFile extends ExportedObject { ); /// Write [data] as gzipped JSON in UTF-8 format. - Future write(Map data) async { + Future write(T data) async { final gzipped = _jsonGzip.encode(data); await Future.wait(_owner._prefixes.map((prefix) async { await _owner._pool.withResource(() async { diff --git a/app/test/package/api_export/exported_api_test.dart b/app/test/package/api_export/exported_api_test.dart new file mode 100644 index 0000000000..7cfe5f006a --- /dev/null +++ b/app/test/package/api_export/exported_api_test.dart @@ -0,0 +1,93 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:_pub_shared/data/package_api.dart'; +import 'package:gcloud/storage.dart'; +import 'package:googleapis/storage/v1.dart' show DetailedApiRequestError; +import 'package:pub_dev/package/api_export/exported_api.dart'; +import 'package:pub_dev/shared/storage.dart'; +import 'package:pub_dev/shared/utils.dart'; +import 'package:test/test.dart'; +import '../../shared/test_services.dart'; + +void main() { + testWithFakeTime('ExportedApi', (fakeTime) async { + await storageService.createBucket('exported-api'); + final bucket = storageService.bucket('exported-api'); + + /// Read bytes from bucket + Future readBytes(String path) async { + try { + return await bucket.readAsBytes(path); + } on DetailedApiRequestError catch (e) { + if (e.status == 404) return null; + rethrow; + } + } + + /// Read gzipped JSON from bucket + Future readGzippedJson(String path) async { + final bytes = await readBytes(path); + if (bytes == null) { + return null; + } + return utf8JsonDecoder.convert(gzip.decode(bytes)); + } + + final exportedApi = ExportedApi(storageService, bucket); + + // Test that deletion works when bucket is empty + await exportedApi.package('retry').delete(); + + // Test that GC works when bucket is empty + await exportedApi.garbageCollect({}); + + final retryPkgData1 = PackageData( + name: 'retry', + latest: VersionInfo( + version: '1.2.3', + retracted: false, + pubspec: {}, + archiveUrl: '-', + archiveSha256: '-', + published: DateTime.now(), + ), + versions: [], + ); + + await exportedApi.package('retry').versions.write(retryPkgData1); + + expect( + await readGzippedJson('latest/api/packages/retry'), + json.decode(json.encode(retryPkgData1.toJson())), + ); + + // Check that GC after 10 mins won't delete a package we don't recognize + fakeTime.elapseSync(minutes: 10); + await exportedApi.garbageCollect({}); + expect( + await readGzippedJson('latest/api/packages/retry'), + isNotNull, + ); + + // Check that GC after 2 days won't delete a package we know + fakeTime.elapseSync(days: 2); + await exportedApi.garbageCollect({'retry'}); + expect( + await readGzippedJson('latest/api/packages/retry'), + isNotNull, + ); + + // Check retry after 2 days will delete a package we don't know. + await exportedApi.garbageCollect({}); + expect( + await readGzippedJson('latest/api/packages/retry'), + isNull, + ); + }); +} From c958fa821aa9bb6877543a22d21c037cb4bca19b Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Fri, 18 Oct 2024 15:57:56 +0200 Subject: [PATCH 2/3] Use clock.now() --- app/test/package/api_export/exported_api_test.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/test/package/api_export/exported_api_test.dart b/app/test/package/api_export/exported_api_test.dart index 7cfe5f006a..61baf14aa2 100644 --- a/app/test/package/api_export/exported_api_test.dart +++ b/app/test/package/api_export/exported_api_test.dart @@ -7,6 +7,7 @@ import 'dart:io'; import 'dart:typed_data'; import 'package:_pub_shared/data/package_api.dart'; +import 'package:clock/clock.dart'; import 'package:gcloud/storage.dart'; import 'package:googleapis/storage/v1.dart' show DetailedApiRequestError; import 'package:pub_dev/package/api_export/exported_api.dart'; @@ -55,7 +56,7 @@ void main() { pubspec: {}, archiveUrl: '-', archiveSha256: '-', - published: DateTime.now(), + published: clock.now(), ), versions: [], ); From 788b436a2ba8ff268c1a77b7bf1a0b391efc641f Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Fri, 18 Oct 2024 16:25:39 +0200 Subject: [PATCH 3/3] Added headers to files --- app/lib/package/api_export/exported_api.dart | 4 ++++ app/test/package/api_export/exported_api_test.dart | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/lib/package/api_export/exported_api.dart b/app/lib/package/api_export/exported_api.dart index e6bd54dfe8..97937ec429 100644 --- a/app/lib/package/api_export/exported_api.dart +++ b/app/lib/package/api_export/exported_api.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'dart:async'; import 'dart:convert'; import 'dart:io'; diff --git a/app/test/package/api_export/exported_api_test.dart b/app/test/package/api_export/exported_api_test.dart index 61baf14aa2..3a05a1eabc 100644 --- a/app/test/package/api_export/exported_api_test.dart +++ b/app/test/package/api_export/exported_api_test.dart @@ -1,4 +1,4 @@ -// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file.