From 396f085faa874889b69a166cdf236ab9ca845dc4 Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Mon, 28 Oct 2024 13:07:37 +0100 Subject: [PATCH 1/3] Added "updated" metadata to the ExportedApi --- app/lib/package/api_export/exported_api.dart | 116 +++++++++++++------ 1 file changed, 83 insertions(+), 33 deletions(-) diff --git a/app/lib/package/api_export/exported_api.dart b/app/lib/package/api_export/exported_api.dart index 370df6c803..cb62a2977d 100644 --- a/app/lib/package/api_export/exported_api.dart +++ b/app/lib/package/api_export/exported_api.dart @@ -259,21 +259,24 @@ final class ExportedJsonFile extends ExportedObject { this._maxAge, ) : super._(_owner, _objectName); - late final _metadata = ObjectMetadata( - contentType: 'application/json; charset="utf-8"', - contentEncoding: 'gzip', - cacheControl: 'public, max-age=${_maxAge.inSeconds}', - ); - /// Write [data] as gzipped JSON in UTF-8 format. Future write(T data) async { final gzipped = _jsonGzip.encode(data); + final metadata = ObjectMetadata( + contentType: 'application/json; charset="utf-8"', + contentEncoding: 'gzip', + cacheControl: 'public, max-age=${_maxAge.inSeconds}', + custom: { + 'updated': clock.now().toIso8601String(), + }, + ); + await Future.wait(_owner._prefixes.map((prefix) async { await _owner._pool.withResource(() async { await _owner._bucket.writeBytesIfDifferent( prefix + _objectName, gzipped, - metadata: _metadata, + metadata, ); }); })); @@ -299,52 +302,88 @@ final class ExportedBlob extends ExportedObject { this._maxAge, ) : super._(_owner, _objectName); - late final _metadata = ObjectMetadata( - contentType: _contentType, - cacheControl: 'public, max-age=${_maxAge.inSeconds}', - contentDisposition: 'attachment; filename="$_filename"', - ); + ObjectMetadata _metadata() { + return ObjectMetadata( + contentType: _contentType, + cacheControl: 'public, max-age=${_maxAge.inSeconds}', + contentDisposition: 'attachment; filename="$_filename"', + custom: { + 'updated': clock.now().toIso8601String(), + }, + ); + } /// Write binary blob to this file. Future write(List data) async { + final metadata = _metadata(); await Future.wait(_owner._prefixes.map((prefix) async { await _owner._pool.withResource(() async { await _owner._bucket.writeBytesIfDifferent( prefix + _objectName, data, - metadata: _metadata, + metadata, ); }); })); } - /// Copy binary blob from [absoluteObjectName] to this file. - /// - /// Notice that [absoluteObjectName] must be an a GCS URI including `gs://`. - /// This means that it must include bucket name. - /// Such URIs can be created with [Bucket.absoluteObjectName]. - Future copyFrom(String absoluteObjectName) async { + /// Copy binary blob from [bucket] and [source] to this file. + Future copyFrom(Bucket bucket, String source) async { + final metadata = _metadata(); + Future? srcInfo; + await Future.wait(_owner._prefixes.map((prefix) async { await _owner._pool.withResource(() async { + final dst = prefix + _objectName; + + // Check if the dst already exists + if (await _owner._bucket.tryInfo(dst) case final dstInfo?) { + // Fetch info for source object (if we haven't already done this) + srcInfo ??= bucket.tryInfo(source); + if (await srcInfo case final srcInfo?) { + if (dstInfo.contentEquals(srcInfo)) { + // If both source and dst exists, and their content matches, then + // we only need to update the "updated" metadata. And we only need + // to update the "updated" timestamp if it's older than + // _retouchDeadline + final retouchDeadline = clock.agoBy(_retouchDeadline); + if (dstInfo.metadata.updated.isBefore(retouchDeadline)) { + await _owner._bucket.updateMetadata(dst, metadata); + } + return; + } + } + } + + // If dst or source doesn't exist, then we shall attempt to make a copy. + // (if source doesn't exist we'll consistently get an error from here!) await _owner._storage.copyObject( - absoluteObjectName, - _owner._bucket.absoluteObjectName(prefix + _objectName), - metadata: _metadata, + bucket.absoluteObjectName(source), + _owner._bucket.absoluteObjectName(dst), + metadata: metadata, ); }); })); } } +const _retouchDeadline = Duration(days: 1); + extension on Bucket { Future writeBytesIfDifferent( String name, - List bytes, { - ObjectMetadata? metadata, - }) async { - if (await _hasSameContent(name, bytes)) { - return; + List bytes, + ObjectMetadata metadata, + ) async { + if (await tryInfo(name) case final info?) { + if (info.isSameContent(bytes)) { + if (info.metadata.updated.isBefore(clock.agoBy(_retouchDeadline))) { + await updateMetadata(name, metadata); + } + return; + } } + await uploadWithRetry( this, name, @@ -353,16 +392,27 @@ extension on Bucket { metadata: metadata, ); } +} - Future _hasSameContent(String name, List bytes) async { - final info = await tryInfo(name); - if (info == null) { +extension on ObjectInfo { + bool isSameContent(List bytes) { + if (length != bytes.length) { return false; } - if (info.length != bytes.length) { + final bytesHash = md5.convert(bytes).bytes; + return fixedTimeIntListEquals(md5Hash, bytesHash); + } + + bool contentEquals(ObjectInfo info) { + if (length != info.length) { return false; } - final md5Hash = md5.convert(bytes).bytes; - return fixedTimeIntListEquals(info.md5Hash, md5Hash); + return fixedTimeIntListEquals(md5Hash, info.md5Hash); + } +} + +extension on ObjectMetadata { + DateTime get updated { + return DateTime.tryParse(custom?['updated'] ?? '') ?? DateTime(0); } } From 7cf9bdc2b1189fcade4812efe9691aee5da3fca4 Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Mon, 28 Oct 2024 13:15:51 +0100 Subject: [PATCH 2/3] Use same pattern for metadata --- app/lib/package/api_export/exported_api.dart | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/lib/package/api_export/exported_api.dart b/app/lib/package/api_export/exported_api.dart index cb62a2977d..d1370af1a1 100644 --- a/app/lib/package/api_export/exported_api.dart +++ b/app/lib/package/api_export/exported_api.dart @@ -259,10 +259,8 @@ final class ExportedJsonFile extends ExportedObject { this._maxAge, ) : super._(_owner, _objectName); - /// Write [data] as gzipped JSON in UTF-8 format. - Future write(T data) async { - final gzipped = _jsonGzip.encode(data); - final metadata = ObjectMetadata( + ObjectMetadata _metadata() { + return ObjectMetadata( contentType: 'application/json; charset="utf-8"', contentEncoding: 'gzip', cacheControl: 'public, max-age=${_maxAge.inSeconds}', @@ -270,6 +268,12 @@ final class ExportedJsonFile extends ExportedObject { 'updated': clock.now().toIso8601String(), }, ); + } + + /// Write [data] as gzipped JSON in UTF-8 format. + Future write(T data) async { + final gzipped = _jsonGzip.encode(data); + final metadata = _metadata(); await Future.wait(_owner._prefixes.map((prefix) async { await _owner._pool.withResource(() async { From 559396691a61621d00ba42b1e85f01e9307991c0 Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Mon, 28 Oct 2024 13:23:35 +0100 Subject: [PATCH 3/3] updated -> validated --- app/lib/package/api_export/exported_api.dart | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/app/lib/package/api_export/exported_api.dart b/app/lib/package/api_export/exported_api.dart index d1370af1a1..0aa8d202c5 100644 --- a/app/lib/package/api_export/exported_api.dart +++ b/app/lib/package/api_export/exported_api.dart @@ -265,7 +265,7 @@ final class ExportedJsonFile extends ExportedObject { contentEncoding: 'gzip', cacheControl: 'public, max-age=${_maxAge.inSeconds}', custom: { - 'updated': clock.now().toIso8601String(), + 'validated': clock.now().toIso8601String(), }, ); } @@ -312,7 +312,7 @@ final class ExportedBlob extends ExportedObject { cacheControl: 'public, max-age=${_maxAge.inSeconds}', contentDisposition: 'attachment; filename="$_filename"', custom: { - 'updated': clock.now().toIso8601String(), + 'validated': clock.now().toIso8601String(), }, ); } @@ -347,11 +347,11 @@ final class ExportedBlob extends ExportedObject { if (await srcInfo case final srcInfo?) { if (dstInfo.contentEquals(srcInfo)) { // If both source and dst exists, and their content matches, then - // we only need to update the "updated" metadata. And we only need - // to update the "updated" timestamp if it's older than + // we only need to update the "validated" metadata. And we only + // need to update the "validated" timestamp if it's older than // _retouchDeadline - final retouchDeadline = clock.agoBy(_retouchDeadline); - if (dstInfo.metadata.updated.isBefore(retouchDeadline)) { + final retouchDeadline = clock.agoBy(_revalidateAfter); + if (dstInfo.metadata.validated.isBefore(retouchDeadline)) { await _owner._bucket.updateMetadata(dst, metadata); } return; @@ -371,7 +371,7 @@ final class ExportedBlob extends ExportedObject { } } -const _retouchDeadline = Duration(days: 1); +const _revalidateAfter = Duration(days: 1); extension on Bucket { Future writeBytesIfDifferent( @@ -381,7 +381,7 @@ extension on Bucket { ) async { if (await tryInfo(name) case final info?) { if (info.isSameContent(bytes)) { - if (info.metadata.updated.isBefore(clock.agoBy(_retouchDeadline))) { + if (info.metadata.validated.isBefore(clock.agoBy(_revalidateAfter))) { await updateMetadata(name, metadata); } return; @@ -416,7 +416,7 @@ extension on ObjectInfo { } extension on ObjectMetadata { - DateTime get updated { - return DateTime.tryParse(custom?['updated'] ?? '') ?? DateTime(0); + DateTime get validated { + return DateTime.tryParse(custom?['validated'] ?? '') ?? DateTime(0); } }