From 32e7cfea3d425469a98289f71db2dcfbdb231dc5 Mon Sep 17 00:00:00 2001 From: Fynn Petersen-Frey <10599762+fyfrey@users.noreply.github.com> Date: Mon, 29 Apr 2024 05:24:21 +0200 Subject: [PATCH] fix(server): stacked assets for full sync, userIds as array for delta sync (#9100) * fix(server): stacked assets for full sync, userIds as array for delta sync * refactor(server): sync * fix getDeltaSync after partner removal --------- Co-authored-by: Jason Rasmussen --- mobile/openapi/.openapi-generator/FILES | 6 + mobile/openapi/README.md | 6 +- mobile/openapi/doc/AssetDeltaSyncDto.md | 16 ++ mobile/openapi/doc/AssetFullSyncDto.md | 19 ++ mobile/openapi/doc/SyncApi.md | 46 ++-- mobile/openapi/lib/api.dart | 2 + mobile/openapi/lib/api/sync_api.dart | 89 +++----- mobile/openapi/lib/api_client.dart | 4 + .../lib/model/asset_delta_sync_dto.dart | 108 ++++++++++ .../lib/model/asset_full_sync_dto.dart | 158 ++++++++++++++ .../test/asset_delta_sync_dto_test.dart | 32 +++ .../test/asset_full_sync_dto_test.dart | 47 ++++ mobile/openapi/test/sync_api_test.dart | 8 +- open-api/immich-openapi-specs.json | 139 ++++++------ open-api/typescript-sdk/src/fetch-client.ts | 49 ++--- server/src/controllers/sync.controller.ts | 16 +- server/src/dtos/sync.dto.ts | 3 +- server/src/interfaces/asset.interface.ts | 2 + server/src/queries/asset.repository.sql | 202 ++++++++++++------ server/src/repositories/asset.repository.ts | 35 ++- server/src/services/sync.service.spec.ts | 13 +- server/src/services/sync.service.ts | 57 +++-- 22 files changed, 744 insertions(+), 313 deletions(-) create mode 100644 mobile/openapi/doc/AssetDeltaSyncDto.md create mode 100644 mobile/openapi/doc/AssetFullSyncDto.md create mode 100644 mobile/openapi/lib/model/asset_delta_sync_dto.dart create mode 100644 mobile/openapi/lib/model/asset_full_sync_dto.dart create mode 100644 mobile/openapi/test/asset_delta_sync_dto_test.dart create mode 100644 mobile/openapi/test/asset_full_sync_dto_test.dart diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index e7cbd570dc4fe..4fb6bbdbb1739 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -28,12 +28,14 @@ doc/AssetBulkUploadCheckDto.md doc/AssetBulkUploadCheckItem.md doc/AssetBulkUploadCheckResponseDto.md doc/AssetBulkUploadCheckResult.md +doc/AssetDeltaSyncDto.md doc/AssetDeltaSyncResponseDto.md doc/AssetFaceResponseDto.md doc/AssetFaceUpdateDto.md doc/AssetFaceUpdateItem.md doc/AssetFaceWithoutPersonResponseDto.md doc/AssetFileUploadResponseDto.md +doc/AssetFullSyncDto.md doc/AssetIdsDto.md doc/AssetIdsResponseDto.md doc/AssetJobName.md @@ -265,12 +267,14 @@ lib/model/asset_bulk_upload_check_dto.dart lib/model/asset_bulk_upload_check_item.dart lib/model/asset_bulk_upload_check_response_dto.dart lib/model/asset_bulk_upload_check_result.dart +lib/model/asset_delta_sync_dto.dart lib/model/asset_delta_sync_response_dto.dart lib/model/asset_face_response_dto.dart lib/model/asset_face_update_dto.dart lib/model/asset_face_update_item.dart lib/model/asset_face_without_person_response_dto.dart lib/model/asset_file_upload_response_dto.dart +lib/model/asset_full_sync_dto.dart lib/model/asset_ids_dto.dart lib/model/asset_ids_response_dto.dart lib/model/asset_job_name.dart @@ -449,12 +453,14 @@ test/asset_bulk_upload_check_dto_test.dart test/asset_bulk_upload_check_item_test.dart test/asset_bulk_upload_check_response_dto_test.dart test/asset_bulk_upload_check_result_test.dart +test/asset_delta_sync_dto_test.dart test/asset_delta_sync_response_dto_test.dart test/asset_face_response_dto_test.dart test/asset_face_update_dto_test.dart test/asset_face_update_item_test.dart test/asset_face_without_person_response_dto_test.dart test/asset_file_upload_response_dto_test.dart +test/asset_full_sync_dto_test.dart test/asset_ids_dto_test.dart test/asset_ids_response_dto_test.dart test/asset_job_name_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index a0872d6f97f5c..9b19236e7b6b5 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -186,8 +186,8 @@ Class | Method | HTTP request | Description *SharedLinkApi* | [**removeSharedLink**](doc//SharedLinkApi.md#removesharedlink) | **DELETE** /shared-link/{id} | *SharedLinkApi* | [**removeSharedLinkAssets**](doc//SharedLinkApi.md#removesharedlinkassets) | **DELETE** /shared-link/{id}/assets | *SharedLinkApi* | [**updateSharedLink**](doc//SharedLinkApi.md#updatesharedlink) | **PATCH** /shared-link/{id} | -*SyncApi* | [**getAllForUserFullSync**](doc//SyncApi.md#getallforuserfullsync) | **GET** /sync/full-sync | -*SyncApi* | [**getDeltaSync**](doc//SyncApi.md#getdeltasync) | **GET** /sync/delta-sync | +*SyncApi* | [**getDeltaSync**](doc//SyncApi.md#getdeltasync) | **POST** /sync/delta-sync | +*SyncApi* | [**getFullSyncForUser**](doc//SyncApi.md#getfullsyncforuser) | **POST** /sync/full-sync | *SystemConfigApi* | [**getConfig**](doc//SystemConfigApi.md#getconfig) | **GET** /system-config | *SystemConfigApi* | [**getConfigDefaults**](doc//SystemConfigApi.md#getconfigdefaults) | **GET** /system-config/defaults | *SystemConfigApi* | [**getMapStyle**](doc//SystemConfigApi.md#getmapstyle) | **GET** /system-config/map/style.json | @@ -244,12 +244,14 @@ Class | Method | HTTP request | Description - [AssetBulkUploadCheckItem](doc//AssetBulkUploadCheckItem.md) - [AssetBulkUploadCheckResponseDto](doc//AssetBulkUploadCheckResponseDto.md) - [AssetBulkUploadCheckResult](doc//AssetBulkUploadCheckResult.md) + - [AssetDeltaSyncDto](doc//AssetDeltaSyncDto.md) - [AssetDeltaSyncResponseDto](doc//AssetDeltaSyncResponseDto.md) - [AssetFaceResponseDto](doc//AssetFaceResponseDto.md) - [AssetFaceUpdateDto](doc//AssetFaceUpdateDto.md) - [AssetFaceUpdateItem](doc//AssetFaceUpdateItem.md) - [AssetFaceWithoutPersonResponseDto](doc//AssetFaceWithoutPersonResponseDto.md) - [AssetFileUploadResponseDto](doc//AssetFileUploadResponseDto.md) + - [AssetFullSyncDto](doc//AssetFullSyncDto.md) - [AssetIdsDto](doc//AssetIdsDto.md) - [AssetIdsResponseDto](doc//AssetIdsResponseDto.md) - [AssetJobName](doc//AssetJobName.md) diff --git a/mobile/openapi/doc/AssetDeltaSyncDto.md b/mobile/openapi/doc/AssetDeltaSyncDto.md new file mode 100644 index 0000000000000..b38329fe966a6 --- /dev/null +++ b/mobile/openapi/doc/AssetDeltaSyncDto.md @@ -0,0 +1,16 @@ +# openapi.model.AssetDeltaSyncDto + +## Load the model package +```dart +import 'package:openapi/api.dart'; +``` + +## Properties +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**updatedAfter** | [**DateTime**](DateTime.md) | | +**userIds** | **List** | | [default to const []] + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/mobile/openapi/doc/AssetFullSyncDto.md b/mobile/openapi/doc/AssetFullSyncDto.md new file mode 100644 index 0000000000000..8635fee22215c --- /dev/null +++ b/mobile/openapi/doc/AssetFullSyncDto.md @@ -0,0 +1,19 @@ +# openapi.model.AssetFullSyncDto + +## Load the model package +```dart +import 'package:openapi/api.dart'; +``` + +## Properties +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**lastCreationDate** | [**DateTime**](DateTime.md) | | [optional] +**lastId** | **String** | | [optional] +**limit** | **int** | | +**updatedUntil** | [**DateTime**](DateTime.md) | | +**userId** | **String** | | [optional] + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/mobile/openapi/doc/SyncApi.md b/mobile/openapi/doc/SyncApi.md index 1b28e10c8cc15..f750f7d4ba3dc 100644 --- a/mobile/openapi/doc/SyncApi.md +++ b/mobile/openapi/doc/SyncApi.md @@ -9,12 +9,12 @@ All URIs are relative to */api* Method | HTTP request | Description ------------- | ------------- | ------------- -[**getAllForUserFullSync**](SyncApi.md#getallforuserfullsync) | **GET** /sync/full-sync | -[**getDeltaSync**](SyncApi.md#getdeltasync) | **GET** /sync/delta-sync | +[**getDeltaSync**](SyncApi.md#getdeltasync) | **POST** /sync/delta-sync | +[**getFullSyncForUser**](SyncApi.md#getfullsyncforuser) | **POST** /sync/full-sync | -# **getAllForUserFullSync** -> List getAllForUserFullSync(limit, updatedUntil, lastCreationDate, lastId, userId) +# **getDeltaSync** +> AssetDeltaSyncResponseDto getDeltaSync(assetDeltaSyncDto) @@ -37,17 +37,13 @@ import 'package:openapi/api.dart'; //defaultApiClient.getAuthentication('bearer').setAccessToken(yourTokenGeneratorFunction); final api_instance = SyncApi(); -final limit = 56; // int | -final updatedUntil = 2013-10-20T19:20:30+01:00; // DateTime | -final lastCreationDate = 2013-10-20T19:20:30+01:00; // DateTime | -final lastId = 38400000-8cf0-11bd-b23e-10b96e4ef00d; // String | -final userId = 38400000-8cf0-11bd-b23e-10b96e4ef00d; // String | +final assetDeltaSyncDto = AssetDeltaSyncDto(); // AssetDeltaSyncDto | try { - final result = api_instance.getAllForUserFullSync(limit, updatedUntil, lastCreationDate, lastId, userId); + final result = api_instance.getDeltaSync(assetDeltaSyncDto); print(result); } catch (e) { - print('Exception when calling SyncApi->getAllForUserFullSync: $e\n'); + print('Exception when calling SyncApi->getDeltaSync: $e\n'); } ``` @@ -55,15 +51,11 @@ try { Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- - **limit** | **int**| | - **updatedUntil** | **DateTime**| | - **lastCreationDate** | **DateTime**| | [optional] - **lastId** | **String**| | [optional] - **userId** | **String**| | [optional] + **assetDeltaSyncDto** | [**AssetDeltaSyncDto**](AssetDeltaSyncDto.md)| | ### Return type -[**List**](AssetResponseDto.md) +[**AssetDeltaSyncResponseDto**](AssetDeltaSyncResponseDto.md) ### Authorization @@ -71,13 +63,13 @@ Name | Type | Description | Notes ### HTTP request headers - - **Content-Type**: Not defined + - **Content-Type**: application/json - **Accept**: application/json [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) -# **getDeltaSync** -> AssetDeltaSyncResponseDto getDeltaSync(updatedAfter, userIds) +# **getFullSyncForUser** +> List getFullSyncForUser(assetFullSyncDto) @@ -100,14 +92,13 @@ import 'package:openapi/api.dart'; //defaultApiClient.getAuthentication('bearer').setAccessToken(yourTokenGeneratorFunction); final api_instance = SyncApi(); -final updatedAfter = 2013-10-20T19:20:30+01:00; // DateTime | -final userIds = []; // List | +final assetFullSyncDto = AssetFullSyncDto(); // AssetFullSyncDto | try { - final result = api_instance.getDeltaSync(updatedAfter, userIds); + final result = api_instance.getFullSyncForUser(assetFullSyncDto); print(result); } catch (e) { - print('Exception when calling SyncApi->getDeltaSync: $e\n'); + print('Exception when calling SyncApi->getFullSyncForUser: $e\n'); } ``` @@ -115,12 +106,11 @@ try { Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- - **updatedAfter** | **DateTime**| | - **userIds** | [**List**](String.md)| | [default to const []] + **assetFullSyncDto** | [**AssetFullSyncDto**](AssetFullSyncDto.md)| | ### Return type -[**AssetDeltaSyncResponseDto**](AssetDeltaSyncResponseDto.md) +[**List**](AssetResponseDto.md) ### Authorization @@ -128,7 +118,7 @@ Name | Type | Description | Notes ### HTTP request headers - - **Content-Type**: Not defined + - **Content-Type**: application/json - **Accept**: application/json [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 3e2f23024e889..6752397884eda 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -77,12 +77,14 @@ part 'model/asset_bulk_upload_check_dto.dart'; part 'model/asset_bulk_upload_check_item.dart'; part 'model/asset_bulk_upload_check_response_dto.dart'; part 'model/asset_bulk_upload_check_result.dart'; +part 'model/asset_delta_sync_dto.dart'; part 'model/asset_delta_sync_response_dto.dart'; part 'model/asset_face_response_dto.dart'; part 'model/asset_face_update_dto.dart'; part 'model/asset_face_update_item.dart'; part 'model/asset_face_without_person_response_dto.dart'; part 'model/asset_file_upload_response_dto.dart'; +part 'model/asset_full_sync_dto.dart'; part 'model/asset_ids_dto.dart'; part 'model/asset_ids_response_dto.dart'; part 'model/asset_job_name.dart'; diff --git a/mobile/openapi/lib/api/sync_api.dart b/mobile/openapi/lib/api/sync_api.dart index fdfd8b9ac7b02..f131d54e9d5f6 100644 --- a/mobile/openapi/lib/api/sync_api.dart +++ b/mobile/openapi/lib/api/sync_api.dart @@ -16,47 +16,27 @@ class SyncApi { final ApiClient apiClient; - /// Performs an HTTP 'GET /sync/full-sync' operation and returns the [Response]. + /// Performs an HTTP 'POST /sync/delta-sync' operation and returns the [Response]. /// Parameters: /// - /// * [int] limit (required): - /// - /// * [DateTime] updatedUntil (required): - /// - /// * [DateTime] lastCreationDate: - /// - /// * [String] lastId: - /// - /// * [String] userId: - Future getAllForUserFullSyncWithHttpInfo(int limit, DateTime updatedUntil, { DateTime? lastCreationDate, String? lastId, String? userId, }) async { + /// * [AssetDeltaSyncDto] assetDeltaSyncDto (required): + Future getDeltaSyncWithHttpInfo(AssetDeltaSyncDto assetDeltaSyncDto,) async { // ignore: prefer_const_declarations - final path = r'/sync/full-sync'; + final path = r'/sync/delta-sync'; // ignore: prefer_final_locals - Object? postBody; + Object? postBody = assetDeltaSyncDto; final queryParams = []; final headerParams = {}; final formParams = {}; - if (lastCreationDate != null) { - queryParams.addAll(_queryParams('', 'lastCreationDate', lastCreationDate)); - } - if (lastId != null) { - queryParams.addAll(_queryParams('', 'lastId', lastId)); - } - queryParams.addAll(_queryParams('', 'limit', limit)); - queryParams.addAll(_queryParams('', 'updatedUntil', updatedUntil)); - if (userId != null) { - queryParams.addAll(_queryParams('', 'userId', userId)); - } - - const contentTypes = []; + const contentTypes = ['application/json']; return apiClient.invokeAPI( path, - 'GET', + 'POST', queryParams, postBody, headerParams, @@ -67,17 +47,9 @@ class SyncApi { /// Parameters: /// - /// * [int] limit (required): - /// - /// * [DateTime] updatedUntil (required): - /// - /// * [DateTime] lastCreationDate: - /// - /// * [String] lastId: - /// - /// * [String] userId: - Future?> getAllForUserFullSync(int limit, DateTime updatedUntil, { DateTime? lastCreationDate, String? lastId, String? userId, }) async { - final response = await getAllForUserFullSyncWithHttpInfo(limit, updatedUntil, lastCreationDate: lastCreationDate, lastId: lastId, userId: userId, ); + /// * [AssetDeltaSyncDto] assetDeltaSyncDto (required): + Future getDeltaSync(AssetDeltaSyncDto assetDeltaSyncDto,) async { + final response = await getDeltaSyncWithHttpInfo(assetDeltaSyncDto,); if (response.statusCode >= HttpStatus.badRequest) { throw ApiException(response.statusCode, await _decodeBodyBytes(response)); } @@ -85,41 +57,33 @@ class SyncApi { // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" // FormatException when trying to decode an empty string. if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { - final responseBody = await _decodeBodyBytes(response); - return (await apiClient.deserializeAsync(responseBody, 'List') as List) - .cast() - .toList(growable: false); - + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'AssetDeltaSyncResponseDto',) as AssetDeltaSyncResponseDto; + } return null; } - /// Performs an HTTP 'GET /sync/delta-sync' operation and returns the [Response]. + /// Performs an HTTP 'POST /sync/full-sync' operation and returns the [Response]. /// Parameters: /// - /// * [DateTime] updatedAfter (required): - /// - /// * [List] userIds (required): - Future getDeltaSyncWithHttpInfo(DateTime updatedAfter, List userIds,) async { + /// * [AssetFullSyncDto] assetFullSyncDto (required): + Future getFullSyncForUserWithHttpInfo(AssetFullSyncDto assetFullSyncDto,) async { // ignore: prefer_const_declarations - final path = r'/sync/delta-sync'; + final path = r'/sync/full-sync'; // ignore: prefer_final_locals - Object? postBody; + Object? postBody = assetFullSyncDto; final queryParams = []; final headerParams = {}; final formParams = {}; - queryParams.addAll(_queryParams('', 'updatedAfter', updatedAfter)); - queryParams.addAll(_queryParams('multi', 'userIds', userIds)); - - const contentTypes = []; + const contentTypes = ['application/json']; return apiClient.invokeAPI( path, - 'GET', + 'POST', queryParams, postBody, headerParams, @@ -130,11 +94,9 @@ class SyncApi { /// Parameters: /// - /// * [DateTime] updatedAfter (required): - /// - /// * [List] userIds (required): - Future getDeltaSync(DateTime updatedAfter, List userIds,) async { - final response = await getDeltaSyncWithHttpInfo(updatedAfter, userIds,); + /// * [AssetFullSyncDto] assetFullSyncDto (required): + Future?> getFullSyncForUser(AssetFullSyncDto assetFullSyncDto,) async { + final response = await getFullSyncForUserWithHttpInfo(assetFullSyncDto,); if (response.statusCode >= HttpStatus.badRequest) { throw ApiException(response.statusCode, await _decodeBodyBytes(response)); } @@ -142,8 +104,11 @@ class SyncApi { // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" // FormatException when trying to decode an empty string. if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { - return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'AssetDeltaSyncResponseDto',) as AssetDeltaSyncResponseDto; - + final responseBody = await _decodeBodyBytes(response); + return (await apiClient.deserializeAsync(responseBody, 'List') as List) + .cast() + .toList(growable: false); + } return null; } diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 5db7b8fbef52d..1026a1f52097c 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -224,6 +224,8 @@ class ApiClient { return AssetBulkUploadCheckResponseDto.fromJson(value); case 'AssetBulkUploadCheckResult': return AssetBulkUploadCheckResult.fromJson(value); + case 'AssetDeltaSyncDto': + return AssetDeltaSyncDto.fromJson(value); case 'AssetDeltaSyncResponseDto': return AssetDeltaSyncResponseDto.fromJson(value); case 'AssetFaceResponseDto': @@ -236,6 +238,8 @@ class ApiClient { return AssetFaceWithoutPersonResponseDto.fromJson(value); case 'AssetFileUploadResponseDto': return AssetFileUploadResponseDto.fromJson(value); + case 'AssetFullSyncDto': + return AssetFullSyncDto.fromJson(value); case 'AssetIdsDto': return AssetIdsDto.fromJson(value); case 'AssetIdsResponseDto': diff --git a/mobile/openapi/lib/model/asset_delta_sync_dto.dart b/mobile/openapi/lib/model/asset_delta_sync_dto.dart new file mode 100644 index 0000000000000..c7f3ce618a741 --- /dev/null +++ b/mobile/openapi/lib/model/asset_delta_sync_dto.dart @@ -0,0 +1,108 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class AssetDeltaSyncDto { + /// Returns a new [AssetDeltaSyncDto] instance. + AssetDeltaSyncDto({ + required this.updatedAfter, + this.userIds = const [], + }); + + DateTime updatedAfter; + + List userIds; + + @override + bool operator ==(Object other) => identical(this, other) || other is AssetDeltaSyncDto && + other.updatedAfter == updatedAfter && + _deepEquality.equals(other.userIds, userIds); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (updatedAfter.hashCode) + + (userIds.hashCode); + + @override + String toString() => 'AssetDeltaSyncDto[updatedAfter=$updatedAfter, userIds=$userIds]'; + + Map toJson() { + final json = {}; + json[r'updatedAfter'] = this.updatedAfter.toUtc().toIso8601String(); + json[r'userIds'] = this.userIds; + return json; + } + + /// Returns a new [AssetDeltaSyncDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static AssetDeltaSyncDto? fromJson(dynamic value) { + if (value is Map) { + final json = value.cast(); + + return AssetDeltaSyncDto( + updatedAfter: mapDateTime(json, r'updatedAfter', r'')!, + userIds: json[r'userIds'] is Iterable + ? (json[r'userIds'] as Iterable).cast().toList(growable: false) + : const [], + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = AssetDeltaSyncDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = AssetDeltaSyncDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of AssetDeltaSyncDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = AssetDeltaSyncDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'updatedAfter', + 'userIds', + }; +} + diff --git a/mobile/openapi/lib/model/asset_full_sync_dto.dart b/mobile/openapi/lib/model/asset_full_sync_dto.dart new file mode 100644 index 0000000000000..fba8d65381a17 --- /dev/null +++ b/mobile/openapi/lib/model/asset_full_sync_dto.dart @@ -0,0 +1,158 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class AssetFullSyncDto { + /// Returns a new [AssetFullSyncDto] instance. + AssetFullSyncDto({ + this.lastCreationDate, + this.lastId, + required this.limit, + required this.updatedUntil, + this.userId, + }); + + /// + /// Please note: This property should have been non-nullable! Since the specification file + /// does not include a default value (using the "default:" property), however, the generated + /// source code must fall back to having a nullable type. + /// Consider adding a "default:" property in the specification file to hide this note. + /// + DateTime? lastCreationDate; + + /// + /// Please note: This property should have been non-nullable! Since the specification file + /// does not include a default value (using the "default:" property), however, the generated + /// source code must fall back to having a nullable type. + /// Consider adding a "default:" property in the specification file to hide this note. + /// + String? lastId; + + /// Minimum value: 1 + int limit; + + DateTime updatedUntil; + + /// + /// Please note: This property should have been non-nullable! Since the specification file + /// does not include a default value (using the "default:" property), however, the generated + /// source code must fall back to having a nullable type. + /// Consider adding a "default:" property in the specification file to hide this note. + /// + String? userId; + + @override + bool operator ==(Object other) => identical(this, other) || other is AssetFullSyncDto && + other.lastCreationDate == lastCreationDate && + other.lastId == lastId && + other.limit == limit && + other.updatedUntil == updatedUntil && + other.userId == userId; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (lastCreationDate == null ? 0 : lastCreationDate!.hashCode) + + (lastId == null ? 0 : lastId!.hashCode) + + (limit.hashCode) + + (updatedUntil.hashCode) + + (userId == null ? 0 : userId!.hashCode); + + @override + String toString() => 'AssetFullSyncDto[lastCreationDate=$lastCreationDate, lastId=$lastId, limit=$limit, updatedUntil=$updatedUntil, userId=$userId]'; + + Map toJson() { + final json = {}; + if (this.lastCreationDate != null) { + json[r'lastCreationDate'] = this.lastCreationDate!.toUtc().toIso8601String(); + } else { + // json[r'lastCreationDate'] = null; + } + if (this.lastId != null) { + json[r'lastId'] = this.lastId; + } else { + // json[r'lastId'] = null; + } + json[r'limit'] = this.limit; + json[r'updatedUntil'] = this.updatedUntil.toUtc().toIso8601String(); + if (this.userId != null) { + json[r'userId'] = this.userId; + } else { + // json[r'userId'] = null; + } + return json; + } + + /// Returns a new [AssetFullSyncDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static AssetFullSyncDto? fromJson(dynamic value) { + if (value is Map) { + final json = value.cast(); + + return AssetFullSyncDto( + lastCreationDate: mapDateTime(json, r'lastCreationDate', r''), + lastId: mapValueOfType(json, r'lastId'), + limit: mapValueOfType(json, r'limit')!, + updatedUntil: mapDateTime(json, r'updatedUntil', r'')!, + userId: mapValueOfType(json, r'userId'), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = AssetFullSyncDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = AssetFullSyncDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of AssetFullSyncDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = AssetFullSyncDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'limit', + 'updatedUntil', + }; +} + diff --git a/mobile/openapi/test/asset_delta_sync_dto_test.dart b/mobile/openapi/test/asset_delta_sync_dto_test.dart new file mode 100644 index 0000000000000..41676d610b68c --- /dev/null +++ b/mobile/openapi/test/asset_delta_sync_dto_test.dart @@ -0,0 +1,32 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +import 'package:openapi/api.dart'; +import 'package:test/test.dart'; + +// tests for AssetDeltaSyncDto +void main() { + // final instance = AssetDeltaSyncDto(); + + group('test AssetDeltaSyncDto', () { + // DateTime updatedAfter + test('to test the property `updatedAfter`', () async { + // TODO + }); + + // List userIds (default value: const []) + test('to test the property `userIds`', () async { + // TODO + }); + + + }); + +} diff --git a/mobile/openapi/test/asset_full_sync_dto_test.dart b/mobile/openapi/test/asset_full_sync_dto_test.dart new file mode 100644 index 0000000000000..cf838ae89ee47 --- /dev/null +++ b/mobile/openapi/test/asset_full_sync_dto_test.dart @@ -0,0 +1,47 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +import 'package:openapi/api.dart'; +import 'package:test/test.dart'; + +// tests for AssetFullSyncDto +void main() { + // final instance = AssetFullSyncDto(); + + group('test AssetFullSyncDto', () { + // DateTime lastCreationDate + test('to test the property `lastCreationDate`', () async { + // TODO + }); + + // String lastId + test('to test the property `lastId`', () async { + // TODO + }); + + // int limit + test('to test the property `limit`', () async { + // TODO + }); + + // DateTime updatedUntil + test('to test the property `updatedUntil`', () async { + // TODO + }); + + // String userId + test('to test the property `userId`', () async { + // TODO + }); + + + }); + +} diff --git a/mobile/openapi/test/sync_api_test.dart b/mobile/openapi/test/sync_api_test.dart index ad9ef0f92f8fe..c2f548aeb2b30 100644 --- a/mobile/openapi/test/sync_api_test.dart +++ b/mobile/openapi/test/sync_api_test.dart @@ -17,13 +17,13 @@ void main() { // final instance = SyncApi(); group('tests for SyncApi', () { - //Future> getAllForUserFullSync(int limit, DateTime updatedUntil, { DateTime lastCreationDate, String lastId, String userId }) async - test('test getAllForUserFullSync', () async { + //Future getDeltaSync(AssetDeltaSyncDto assetDeltaSyncDto) async + test('test getDeltaSync', () async { // TODO }); - //Future getDeltaSync(DateTime updatedAfter, List userIds) async - test('test getDeltaSync', () async { + //Future> getFullSyncForUser(AssetFullSyncDto assetFullSyncDto) async + test('test getFullSyncForUser', () async { // TODO }); diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index e831c6f3e7924..ec859d56e21b5 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -4958,31 +4958,19 @@ } }, "/sync/delta-sync": { - "get": { + "post": { "operationId": "getDeltaSync", - "parameters": [ - { - "name": "updatedAfter", - "required": true, - "in": "query", - "schema": { - "format": "date-time", - "type": "string" - } - }, - { - "name": "userIds", - "required": true, - "in": "query", - "schema": { - "format": "uuid", - "type": "array", - "items": { - "type": "string" + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssetDeltaSyncDto" } } - } - ], + }, + "required": true + }, "responses": { "200": { "content": { @@ -5012,55 +5000,19 @@ } }, "/sync/full-sync": { - "get": { - "operationId": "getAllForUserFullSync", - "parameters": [ - { - "name": "lastCreationDate", - "required": false, - "in": "query", - "schema": { - "format": "date-time", - "type": "string" - } - }, - { - "name": "lastId", - "required": false, - "in": "query", - "schema": { - "format": "uuid", - "type": "string" - } - }, - { - "name": "limit", - "required": true, - "in": "query", - "schema": { - "minimum": 1, - "type": "integer" - } - }, - { - "name": "updatedUntil", - "required": true, - "in": "query", - "schema": { - "format": "date-time", - "type": "string" + "post": { + "operationId": "getFullSyncForUser", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssetFullSyncDto" + } } }, - { - "name": "userId", - "required": false, - "in": "query", - "schema": { - "format": "uuid", - "type": "string" - } - } - ], + "required": true + }, "responses": { "200": { "content": { @@ -7023,6 +6975,26 @@ ], "type": "object" }, + "AssetDeltaSyncDto": { + "properties": { + "updatedAfter": { + "format": "date-time", + "type": "string" + }, + "userIds": { + "items": { + "format": "uuid", + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "updatedAfter", + "userIds" + ], + "type": "object" + }, "AssetDeltaSyncResponseDto": { "properties": { "deleted": { @@ -7175,6 +7147,35 @@ ], "type": "object" }, + "AssetFullSyncDto": { + "properties": { + "lastCreationDate": { + "format": "date-time", + "type": "string" + }, + "lastId": { + "format": "uuid", + "type": "string" + }, + "limit": { + "minimum": 1, + "type": "integer" + }, + "updatedUntil": { + "format": "date-time", + "type": "string" + }, + "userId": { + "format": "uuid", + "type": "string" + } + }, + "required": [ + "limit", + "updatedUntil" + ], + "type": "object" + }, "AssetIdsDto": { "properties": { "assetIds": { diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 378f77c54a297..92fc5cd59c1f8 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -836,11 +836,22 @@ export type AssetIdsResponseDto = { error?: Error2; success: boolean; }; +export type AssetDeltaSyncDto = { + updatedAfter: string; + userIds: string[]; +}; export type AssetDeltaSyncResponseDto = { deleted: string[]; needsFullSync: boolean; upserted: AssetResponseDto[]; }; +export type AssetFullSyncDto = { + lastCreationDate?: string; + lastId?: string; + limit: number; + updatedUntil: string; + userId?: string; +}; export type SystemConfigFFmpegDto = { accel: TranscodeHWAccel; acceptedAudioCodecs: AudioCodec[]; @@ -2372,39 +2383,29 @@ export function addSharedLinkAssets({ id, key, assetIdsDto }: { body: assetIdsDto }))); } -export function getDeltaSync({ updatedAfter, userIds }: { - updatedAfter: string; - userIds: string[]; +export function getDeltaSync({ assetDeltaSyncDto }: { + assetDeltaSyncDto: AssetDeltaSyncDto; }, opts?: Oazapfts.RequestOpts) { return oazapfts.ok(oazapfts.fetchJson<{ status: 200; data: AssetDeltaSyncResponseDto; - }>(`/sync/delta-sync${QS.query(QS.explode({ - updatedAfter, - userIds - }))}`, { - ...opts - })); + }>("/sync/delta-sync", oazapfts.json({ + ...opts, + method: "POST", + body: assetDeltaSyncDto + }))); } -export function getAllForUserFullSync({ lastCreationDate, lastId, limit, updatedUntil, userId }: { - lastCreationDate?: string; - lastId?: string; - limit: number; - updatedUntil: string; - userId?: string; +export function getFullSyncForUser({ assetFullSyncDto }: { + assetFullSyncDto: AssetFullSyncDto; }, opts?: Oazapfts.RequestOpts) { return oazapfts.ok(oazapfts.fetchJson<{ status: 200; data: AssetResponseDto[]; - }>(`/sync/full-sync${QS.query(QS.explode({ - lastCreationDate, - lastId, - limit, - updatedUntil, - userId - }))}`, { - ...opts - })); + }>("/sync/full-sync", oazapfts.json({ + ...opts, + method: "POST", + body: assetFullSyncDto + }))); } export function getConfig(opts?: Oazapfts.RequestOpts) { return oazapfts.ok(oazapfts.fetchJson<{ diff --git a/server/src/controllers/sync.controller.ts b/server/src/controllers/sync.controller.ts index c12d42df23ccc..63757f73f3e60 100644 --- a/server/src/controllers/sync.controller.ts +++ b/server/src/controllers/sync.controller.ts @@ -1,4 +1,4 @@ -import { Controller, Get, Query } from '@nestjs/common'; +import { Body, Controller, HttpCode, HttpStatus, Post } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; import { AssetResponseDto } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; @@ -12,13 +12,15 @@ import { SyncService } from 'src/services/sync.service'; export class SyncController { constructor(private service: SyncService) {} - @Get('full-sync') - getAllForUserFullSync(@Auth() auth: AuthDto, @Query() dto: AssetFullSyncDto): Promise { - return this.service.getAllAssetsForUserFullSync(auth, dto); + @Post('full-sync') + @HttpCode(HttpStatus.OK) + getFullSyncForUser(@Auth() auth: AuthDto, @Body() dto: AssetFullSyncDto): Promise { + return this.service.getFullSync(auth, dto); } - @Get('delta-sync') - getDeltaSync(@Auth() auth: AuthDto, @Query() dto: AssetDeltaSyncDto): Promise { - return this.service.getChangesForDeltaSync(auth, dto); + @Post('delta-sync') + @HttpCode(HttpStatus.OK) + getDeltaSync(@Auth() auth: AuthDto, @Body() dto: AssetDeltaSyncDto): Promise { + return this.service.getDeltaSync(auth, dto); } } diff --git a/server/src/dtos/sync.dto.ts b/server/src/dtos/sync.dto.ts index a69062ec2d83b..1a02ba5ca080b 100644 --- a/server/src/dtos/sync.dto.ts +++ b/server/src/dtos/sync.dto.ts @@ -1,5 +1,4 @@ import { ApiProperty } from '@nestjs/swagger'; -import { Type } from 'class-transformer'; import { IsInt, IsPositive } from 'class-validator'; import { AssetResponseDto } from 'src/dtos/asset-response.dto'; import { ValidateDate, ValidateUUID } from 'src/validation'; @@ -16,7 +15,6 @@ export class AssetFullSyncDto { @IsInt() @IsPositive() - @Type(() => Number) @ApiProperty({ type: 'integer' }) limit!: number; @@ -27,6 +25,7 @@ export class AssetFullSyncDto { export class AssetDeltaSyncDto { @ValidateDate() updatedAfter!: Date; + @ValidateUUID({ each: true }) userIds!: string[]; } diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts index fb6345df7ca80..cad83f09d4024 100644 --- a/server/src/interfaces/asset.interface.ts +++ b/server/src/interfaces/asset.interface.ts @@ -134,6 +134,8 @@ export interface AssetFullSyncOptions { lastCreationDate?: Date; lastId?: string; updatedUntil: Date; + isArchived?: false; + withStacked?: true; limit: number; } diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index 81dce80d0f7e3..7d49fb18df6df 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -798,16 +798,47 @@ SELECT "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample", "exifInfo"."fps" AS "exifInfo_fps", "stack"."id" AS "stack_id", - "stack"."primaryAssetId" AS "stack_primaryAssetId" + "stack"."primaryAssetId" AS "stack_primaryAssetId", + "stackedAssets"."id" AS "stackedAssets_id", + "stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId", + "stackedAssets"."ownerId" AS "stackedAssets_ownerId", + "stackedAssets"."libraryId" AS "stackedAssets_libraryId", + "stackedAssets"."deviceId" AS "stackedAssets_deviceId", + "stackedAssets"."type" AS "stackedAssets_type", + "stackedAssets"."originalPath" AS "stackedAssets_originalPath", + "stackedAssets"."previewPath" AS "stackedAssets_previewPath", + "stackedAssets"."thumbnailPath" AS "stackedAssets_thumbnailPath", + "stackedAssets"."thumbhash" AS "stackedAssets_thumbhash", + "stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath", + "stackedAssets"."createdAt" AS "stackedAssets_createdAt", + "stackedAssets"."updatedAt" AS "stackedAssets_updatedAt", + "stackedAssets"."deletedAt" AS "stackedAssets_deletedAt", + "stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt", + "stackedAssets"."localDateTime" AS "stackedAssets_localDateTime", + "stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt", + "stackedAssets"."isFavorite" AS "stackedAssets_isFavorite", + "stackedAssets"."isArchived" AS "stackedAssets_isArchived", + "stackedAssets"."isExternal" AS "stackedAssets_isExternal", + "stackedAssets"."isReadOnly" AS "stackedAssets_isReadOnly", + "stackedAssets"."isOffline" AS "stackedAssets_isOffline", + "stackedAssets"."checksum" AS "stackedAssets_checksum", + "stackedAssets"."duration" AS "stackedAssets_duration", + "stackedAssets"."isVisible" AS "stackedAssets_isVisible", + "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId", + "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName", + "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath", + "stackedAssets"."stackId" AS "stackedAssets_stackId" FROM "assets" "asset" LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" + LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id" + AND ("stackedAssets"."deletedAt" IS NULL) WHERE - "asset"."ownerId" = $1 + "asset"."isVisible" = true + AND "asset"."ownerId" IN ($1) AND ("asset"."fileCreatedAt", "asset"."id") < ($2, $3) AND "asset"."updatedAt" <= $4 - AND "asset"."isVisible" = true ORDER BY "asset"."fileCreatedAt" DESC, "asset"."id" DESC @@ -816,72 +847,105 @@ LIMIT -- AssetRepository.getChangedDeltaSync SELECT - "AssetEntity"."id" AS "AssetEntity_id", - "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId", - "AssetEntity"."ownerId" AS "AssetEntity_ownerId", - "AssetEntity"."libraryId" AS "AssetEntity_libraryId", - "AssetEntity"."deviceId" AS "AssetEntity_deviceId", - "AssetEntity"."type" AS "AssetEntity_type", - "AssetEntity"."originalPath" AS "AssetEntity_originalPath", - "AssetEntity"."previewPath" AS "AssetEntity_previewPath", - "AssetEntity"."thumbnailPath" AS "AssetEntity_thumbnailPath", - "AssetEntity"."thumbhash" AS "AssetEntity_thumbhash", - "AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath", - "AssetEntity"."createdAt" AS "AssetEntity_createdAt", - "AssetEntity"."updatedAt" AS "AssetEntity_updatedAt", - "AssetEntity"."deletedAt" AS "AssetEntity_deletedAt", - "AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt", - "AssetEntity"."localDateTime" AS "AssetEntity_localDateTime", - "AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt", - "AssetEntity"."isFavorite" AS "AssetEntity_isFavorite", - "AssetEntity"."isArchived" AS "AssetEntity_isArchived", - "AssetEntity"."isExternal" AS "AssetEntity_isExternal", - "AssetEntity"."isReadOnly" AS "AssetEntity_isReadOnly", - "AssetEntity"."isOffline" AS "AssetEntity_isOffline", - "AssetEntity"."checksum" AS "AssetEntity_checksum", - "AssetEntity"."duration" AS "AssetEntity_duration", - "AssetEntity"."isVisible" AS "AssetEntity_isVisible", - "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId", - "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName", - "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath", - "AssetEntity"."stackId" AS "AssetEntity_stackId", - "AssetEntity__AssetEntity_exifInfo"."assetId" AS "AssetEntity__AssetEntity_exifInfo_assetId", - "AssetEntity__AssetEntity_exifInfo"."description" AS "AssetEntity__AssetEntity_exifInfo_description", - "AssetEntity__AssetEntity_exifInfo"."exifImageWidth" AS "AssetEntity__AssetEntity_exifInfo_exifImageWidth", - "AssetEntity__AssetEntity_exifInfo"."exifImageHeight" AS "AssetEntity__AssetEntity_exifInfo_exifImageHeight", - "AssetEntity__AssetEntity_exifInfo"."fileSizeInByte" AS "AssetEntity__AssetEntity_exifInfo_fileSizeInByte", - "AssetEntity__AssetEntity_exifInfo"."orientation" AS "AssetEntity__AssetEntity_exifInfo_orientation", - "AssetEntity__AssetEntity_exifInfo"."dateTimeOriginal" AS "AssetEntity__AssetEntity_exifInfo_dateTimeOriginal", - "AssetEntity__AssetEntity_exifInfo"."modifyDate" AS "AssetEntity__AssetEntity_exifInfo_modifyDate", - "AssetEntity__AssetEntity_exifInfo"."timeZone" AS "AssetEntity__AssetEntity_exifInfo_timeZone", - "AssetEntity__AssetEntity_exifInfo"."latitude" AS "AssetEntity__AssetEntity_exifInfo_latitude", - "AssetEntity__AssetEntity_exifInfo"."longitude" AS "AssetEntity__AssetEntity_exifInfo_longitude", - "AssetEntity__AssetEntity_exifInfo"."projectionType" AS "AssetEntity__AssetEntity_exifInfo_projectionType", - "AssetEntity__AssetEntity_exifInfo"."city" AS "AssetEntity__AssetEntity_exifInfo_city", - "AssetEntity__AssetEntity_exifInfo"."livePhotoCID" AS "AssetEntity__AssetEntity_exifInfo_livePhotoCID", - "AssetEntity__AssetEntity_exifInfo"."autoStackId" AS "AssetEntity__AssetEntity_exifInfo_autoStackId", - "AssetEntity__AssetEntity_exifInfo"."state" AS "AssetEntity__AssetEntity_exifInfo_state", - "AssetEntity__AssetEntity_exifInfo"."country" AS "AssetEntity__AssetEntity_exifInfo_country", - "AssetEntity__AssetEntity_exifInfo"."make" AS "AssetEntity__AssetEntity_exifInfo_make", - "AssetEntity__AssetEntity_exifInfo"."model" AS "AssetEntity__AssetEntity_exifInfo_model", - "AssetEntity__AssetEntity_exifInfo"."lensModel" AS "AssetEntity__AssetEntity_exifInfo_lensModel", - "AssetEntity__AssetEntity_exifInfo"."fNumber" AS "AssetEntity__AssetEntity_exifInfo_fNumber", - "AssetEntity__AssetEntity_exifInfo"."focalLength" AS "AssetEntity__AssetEntity_exifInfo_focalLength", - "AssetEntity__AssetEntity_exifInfo"."iso" AS "AssetEntity__AssetEntity_exifInfo_iso", - "AssetEntity__AssetEntity_exifInfo"."exposureTime" AS "AssetEntity__AssetEntity_exifInfo_exposureTime", - "AssetEntity__AssetEntity_exifInfo"."profileDescription" AS "AssetEntity__AssetEntity_exifInfo_profileDescription", - "AssetEntity__AssetEntity_exifInfo"."colorspace" AS "AssetEntity__AssetEntity_exifInfo_colorspace", - "AssetEntity__AssetEntity_exifInfo"."bitsPerSample" AS "AssetEntity__AssetEntity_exifInfo_bitsPerSample", - "AssetEntity__AssetEntity_exifInfo"."fps" AS "AssetEntity__AssetEntity_exifInfo_fps", - "AssetEntity__AssetEntity_stack"."id" AS "AssetEntity__AssetEntity_stack_id", - "AssetEntity__AssetEntity_stack"."primaryAssetId" AS "AssetEntity__AssetEntity_stack_primaryAssetId" + "asset"."id" AS "asset_id", + "asset"."deviceAssetId" AS "asset_deviceAssetId", + "asset"."ownerId" AS "asset_ownerId", + "asset"."libraryId" AS "asset_libraryId", + "asset"."deviceId" AS "asset_deviceId", + "asset"."type" AS "asset_type", + "asset"."originalPath" AS "asset_originalPath", + "asset"."previewPath" AS "asset_previewPath", + "asset"."thumbnailPath" AS "asset_thumbnailPath", + "asset"."thumbhash" AS "asset_thumbhash", + "asset"."encodedVideoPath" AS "asset_encodedVideoPath", + "asset"."createdAt" AS "asset_createdAt", + "asset"."updatedAt" AS "asset_updatedAt", + "asset"."deletedAt" AS "asset_deletedAt", + "asset"."fileCreatedAt" AS "asset_fileCreatedAt", + "asset"."localDateTime" AS "asset_localDateTime", + "asset"."fileModifiedAt" AS "asset_fileModifiedAt", + "asset"."isFavorite" AS "asset_isFavorite", + "asset"."isArchived" AS "asset_isArchived", + "asset"."isExternal" AS "asset_isExternal", + "asset"."isReadOnly" AS "asset_isReadOnly", + "asset"."isOffline" AS "asset_isOffline", + "asset"."checksum" AS "asset_checksum", + "asset"."duration" AS "asset_duration", + "asset"."isVisible" AS "asset_isVisible", + "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", + "asset"."originalFileName" AS "asset_originalFileName", + "asset"."sidecarPath" AS "asset_sidecarPath", + "asset"."stackId" AS "asset_stackId", + "exifInfo"."assetId" AS "exifInfo_assetId", + "exifInfo"."description" AS "exifInfo_description", + "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth", + "exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight", + "exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte", + "exifInfo"."orientation" AS "exifInfo_orientation", + "exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal", + "exifInfo"."modifyDate" AS "exifInfo_modifyDate", + "exifInfo"."timeZone" AS "exifInfo_timeZone", + "exifInfo"."latitude" AS "exifInfo_latitude", + "exifInfo"."longitude" AS "exifInfo_longitude", + "exifInfo"."projectionType" AS "exifInfo_projectionType", + "exifInfo"."city" AS "exifInfo_city", + "exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID", + "exifInfo"."autoStackId" AS "exifInfo_autoStackId", + "exifInfo"."state" AS "exifInfo_state", + "exifInfo"."country" AS "exifInfo_country", + "exifInfo"."make" AS "exifInfo_make", + "exifInfo"."model" AS "exifInfo_model", + "exifInfo"."lensModel" AS "exifInfo_lensModel", + "exifInfo"."fNumber" AS "exifInfo_fNumber", + "exifInfo"."focalLength" AS "exifInfo_focalLength", + "exifInfo"."iso" AS "exifInfo_iso", + "exifInfo"."exposureTime" AS "exifInfo_exposureTime", + "exifInfo"."profileDescription" AS "exifInfo_profileDescription", + "exifInfo"."colorspace" AS "exifInfo_colorspace", + "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample", + "exifInfo"."fps" AS "exifInfo_fps", + "stack"."id" AS "stack_id", + "stack"."primaryAssetId" AS "stack_primaryAssetId", + "stackedAssets"."id" AS "stackedAssets_id", + "stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId", + "stackedAssets"."ownerId" AS "stackedAssets_ownerId", + "stackedAssets"."libraryId" AS "stackedAssets_libraryId", + "stackedAssets"."deviceId" AS "stackedAssets_deviceId", + "stackedAssets"."type" AS "stackedAssets_type", + "stackedAssets"."originalPath" AS "stackedAssets_originalPath", + "stackedAssets"."previewPath" AS "stackedAssets_previewPath", + "stackedAssets"."thumbnailPath" AS "stackedAssets_thumbnailPath", + "stackedAssets"."thumbhash" AS "stackedAssets_thumbhash", + "stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath", + "stackedAssets"."createdAt" AS "stackedAssets_createdAt", + "stackedAssets"."updatedAt" AS "stackedAssets_updatedAt", + "stackedAssets"."deletedAt" AS "stackedAssets_deletedAt", + "stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt", + "stackedAssets"."localDateTime" AS "stackedAssets_localDateTime", + "stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt", + "stackedAssets"."isFavorite" AS "stackedAssets_isFavorite", + "stackedAssets"."isArchived" AS "stackedAssets_isArchived", + "stackedAssets"."isExternal" AS "stackedAssets_isExternal", + "stackedAssets"."isReadOnly" AS "stackedAssets_isReadOnly", + "stackedAssets"."isOffline" AS "stackedAssets_isOffline", + "stackedAssets"."checksum" AS "stackedAssets_checksum", + "stackedAssets"."duration" AS "stackedAssets_duration", + "stackedAssets"."isVisible" AS "stackedAssets_isVisible", + "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId", + "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName", + "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath", + "stackedAssets"."stackId" AS "stackedAssets_stackId" FROM - "assets" "AssetEntity" - LEFT JOIN "exif" "AssetEntity__AssetEntity_exifInfo" ON "AssetEntity__AssetEntity_exifInfo"."assetId" = "AssetEntity"."id" - LEFT JOIN "asset_stack" "AssetEntity__AssetEntity_stack" ON "AssetEntity__AssetEntity_stack"."id" = "AssetEntity"."stackId" + "assets" "asset" + LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" + LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" + LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id" + AND ("stackedAssets"."deletedAt" IS NULL) WHERE - ( - ("AssetEntity"."ownerId" IN ($1)) - AND ("AssetEntity"."isVisible" = $2) - AND ("AssetEntity"."updatedAt" > $3) + "asset"."isVisible" = true + AND "asset"."ownerId" IN ($1) + AND ( + "stack"."primaryAssetId" = "asset"."id" + OR "asset"."stackId" IS NULL ) + AND "asset"."updatedAt" > $2 diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 6bbc8cad89af3..a961ab97d6b5f 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -710,21 +710,23 @@ export class AssetRepository implements IAssetRepository { ], }) getAllForUserFullSync(options: AssetFullSyncOptions): Promise { - const { ownerId, lastCreationDate, lastId, updatedUntil, limit } = options; - const builder = this.repository - .createQueryBuilder('asset') - .leftJoinAndSelect('asset.exifInfo', 'exifInfo') - .leftJoinAndSelect('asset.stack', 'stack') - .where('asset.ownerId = :ownerId', { ownerId }); + const { ownerId, isArchived, withStacked, lastCreationDate, lastId, updatedUntil, limit } = options; + const builder = this.getBuilder({ + userIds: [ownerId], + exifInfo: true, + withStacked, + isArchived, + }); + if (lastCreationDate !== undefined && lastId !== undefined) { builder.andWhere('(asset.fileCreatedAt, asset.id) < (:lastCreationDate, :lastId)', { lastCreationDate, lastId, }); } + return builder .andWhere('asset.updatedAt <= :updatedUntil', { updatedUntil }) - .andWhere('asset.isVisible = true') .orderBy('asset.fileCreatedAt', 'DESC') .addOrderBy('asset.id', 'DESC') .limit(limit) @@ -734,18 +736,11 @@ export class AssetRepository implements IAssetRepository { @GenerateSql({ params: [{ userIds: [DummyValue.UUID], updatedAfter: DummyValue.DATE }] }) getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise { - return this.repository.find({ - where: { - ownerId: In(options.userIds), - isVisible: true, - updatedAt: MoreThan(options.updatedAfter), - }, - relations: { - exifInfo: true, - stack: true, - }, - take: options.limit, - withDeleted: true, - }); + const builder = this.getBuilder({ userIds: options.userIds, exifInfo: true, withStacked: true }) + .andWhere({ updatedAt: MoreThan(options.updatedAfter) }) + .take(options.limit) + .withDeleted(); + + return builder.getMany(); } } diff --git a/server/src/services/sync.service.spec.ts b/server/src/services/sync.service.spec.ts index 87205c08f183e..9a7dbbc15235a 100644 --- a/server/src/services/sync.service.spec.ts +++ b/server/src/services/sync.service.spec.ts @@ -39,13 +39,12 @@ describe(SyncService.name, () => { describe('getAllAssetsForUserFullSync', () => { it('should return a list of all assets owned by the user', async () => { assetMock.getAllForUserFullSync.mockResolvedValue([assetStub.external, assetStub.hasEncodedVideo]); - await expect( - sut.getAllAssetsForUserFullSync(authStub.user1, { limit: 2, updatedUntil: untilDate }), - ).resolves.toEqual([ + await expect(sut.getFullSync(authStub.user1, { limit: 2, updatedUntil: untilDate })).resolves.toEqual([ mapAsset(assetStub.external, mapAssetOpts), mapAsset(assetStub.hasEncodedVideo, mapAssetOpts), ]); expect(assetMock.getAllForUserFullSync).toHaveBeenCalledWith({ + withStacked: true, ownerId: authStub.user1.user.id, updatedUntil: untilDate, limit: 2, @@ -57,7 +56,7 @@ describe(SyncService.name, () => { it('should return a response requiring a full sync when partners are out of sync', async () => { partnerMock.getAll.mockResolvedValue([partnerStub.adminToUser1]); await expect( - sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), + sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), ).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] }); expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0); expect(auditMock.getAfter).toHaveBeenCalledTimes(0); @@ -66,7 +65,7 @@ describe(SyncService.name, () => { it('should return a response requiring a full sync when last sync was too long ago', async () => { partnerMock.getAll.mockResolvedValue([]); await expect( - sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(2000), userIds: [authStub.user1.user.id] }), + sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(2000), userIds: [authStub.user1.user.id] }), ).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] }); expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0); expect(auditMock.getAfter).toHaveBeenCalledTimes(0); @@ -78,7 +77,7 @@ describe(SyncService.name, () => { Array.from({ length: 10_000 }).fill(assetStub.image), ); await expect( - sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), + sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), ).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] }); expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1); expect(auditMock.getAfter).toHaveBeenCalledTimes(0); @@ -89,7 +88,7 @@ describe(SyncService.name, () => { assetMock.getChangedDeltaSync.mockResolvedValue([assetStub.image1]); auditMock.getAfter.mockResolvedValue([assetStub.external.id]); await expect( - sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), + sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), ).resolves.toEqual({ needsFullSync: false, upserted: [mapAsset(assetStub.image1, mapAssetOpts)], diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index be11d36fa08a5..88a4e172a6074 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -1,5 +1,4 @@ import { Inject } from '@nestjs/common'; -import _ from 'lodash'; import { DateTime } from 'luxon'; import { AUDIT_LOG_MAX_DURATION } from 'src/constants'; import { AccessCore, Permission } from 'src/cores/access.core'; @@ -11,6 +10,9 @@ import { IAccessRepository } from 'src/interfaces/access.interface'; import { IAssetRepository } from 'src/interfaces/asset.interface'; import { IAuditRepository } from 'src/interfaces/audit.interface'; import { IPartnerRepository } from 'src/interfaces/partner.interface'; +import { setIsEqual } from 'src/utils/set'; + +const FULL_SYNC = { needsFullSync: true, deleted: [], upserted: [] }; export class SyncService { private access: AccessCore; @@ -24,52 +26,69 @@ export class SyncService { this.access = AccessCore.create(accessRepository); } - async getAllAssetsForUserFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise { + async getFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise { + // mobile implementation is faster if this is a single id const userId = dto.userId || auth.user.id; await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId); const assets = await this.assetRepository.getAllForUserFullSync({ ownerId: userId, + // no archived assets for partner user + isArchived: userId === auth.user.id ? undefined : false, + // no stack for partner user + withStacked: userId === auth.user.id ? true : undefined, lastCreationDate: dto.lastCreationDate, updatedUntil: dto.updatedUntil, lastId: dto.lastId, limit: dto.limit, }); - const options = { auth, stripMetadata: false, withStack: true }; - return assets.map((a) => mapAsset(a, options)); + return assets.map((a) => mapAsset(a, { auth, stripMetadata: false, withStack: true })); } - async getChangesForDeltaSync(auth: AuthDto, dto: AssetDeltaSyncDto): Promise { - await this.access.requirePermission(auth, Permission.TIMELINE_READ, dto.userIds); - const partner = await this.partnerRepository.getAll(auth.user.id); - const userIds = [auth.user.id, ...partner.filter((p) => p.sharedWithId == auth.user.id).map((p) => p.sharedById)]; - userIds.sort(); - dto.userIds.sort(); + async getDeltaSync(auth: AuthDto, dto: AssetDeltaSyncDto): Promise { + // app has not synced in the last 100 days const duration = DateTime.now().diff(DateTime.fromJSDate(dto.updatedAfter)); + if (duration > AUDIT_LOG_MAX_DURATION) { + return FULL_SYNC; + } - if (!_.isEqual(userIds, dto.userIds) || duration > AUDIT_LOG_MAX_DURATION) { - // app does not have the correct partners synced - // or app has not synced in the last 100 days - return { needsFullSync: true, deleted: [], upserted: [] }; + const authUserId = auth.user.id; + + // app does not have the correct partners synced + const partner = await this.partnerRepository.getAll(authUserId); + const userIds = [authUserId, ...partner.filter((p) => p.sharedWithId == auth.user.id).map((p) => p.sharedById)]; + if (!setIsEqual(new Set(userIds), new Set(dto.userIds))) { + return FULL_SYNC; } + await this.access.requirePermission(auth, Permission.TIMELINE_READ, dto.userIds); + const limit = 10_000; const upserted = await this.assetRepository.getChangedDeltaSync({ limit, updatedAfter: dto.updatedAfter, userIds }); + // too many changes, need to do a full sync if (upserted.length === limit) { - // too many changes -> do a full sync (paginated) instead - return { needsFullSync: true, deleted: [], upserted: [] }; + return FULL_SYNC; } const deleted = await this.auditRepository.getAfter(dto.updatedAfter, { - userIds: userIds, + userIds, entityType: EntityType.ASSET, action: DatabaseAction.DELETE, }); - const options = { auth, stripMetadata: false, withStack: true }; const result = { needsFullSync: false, - upserted: upserted.map((a) => mapAsset(a, options)), + upserted: upserted + // do not return archived assets for partner users + .filter((a) => a.ownerId === auth.user.id || (a.ownerId !== auth.user.id && !a.isArchived)) + .map((a) => + mapAsset(a, { + auth, + stripMetadata: false, + // ignore stacks for non partner users + withStack: a.ownerId === authUserId, + }), + ), deleted, }; return result;