From 2f8a48e9d5f4a9d01b5e511da1254182d852f4d1 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 6 Nov 2025 12:52:11 +0300 Subject: [PATCH 1/6] encryption --- ydb/public/api/protos/ydb_export.proto | 27 ++++++++++++++++++---- ydb/public/api/protos/ydb_import.proto | 32 ++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/ydb/public/api/protos/ydb_export.proto b/ydb/public/api/protos/ydb_export.proto index 7c22e9f57e8f..718954e640bc 100644 --- a/ydb/public/api/protos/ydb_export.proto +++ b/ydb/public/api/protos/ydb_export.proto @@ -185,12 +185,17 @@ message EncryptionSettings { /// File system (FS) message ExportToFsSettings { message Item { - // Database path to a table to be exported + // Database path to a table/directory to be exported string source_path = 1 [(required) = true]; - /* The tables are exported to one or more files in FS. - The files are saved to the destination_path directory. */ - string destination_path = 2 [(required) = true]; + /* Tables are exported to one or more files in FS. + The path begins with 'destination_path'. + Not required if the default `destination_prefix` is set. + If not specified, actual FS path is the default `destination_prefix` concatenated with: + * The object path relative to the global `source_path` for a non-encrypted export + * The anonymized path for an encrypted export + */ + string destination_path = 2; } // Base path on FS where to write export @@ -211,6 +216,20 @@ message ExportToFsSettings { // - zstd. // - zstd-N, where N is compression level, e.g. zstd-3. string compression = 5; + + // Settings for data encryption. + // If encryption_settings field is not specified, + // the resulting data will not be encrypted. + EncryptionSettings encryption_settings = 6; + + // Database root if not provided. + // All object names are calculated and written relative to this path. + string source_path = 7; + + // A default FS path prefix for all export items. + // When specified, export writes SchemaMapping file with the list of objects. + // Must be provided for an encrypted backup. + string destination_prefix = 8; } message ExportToFsResult { diff --git a/ydb/public/api/protos/ydb_import.proto b/ydb/public/api/protos/ydb_import.proto index cad19e6bd9e5..b9c706137716 100644 --- a/ydb/public/api/protos/ydb_import.proto +++ b/ydb/public/api/protos/ydb_import.proto @@ -131,13 +131,21 @@ message ImportFromFsSettings { * '/scheme.pb' - object with information about scheme, indexes, etc; * '/permissions.pb' - object with information about ACL and owner; * '/metadata.json' - object with metadata about the backup. - The path in FS is specified relative to base_path. - Example: "my_export/table1" */ - string source_path = 1 [(required) = true]; + oneof Source { + // The FS path prefix can be either provided explicitly + // (relative to base_path) + string source_prefix = 1; + + // Or, if the export contains the database objects list, you may specify the database object name, + // and the FS prefix will be looked up in the database objects list by the import procedure + string source_path = 3; + } - // Database path to a table to import to. - string destination_path = 2 [(required) = true]; + // Database path to a database object to import the item to + // Resolved relative to the default destination_path + // May be omitted if the item's source_path is specified, in this case will be taken equal to it + string destination_path = 2; } // Base path on FS where the export is located @@ -159,6 +167,20 @@ message ImportFromFsSettings { // Skip checksum validation during import bool skip_checksum_validation = 6; + + // A default path prefix for all items, + // determines that the import works with the list of objects in the SchemaMapping file. + // Must be provided to import an encrypted export. + string source_prefix = 7; + + // Destination path to restore paths inside database + // Default value is database root + string destination_path = 8; + + // Settings how data is encrypted. + // If encryption_settings field is not specified, + // the resulting data is considered not encrypted. + Ydb.Export.EncryptionSettings encryption_settings = 9; } message ImportFromFsResult { From 4c579c4e2edc7881b6201471a91c0b77082b8520 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 6 Nov 2025 17:39:56 +0300 Subject: [PATCH 2/6] fix --- ydb/public/api/protos/ydb_export.proto | 13 ++++--------- ydb/public/api/protos/ydb_import.proto | 21 ++++++--------------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/ydb/public/api/protos/ydb_export.proto b/ydb/public/api/protos/ydb_export.proto index 718954e640bc..20c4c945c9e6 100644 --- a/ydb/public/api/protos/ydb_export.proto +++ b/ydb/public/api/protos/ydb_export.proto @@ -190,17 +190,17 @@ message ExportToFsSettings { /* Tables are exported to one or more files in FS. The path begins with 'destination_path'. - Not required if the default `destination_prefix` is set. - If not specified, actual FS path is the default `destination_prefix` concatenated with: + If not specified, actual FS path is the default `base_path` concatenated with: * The object path relative to the global `source_path` for a non-encrypted export * The anonymized path for an encrypted export */ string destination_path = 2; } - // Base path on FS where to write export - // Path to the mounted directory in the case of NFS + // Base path on FS where to write all export items + // In the case of NFS, one of the directories in the path must be mounted // Example: /mnt/exports + // SchemaMapping file with the list of objects is written to this path string base_path = 1 [(required) = true]; // List of items to export @@ -225,11 +225,6 @@ message ExportToFsSettings { // Database root if not provided. // All object names are calculated and written relative to this path. string source_path = 7; - - // A default FS path prefix for all export items. - // When specified, export writes SchemaMapping file with the list of objects. - // Must be provided for an encrypted backup. - string destination_prefix = 8; } message ExportToFsResult { diff --git a/ydb/public/api/protos/ydb_import.proto b/ydb/public/api/protos/ydb_import.proto index b9c706137716..3f2e26681060 100644 --- a/ydb/public/api/protos/ydb_import.proto +++ b/ydb/public/api/protos/ydb_import.proto @@ -131,16 +131,11 @@ message ImportFromFsSettings { * '/scheme.pb' - object with information about scheme, indexes, etc; * '/permissions.pb' - object with information about ACL and owner; * '/metadata.json' - object with metadata about the backup. + The FS path can be either provided explicitly (relative to base_path) + Or, if the export contains the database objects list, you may specify the database object name, + and the FS prefix will be looked up in the database objects list by the import procedure */ - oneof Source { - // The FS path prefix can be either provided explicitly - // (relative to base_path) - string source_prefix = 1; - - // Or, if the export contains the database objects list, you may specify the database object name, - // and the FS prefix will be looked up in the database objects list by the import procedure - string source_path = 3; - } + string source_path = 3; // Database path to a database object to import the item to // Resolved relative to the default destination_path @@ -149,8 +144,9 @@ message ImportFromFsSettings { } // Base path on FS where the export is located - // Path to the mounted directory in the case of NFS + // In the case of NFS, one of the directories in the path must be mounted // Example: /mnt/exports + // SchemaMapping file with the list of objects is written at this path string base_path = 1 [(required) = true]; repeated Item items = 2; // Empty collection means import of all export objects @@ -168,11 +164,6 @@ message ImportFromFsSettings { // Skip checksum validation during import bool skip_checksum_validation = 6; - // A default path prefix for all items, - // determines that the import works with the list of objects in the SchemaMapping file. - // Must be provided to import an encrypted export. - string source_prefix = 7; - // Destination path to restore paths inside database // Default value is database root string destination_path = 8; From a4e5ebd91dd0f2159969949b89c364ef5c96e851 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 6 Nov 2025 17:42:00 +0300 Subject: [PATCH 3/6] fix nums --- ydb/public/api/protos/ydb_import.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ydb/public/api/protos/ydb_import.proto b/ydb/public/api/protos/ydb_import.proto index 3f2e26681060..d9f0e077bcec 100644 --- a/ydb/public/api/protos/ydb_import.proto +++ b/ydb/public/api/protos/ydb_import.proto @@ -135,7 +135,7 @@ message ImportFromFsSettings { Or, if the export contains the database objects list, you may specify the database object name, and the FS prefix will be looked up in the database objects list by the import procedure */ - string source_path = 3; + string source_path = 1; // Database path to a database object to import the item to // Resolved relative to the default destination_path @@ -166,12 +166,12 @@ message ImportFromFsSettings { // Destination path to restore paths inside database // Default value is database root - string destination_path = 8; + string destination_path = 7; // Settings how data is encrypted. // If encryption_settings field is not specified, // the resulting data is considered not encrypted. - Ydb.Export.EncryptionSettings encryption_settings = 9; + Ydb.Export.EncryptionSettings encryption_settings = 8; } message ImportFromFsResult { From 7c83b4ae4e1e1870b050fe49ca057fbc1427d06c Mon Sep 17 00:00:00 2001 From: stanislav_shchetinin Date: Thu, 6 Nov 2025 18:57:07 +0300 Subject: [PATCH 4/6] Update ydb/public/api/protos/ydb_import.proto Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ydb/public/api/protos/ydb_import.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/public/api/protos/ydb_import.proto b/ydb/public/api/protos/ydb_import.proto index d9f0e077bcec..2effc769e538 100644 --- a/ydb/public/api/protos/ydb_import.proto +++ b/ydb/public/api/protos/ydb_import.proto @@ -146,7 +146,7 @@ message ImportFromFsSettings { // Base path on FS where the export is located // In the case of NFS, one of the directories in the path must be mounted // Example: /mnt/exports - // SchemaMapping file with the list of objects is written at this path + // SchemaMapping file with the list of objects is read from this path string base_path = 1 [(required) = true]; repeated Item items = 2; // Empty collection means import of all export objects From 91547113e43239323a1f8b21358884b1b2759e02 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Mon, 10 Nov 2025 11:23:39 +0300 Subject: [PATCH 5/6] list obj + review --- ydb/public/api/grpc/ydb_export_v1.proto | 2 +- ydb/public/api/grpc/ydb_import_v1.proto | 3 ++ ydb/public/api/protos/ydb_export.proto | 4 +- ydb/public/api/protos/ydb_import.proto | 61 +++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/ydb/public/api/grpc/ydb_export_v1.proto b/ydb/public/api/grpc/ydb_export_v1.proto index 966fa5010030..1127b7939b75 100644 --- a/ydb/public/api/grpc/ydb_export_v1.proto +++ b/ydb/public/api/grpc/ydb_export_v1.proto @@ -17,5 +17,5 @@ service ExportService { // Exports data to file system. // Method starts an asynchronous operation that can be cancelled while it is in progress. - rpc ExportToFs(ExportToFsRequest) returns (ExportToFsResponse); + rpc ExportToFs(Export.ExportToFsRequest) returns (Export.ExportToFsResponse); } diff --git a/ydb/public/api/grpc/ydb_import_v1.proto b/ydb/public/api/grpc/ydb_import_v1.proto index 3ac2d0ca53a9..3a500dbcd389 100644 --- a/ydb/public/api/grpc/ydb_import_v1.proto +++ b/ydb/public/api/grpc/ydb_import_v1.proto @@ -18,6 +18,9 @@ service ImportService { // List objects from existing export stored in S3 bucket rpc ListObjectsInS3Export(Import.ListObjectsInS3ExportRequest) returns (Import.ListObjectsInS3ExportResponse); + // List objects from existing export stored in FS + rpc ListObjectsInFsExport(Import.ListObjectsInFsExportRequest) returns (Import.ListObjectsInFsExportResponse); + // Writes data to a table. // Method accepts serialized data in the selected format and writes it non-transactionally. rpc ImportData(Import.ImportDataRequest) returns (Import.ImportDataResponse); diff --git a/ydb/public/api/protos/ydb_export.proto b/ydb/public/api/protos/ydb_export.proto index 20c4c945c9e6..15dbaa598705 100644 --- a/ydb/public/api/protos/ydb_export.proto +++ b/ydb/public/api/protos/ydb_export.proto @@ -126,7 +126,7 @@ message ExportToS3Settings { // it is especially useful for custom s3 implementations bool disable_virtual_addressing = 12; - // Database root if not provided. + // Defaults to database root if not provided. // All object names are calculated and written relative to this path. string source_path = 13; @@ -222,7 +222,7 @@ message ExportToFsSettings { // the resulting data will not be encrypted. EncryptionSettings encryption_settings = 6; - // Database root if not provided. + // Defaults to database root if not provided. // All object names are calculated and written relative to this path. string source_path = 7; } diff --git a/ydb/public/api/protos/ydb_import.proto b/ydb/public/api/protos/ydb_import.proto index 2effc769e538..d6371e52848e 100644 --- a/ydb/public/api/protos/ydb_import.proto +++ b/ydb/public/api/protos/ydb_import.proto @@ -275,6 +275,67 @@ message ListObjectsInS3ExportResponse { Ydb.Operations.Operation operation = 1; } +message ListObjectsInFsExportSettings { + message Item { + // Database object path + // Recursive for directories + string path = 1; + } + + string endpoint = 1 [(required) = true]; + string base_path = 2 [(required) = true]; + repeated Item items = 3; + uint32 number_of_retries = 4; + + // Settings how data is encrypted. + // If encryption_settings field is not specified, + // the resulting data is considered not encrypted. + Ydb.Export.EncryptionSettings encryption_settings = 5; +} + +message ListObjectsInFsExportResult { + message Item { + /* YDB database objects in S3 are stored in one or more S3 objects (see ydb_export.proto). + The S3 object name begins with a prefix, followed by: + * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero; + * '/scheme.pb' - object with information about scheme, indexes, etc; + * '/permissions.pb' - object with information about ACL and owner; + * '/metadata.json' - object with metadata about the backup. + */ + string fs_path = 1; + string db_path = 2; + } + + repeated Item items = 1; + + // This token allows you to get the next page of results for ListObjectsInFsExport requests, + // if the number of results is larger than `page_size` specified in the request. + // To get the next page, specify the value of `next_page_token` as a value for + // the `page_token` parameter in the next ListObjectsInFsExport request. Subsequent ListObjectsInFsExport + // requests will have their own `next_page_token` to continue paging through the results. + string next_page_token = 2; +} + +message ListObjectsInFsExportRequest { + Ydb.Operations.OperationParams operation_params = 1; + ListObjectsInFsExportSettings settings = 2 [(required) = true]; + + // The maximum number of results per page that should be returned. If the number of available + // results is larger than `page_size`, the service returns a `next_page_token` that can be used + // to get the next page of results in subsequent ListObjectsInFsExport requests. + // 0 means that server returns all objects. + int64 page_size = 3 [(value) = "<= 10000"]; + + // Page token. Set `page_token` to the `next_page_token` returned by a previous ListObjectsInFsExport + // request to get the next page of results. + string page_token = 4; +} + +message ListObjectsInFsExportResponse { + // operation.result = ListObjectsInFsExportResult + Ydb.Operations.Operation operation = 1; +} + /// Data message YdbDumpFormat { repeated string columns = 1; From 743a643c8484be64f4ae3a5316042a3b741ad300 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 13 Nov 2025 11:58:27 +0300 Subject: [PATCH 6/6] remove test --- .../ydb/backup_ut/fs_backup_validation_ut.cpp | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/ydb/services/ydb/backup_ut/fs_backup_validation_ut.cpp b/ydb/services/ydb/backup_ut/fs_backup_validation_ut.cpp index 80d8e1e053da..1629b5cf5084 100644 --- a/ydb/services/ydb/backup_ut/fs_backup_validation_ut.cpp +++ b/ydb/services/ydb/backup_ut/fs_backup_validation_ut.cpp @@ -280,20 +280,5 @@ Y_UNIT_TEST_SUITE_F(FsImportParamsValidationTest, TFsBackupParamsValidationTestF // WaitOpSuccess(res, "Import with absolute base_path should succeed"); } } - - Y_UNIT_TEST(EmptyImportItem) { - // Test that import item cannot be completely empty - NImport::TImportFromFsSettings settings = MakeImportSettings(TString(TempDir().Path())); - settings.AppendItem({}); // Empty item - - auto res = YdbImportClient().ImportFromFs(settings).GetValueSync(); - UNIT_ASSERT_C(!res.Status().IsSuccess(), - "Status: " << res.Status().GetStatus() << Endl << res.Status().GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL_C(res.Status().GetStatus(), NYdb::EStatus::BAD_REQUEST, - res.Status().GetIssues().ToString()); - UNIT_ASSERT_STRING_CONTAINS_C(res.Status().GetIssues().ToString(), - "source_path is required but not set", - res.Status().GetIssues().ToString()); - } }