diff --git a/ydb/public/api/grpc/ydb_export_v1.proto b/ydb/public/api/grpc/ydb_export_v1.proto index 966fa5010030..1127b7939b75 100644 --- a/ydb/public/api/grpc/ydb_export_v1.proto +++ b/ydb/public/api/grpc/ydb_export_v1.proto @@ -17,5 +17,5 @@ service ExportService { // Exports data to file system. // Method starts an asynchronous operation that can be cancelled while it is in progress. - rpc ExportToFs(ExportToFsRequest) returns (ExportToFsResponse); + rpc ExportToFs(Export.ExportToFsRequest) returns (Export.ExportToFsResponse); } diff --git a/ydb/public/api/grpc/ydb_import_v1.proto b/ydb/public/api/grpc/ydb_import_v1.proto index 3ac2d0ca53a9..3a500dbcd389 100644 --- a/ydb/public/api/grpc/ydb_import_v1.proto +++ b/ydb/public/api/grpc/ydb_import_v1.proto @@ -18,6 +18,9 @@ service ImportService { // List objects from existing export stored in S3 bucket rpc ListObjectsInS3Export(Import.ListObjectsInS3ExportRequest) returns (Import.ListObjectsInS3ExportResponse); + // List objects from existing export stored in FS + rpc ListObjectsInFsExport(Import.ListObjectsInFsExportRequest) returns (Import.ListObjectsInFsExportResponse); + // Writes data to a table. // Method accepts serialized data in the selected format and writes it non-transactionally. rpc ImportData(Import.ImportDataRequest) returns (Import.ImportDataResponse); diff --git a/ydb/public/api/protos/ydb_export.proto b/ydb/public/api/protos/ydb_export.proto index 055262630b84..d9cb6d541080 100644 --- a/ydb/public/api/protos/ydb_export.proto +++ b/ydb/public/api/protos/ydb_export.proto @@ -126,7 +126,7 @@ message ExportToS3Settings { // it is especially useful for custom s3 implementations bool disable_virtual_addressing = 12; - // Database root if not provided. + // Defaults to database root if not provided. // All object names are calculated and written relative to this path. string source_path = 13; @@ -185,18 +185,23 @@ message EncryptionSettings { /// File system (FS) message ExportToFsSettings { message Item { - // Database path to a table to be exported + // Database path to a table/directory to be exported string source_path = 1 [(required) = true]; - /* The tables are exported to one or more files in FS. - The files are saved to the destination_path directory. */ - string destination_path = 2 [(required) = true]; + /* Tables are exported to one or more files in FS. + The path begins with 'destination_path'. + If not specified, actual FS path is the default `base_path` concatenated with: + * The object path relative to the global `source_path` for a non-encrypted export + * The anonymized path for an encrypted export + */ + string destination_path = 2; } - // Base path on FS where to write export - // Path to the mounted directory in the case of NFS + // Base path on FS where to write all export items + // In the case of NFS, one of the directories in the path must be mounted // Must be an absolute path // Example: /mnt/exports + // SchemaMapping file with the list of objects is written to this path string base_path = 1 [(required) = true]; // List of items to export @@ -212,6 +217,15 @@ message ExportToFsSettings { // - zstd. // - zstd-N, where N is compression level, e.g. zstd-3. string compression = 5; + + // Settings for data encryption. + // If encryption_settings field is not specified, + // the resulting data will not be encrypted. + EncryptionSettings encryption_settings = 6; + + // Defaults to database root if not provided. + // All object names are calculated and written relative to this path. + string source_path = 7; } message ExportToFsResult { diff --git a/ydb/public/api/protos/ydb_import.proto b/ydb/public/api/protos/ydb_import.proto index 61057dc5527f..fc251236c726 100644 --- a/ydb/public/api/protos/ydb_import.proto +++ b/ydb/public/api/protos/ydb_import.proto @@ -131,19 +131,23 @@ message ImportFromFsSettings { * '/scheme.pb' - object with information about scheme, indexes, etc; * '/permissions.pb' - object with information about ACL and owner; * '/metadata.json' - object with metadata about the backup. - The path in FS is specified relative to base_path. - Example: "my_export/table1" + The FS path can be either provided explicitly (relative to base_path) + Or, if the export contains the database objects list, you may specify the database object name, + and the FS prefix will be looked up in the database objects list by the import procedure */ - string source_path = 1 [(required) = true]; + string source_path = 1; - // Database path to a table to import to. - string destination_path = 2 [(required) = true]; + // Database path to a database object to import the item to + // Resolved relative to the default destination_path + // May be omitted if the item's source_path is specified, in this case will be taken equal to it + string destination_path = 2; } // Base path on FS where the export is located - // Path to the mounted directory in the case of NFS + // In the case of NFS, one of the directories in the path must be mounted // Must be an absolute path // Example: /mnt/exports + // SchemaMapping file with the list of objects is read from this path string base_path = 1 [(required) = true]; repeated Item items = 2; // Empty collection means import of all export objects @@ -160,6 +164,15 @@ message ImportFromFsSettings { // Skip checksum validation during import bool skip_checksum_validation = 6; + + // Destination path to restore paths inside database + // Default value is database root + string destination_path = 7; + + // Settings how data is encrypted. + // If encryption_settings field is not specified, + // the resulting data is considered not encrypted. + Ydb.Export.EncryptionSettings encryption_settings = 8; } message ImportFromFsResult { @@ -263,6 +276,67 @@ message ListObjectsInS3ExportResponse { Ydb.Operations.Operation operation = 1; } +message ListObjectsInFsExportSettings { + message Item { + // Database object path + // Recursive for directories + string path = 1; + } + + string endpoint = 1 [(required) = true]; + string base_path = 2 [(required) = true]; + repeated Item items = 3; + uint32 number_of_retries = 4; + + // Settings how data is encrypted. + // If encryption_settings field is not specified, + // the resulting data is considered not encrypted. + Ydb.Export.EncryptionSettings encryption_settings = 5; +} + +message ListObjectsInFsExportResult { + message Item { + /* YDB database objects in S3 are stored in one or more S3 objects (see ydb_export.proto). + The S3 object name begins with a prefix, followed by: + * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero; + * '/scheme.pb' - object with information about scheme, indexes, etc; + * '/permissions.pb' - object with information about ACL and owner; + * '/metadata.json' - object with metadata about the backup. + */ + string fs_path = 1; + string db_path = 2; + } + + repeated Item items = 1; + + // This token allows you to get the next page of results for ListObjectsInFsExport requests, + // if the number of results is larger than `page_size` specified in the request. + // To get the next page, specify the value of `next_page_token` as a value for + // the `page_token` parameter in the next ListObjectsInFsExport request. Subsequent ListObjectsInFsExport + // requests will have their own `next_page_token` to continue paging through the results. + string next_page_token = 2; +} + +message ListObjectsInFsExportRequest { + Ydb.Operations.OperationParams operation_params = 1; + ListObjectsInFsExportSettings settings = 2 [(required) = true]; + + // The maximum number of results per page that should be returned. If the number of available + // results is larger than `page_size`, the service returns a `next_page_token` that can be used + // to get the next page of results in subsequent ListObjectsInFsExport requests. + // 0 means that server returns all objects. + int64 page_size = 3 [(value) = "<= 10000"]; + + // Page token. Set `page_token` to the `next_page_token` returned by a previous ListObjectsInFsExport + // request to get the next page of results. + string page_token = 4; +} + +message ListObjectsInFsExportResponse { + // operation.result = ListObjectsInFsExportResult + Ydb.Operations.Operation operation = 1; +} + /// Data message YdbDumpFormat { repeated string columns = 1; diff --git a/ydb/services/ydb/backup_ut/fs_backup_validation_ut.cpp b/ydb/services/ydb/backup_ut/fs_backup_validation_ut.cpp index 80d8e1e053da..1629b5cf5084 100644 --- a/ydb/services/ydb/backup_ut/fs_backup_validation_ut.cpp +++ b/ydb/services/ydb/backup_ut/fs_backup_validation_ut.cpp @@ -280,20 +280,5 @@ Y_UNIT_TEST_SUITE_F(FsImportParamsValidationTest, TFsBackupParamsValidationTestF // WaitOpSuccess(res, "Import with absolute base_path should succeed"); } } - - Y_UNIT_TEST(EmptyImportItem) { - // Test that import item cannot be completely empty - NImport::TImportFromFsSettings settings = MakeImportSettings(TString(TempDir().Path())); - settings.AppendItem({}); // Empty item - - auto res = YdbImportClient().ImportFromFs(settings).GetValueSync(); - UNIT_ASSERT_C(!res.Status().IsSuccess(), - "Status: " << res.Status().GetStatus() << Endl << res.Status().GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL_C(res.Status().GetStatus(), NYdb::EStatus::BAD_REQUEST, - res.Status().GetIssues().ToString()); - UNIT_ASSERT_STRING_CONTAINS_C(res.Status().GetIssues().ToString(), - "source_path is required but not set", - res.Status().GetIssues().ToString()); - } }