From 2fde6c2ac73c634ec221cb267173cc45fa0e54a2 Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Tue, 7 Oct 2025 17:44:19 +0000 Subject: [PATCH 01/18] add Full cycle local backup restore --- .../backup_collection/basic_user_scenarios.py | 309 ++++++++++++++++++ 1 file changed, 309 insertions(+) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index 9c8faf0ed1e4..5507e5db2e84 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -12,8 +12,10 @@ from ydb.tests.library.harness.kikimr_runner import KiKiMR from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator from ydb.tests.oss.ydb_sdk_import import ydb +<<<<<<< HEAD from contextlib import contextmanager +>>>>>>> 958a7211813 (add Full cycle local backup restore) logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -34,6 +36,7 @@ def is_system_object(obj): return obj.name.startswith(".") +<<<<<<< HEAD def sdk_select_table_rows(session, table, path_prefix="/Root"): sql = f'PRAGMA TablePathPrefix("{path_prefix}"); SELECT id, number, txt FROM {table} ORDER BY id;' result_sets = session.transaction().execute(sql, commit_tx=True) @@ -91,6 +94,32 @@ def create_table_with_data(session, path, not_null=False): ) +======= +def parse_yql_table(text): + if not text: + return [] + + lines = text.splitlines() + rows = [] + border_re = re.compile(r'^[\s┌┬┐└┴┘├┼┤─]+$') + for ln in lines: + ln = ln.rstrip("\r\n") + if not ln: + continue + if border_re.match(ln): + continue + if '│' not in ln: + continue + parts = [p.strip() for p in ln.split('│')] + if parts and parts[0] == '': + parts = parts[1:] + if parts and parts[-1] == '': + parts = parts[:-1] + rows.append(parts) + return rows + + +>>>>>>> c4120382b23 (add Full cycle local backup restore) class BaseTestBackupInFiles(object): @classmethod def setup_class(cls): @@ -117,6 +146,7 @@ def teardown_class(cls): def set_test_name(cls, request): cls.test_name = request.node.name +<<<<<<< HEAD @contextmanager def session_scope(self): session = self.driver.table_client.session().create() @@ -135,6 +165,10 @@ def run_tools_dump(cls, path, output_dir): if not path.startswith('/Root'): path = os.path.join('/Root', path) +======= + @classmethod + def run_tools_dump(cls, path, output_dir): +>>>>>>> c4120382b23 (add Full cycle local backup restore) _, tail = os.path.split(path) out_subdir = os.path.join(output_dir, tail) if os.path.exists(out_subdir): @@ -159,9 +193,12 @@ def run_tools_dump(cls, path, output_dir): @classmethod def run_tools_restore_import(cls, input_dir, collection_path): +<<<<<<< HEAD if not collection_path.startswith('/Root'): collection_path = os.path.join('/Root', collection_path) +======= +>>>>>>> c4120382b23 (add Full cycle local backup restore) cmd = [ backup_bin(), "--verbose", @@ -171,9 +208,15 @@ def run_tools_restore_import(cls, input_dir, collection_path): cls.root_dir, "tools", "restore", +<<<<<<< HEAD "--path", collection_path, "--input", +======= + "-p", + collection_path, + "-i", +>>>>>>> c4120382b23 (add Full cycle local backup restore) input_dir, ] return yatest.common.execute(cmd, check_exit_code=False) @@ -182,6 +225,29 @@ def scheme_listdir(self, path): return [child.name for child in self.driver.scheme_client.list_directory(path).children if not is_system_object(child)] +<<<<<<< HEAD +======= + def create_user(self, user, password="password"): + cmd = [ + backup_bin(), + "--verbose", + "--endpoint", + "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", + self.root_dir, + "yql", + "--script", + f"CREATE USER {user} PASSWORD '{password}'", + ] + yatest.common.execute(cmd) + + def create_users(self): + self.create_user("alice") + self.create_user("bob") + self.create_user("eve") + +<<<<<<< HEAD +>>>>>>> 958a7211813 (add Full cycle local backup restore) def collection_scheme_path(self, collection_name: str) -> str: return os.path.join(self.root_dir, ".backups", "collections", collection_name) @@ -408,6 +474,118 @@ def _modify_and_backup(self, collection_src): def _export_backups(self, collection_src): export_dir = output_path(self.test_name, collection_src) +======= + +def create_table_with_data(session, path, not_null=False): + full_path = "/Root/" + path + session.create_table( + full_path, + ydb.TableDescription() + .with_column( + ydb.Column( + "id", + ydb.PrimitiveType.Uint32 if not_null else ydb.OptionalType(ydb.PrimitiveType.Uint32), + ) + ) + .with_column(ydb.Column("number", ydb.OptionalType(ydb.PrimitiveType.Uint64))) + .with_column(ydb.Column("txt", ydb.OptionalType(ydb.PrimitiveType.String))) + .with_primary_keys("id"), + ) + + path_prefix, table = os.path.split(full_path) + session.transaction().execute( + ( + f'PRAGMA TablePathPrefix("{path_prefix}"); ' + f'UPSERT INTO {table} (id, number, txt) VALUES ' + f'(1, 10, "one"), (2, 20, "two"), (3, 30, "three");' + ), + commit_tx=True, + ) + + +class TestFullCycleLocalBackupRestore(BaseTestBackupInFiles): + def test_full_cycle_local_backup_restore(self): + collection_src = f"coll_src_{int(time.time())}" + collection_restore_v1 = f"coll_restore_v1_{int(time.time())}" + collection_restore_v2 = f"coll_restore_v2_{int(time.time())}" + export_dir = output_path(self.test_name, collection_src) + + t1 = "orders" + t2 = "products" + full_t1 = f"/Root/{t1}" + full_t2 = f"/Root/{t2}" + + session = self.driver.table_client.session().create() + create_table_with_data(session, t1) + create_table_with_data(session, t2) + + yql_cmd = ('PRAGMA TablePathPrefix("/Root"); ' f"SELECT id, number, txt FROM {t1} ORDER BY id;") + execution = yatest.common.execute( + [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", "/Root", "yql", "--script", yql_cmd], + check_exit_code=False, + ) + assert execution.exit_code == 0 + + create_collection_sql = ( + f"CREATE BACKUP COLLECTION `{collection_src}`\n" + f" ( TABLE `{full_t1}`\n" + f" , TABLE `{full_t2}`\n" + f" )\n" + "WITH\n" + " ( STORAGE = 'cluster'\n" + " , INCREMENTAL_BACKUP_ENABLED = 'false'\n" + " );\n" + ) + create_res = yatest.common.execute( + [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", "/Root", "yql", "--script", create_collection_sql], + check_exit_code=False, + ) + + stderr_out = "" + if create_res.std_err: + stderr_out += create_res.std_err.decode("utf-8") + if create_res.std_out: + stderr_out += create_res.std_out.decode("utf-8") + assert create_res.exit_code == 0, f"CREATE BACKUP COLLECTION failed: {stderr_out}" + + backup_res1 = yatest.common.execute( + [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", "/Root", "yql", "--script", f"BACKUP `{collection_src}`;"], + check_exit_code=False, + ) + assert backup_res1.exit_code == 0, "BACKUP (1) failed" + + snap1_res = yatest.common.execute( + [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", "/Root", "yql", "--script", yql_cmd], + check_exit_code=False, + ) + out_t1_after_full1_s = snap1_res.std_out.decode("utf-8") if snap1_res.std_out else "" + out_t1_after_full1_rows = parse_yql_table(out_t1_after_full1_s) + + create_table_with_data(session, "extra_table_1") + session.transaction().execute( + 'PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES (11, 111, "added1");', + commit_tx=True, + ) + backup_res2 = yatest.common.execute( + [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", "/Root", "yql", "--script", f"BACKUP `{collection_src}`;"], + check_exit_code=False, + ) + assert backup_res2.exit_code == 0, "BACKUP (2) failed" + + snap2_res = yatest.common.execute( + [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", "/Root", "yql", "--script", yql_cmd], + check_exit_code=False, + ) + out_t1_after_full2_s = snap2_res.std_out.decode("utf-8") if snap2_res.std_out else "" + out_t1_after_full2_rows = parse_yql_table(out_t1_after_full2_s) + +>>>>>>> c4120382b23 (add Full cycle local backup restore) if os.path.exists(export_dir): shutil.rmtree(export_dir) os.makedirs(export_dir, exist_ok=True) @@ -418,6 +596,7 @@ def _export_backups(self, collection_src): "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, "--database", +<<<<<<< HEAD self.root_dir, "tools", "dump", @@ -470,20 +649,83 @@ def _verify_restore(self, export_info, snapshot1, snapshot2): self.wait_for_collection(collection_restore_v1, timeout_s=30) self.wait_for_collection(collection_restore_v2, timeout_s=30) +======= + "/Root", + "tools", + "dump", + "-p", + f".backups/collections/{collection_src}", + "-o", + export_dir, + ] + dump_res = yatest.common.execute(dump_cmd, check_exit_code=False) + assert dump_res.exit_code == 0, "tools dump failed" + + exported_items = sorted([name for name in os.listdir(export_dir) + if os.path.isdir(os.path.join(export_dir, name))]) + assert len(exported_items) >= 2, "Expected at least 2 exported backups for this test" + + create_restore_v1_sql = ( + f"CREATE BACKUP COLLECTION `{collection_restore_v1}`\n" + f" ( TABLE `{full_t1}`\n" + f" , TABLE `{full_t2}`\n" + f" )\n" + "WITH ( STORAGE = 'cluster' );\n" + ) + create_restore_v2_sql = ( + f"CREATE BACKUP COLLECTION `{collection_restore_v2}`\n" + f" ( TABLE `{full_t1}`\n" + f" , TABLE `{full_t2}`\n" + f" )\n" + "WITH ( STORAGE = 'cluster' );\n" + ) + + res_v1 = yatest.common.execute( + [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", "/Root", "yql", "--script", create_restore_v1_sql], + check_exit_code=False, + ) + stderr_out = "" + if res_v1.std_err: + stderr_out += res_v1.std_err.decode("utf-8") + if res_v1.std_out: + stderr_out += res_v1.std_out.decode("utf-8") + assert res_v1.exit_code == 0, f"CREATE restore collection v1 failed: {stderr_out}" + + res_v2 = yatest.common.execute( + [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", "/Root", "yql", "--script", create_restore_v2_sql], + check_exit_code=False, + ) + stderr_out = "" + if res_v2.std_err: + stderr_out += res_v2.std_err.decode("utf-8") + if res_v2.std_out: + stderr_out += res_v2.std_out.decode("utf-8") + assert res_v2.exit_code == 0, f"CREATE restore collection v2 failed: {stderr_out}" +>>>>>>> c4120382b23 (add Full cycle local backup restore) bdir_v1 = os.path.join(export_dir, exported_items[0]) r1 = yatest.common.execute( [backup_bin(), "--verbose", "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, +<<<<<<< HEAD "--database", self.root_dir, "tools", "restore", "--path", f"/Root/.backups/collections/{collection_restore_v1}", "--input", bdir_v1], check_exit_code=False, ) assert r1.exit_code == 0, f"tools restore import v1 failed: {r1.std_err}" +======= + "--database", "/Root", "tools", "restore", "-p", f".backups/collections/{collection_restore_v1}", "-i", bdir_v1], + check_exit_code=False, + ) + assert r1.exit_code == 0, f"tools restore import v1 failed for {bdir_v1}" +>>>>>>> c4120382b23 (add Full cycle local backup restore) bdir_v2 = os.path.join(export_dir, exported_items[1]) r2 = yatest.common.execute( [backup_bin(), "--verbose", "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, +<<<<<<< HEAD "--database", self.root_dir, "tools", "restore", "--path", f"/Root/.backups/collections/{collection_restore_v2}", "--input", bdir_v2], @@ -540,6 +782,7 @@ def test_full_cycle_local_backup_restore(self): # Restore and verify self._verify_restore(export_info, snapshot1, snapshot2) +<<<<<<< HEAD class TestFullCycleLocalBackupRestoreWIncr(TestFullCycleLocalBackupRestore): @@ -941,3 +1184,69 @@ def record_last_snapshot(): # cleanup if os.path.exists(export_dir): shutil.rmtree(export_dir) +======= +======= + "--database", "/Root", "tools", "restore", "-p", f".backups/collections/{collection_restore_v2}", "-i", bdir_v2], + check_exit_code=False, + ) + assert r2.exit_code == 0, f"tools restore import v2 failed for {bdir_v2}" + + rest_call_v1 = yatest.common.execute( + [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", "/Root", "yql", "--script", f"RESTORE `{collection_restore_v1}`;"], + check_exit_code=False, + ) + assert rest_call_v1.exit_code != 0, "Expected RESTORE v1 to fail when target tables already exist" + + session.execute_scheme(f"DROP TABLE `{full_t1}`;") + session.execute_scheme(f"DROP TABLE `{full_t2}`;") + try: + session.execute_scheme('DROP TABLE `/Root/extra_table_1`;') + except Exception: + pass + + restore_exec_v1 = yatest.common.execute( + [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", "/Root", "yql", "--script", f"RESTORE `{collection_restore_v1}`;"], + check_exit_code=False, + ) + assert restore_exec_v1.exit_code == 0, "RESTORE v1 failed" + + select_cmd = [ + backup_bin(), + "--endpoint", + "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", + "/Root", + "yql", + "--script", + f'PRAGMA TablePathPrefix("/Root"); SELECT id, number, txt FROM {t1} ORDER BY id;', + ] + select_res1 = yatest.common.execute(select_cmd, check_exit_code=False) + select_text1 = select_res1.std_out.decode("utf-8") if select_res1.std_out else "" + select_rows1 = parse_yql_table(select_text1) + assert select_rows1 == out_t1_after_full1_rows, "Restored data (v1) does not match snapshot after full1" + + session.execute_scheme(f"DROP TABLE `{full_t1}`;") + session.execute_scheme(f"DROP TABLE `{full_t2}`;") + try: + session.execute_scheme('DROP TABLE `/Root/extra_table_1`;') + except Exception: + pass + + restore_exec_v2 = yatest.common.execute( + [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", "/Root", "yql", "--script", f"RESTORE `{collection_restore_v2}`;"], + check_exit_code=False, + ) + assert restore_exec_v2.exit_code == 0, "RESTORE v2 failed" + + select_res2 = yatest.common.execute(select_cmd, check_exit_code=False) + select_text2 = select_res2.std_out.decode("utf-8") if select_res2.std_out else "" + select_rows2 = parse_yql_table(select_text2) + assert select_rows2 == out_t1_after_full2_rows, "Restored data (v2) does not match snapshot after full2" + + if os.path.exists(export_dir): + shutil.rmtree(export_dir) +>>>>>>> c4120382b23 (add Full cycle local backup restore) +>>>>>>> 958a7211813 (add Full cycle local backup restore) From 51618da964cbb0b938b83d0f2127691a75e603b8 Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Thu, 9 Oct 2025 18:02:56 +0000 Subject: [PATCH 02/18] Few changes --- .../backup_collection/basic_user_scenarios.py | 715 +++++++++++------- 1 file changed, 452 insertions(+), 263 deletions(-) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index 5507e5db2e84..f36d1acf82b7 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -12,10 +12,13 @@ from ydb.tests.library.harness.kikimr_runner import KiKiMR from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator from ydb.tests.oss.ydb_sdk_import import ydb -<<<<<<< HEAD from contextlib import contextmanager +<<<<<<< HEAD >>>>>>> 958a7211813 (add Full cycle local backup restore) +======= + +>>>>>>> d2ba25d3a53 (Few changes) logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -36,7 +39,6 @@ def is_system_object(obj): return obj.name.startswith(".") -<<<<<<< HEAD def sdk_select_table_rows(session, table, path_prefix="/Root"): sql = f'PRAGMA TablePathPrefix("{path_prefix}"); SELECT id, number, txt FROM {table} ORDER BY id;' result_sets = session.transaction().execute(sql, commit_tx=True) @@ -94,32 +96,6 @@ def create_table_with_data(session, path, not_null=False): ) -======= -def parse_yql_table(text): - if not text: - return [] - - lines = text.splitlines() - rows = [] - border_re = re.compile(r'^[\s┌┬┐└┴┘├┼┤─]+$') - for ln in lines: - ln = ln.rstrip("\r\n") - if not ln: - continue - if border_re.match(ln): - continue - if '│' not in ln: - continue - parts = [p.strip() for p in ln.split('│')] - if parts and parts[0] == '': - parts = parts[1:] - if parts and parts[-1] == '': - parts = parts[:-1] - rows.append(parts) - return rows - - ->>>>>>> c4120382b23 (add Full cycle local backup restore) class BaseTestBackupInFiles(object): @classmethod def setup_class(cls): @@ -146,7 +122,6 @@ def teardown_class(cls): def set_test_name(cls, request): cls.test_name = request.node.name -<<<<<<< HEAD @contextmanager def session_scope(self): session = self.driver.table_client.session().create() @@ -165,10 +140,6 @@ def run_tools_dump(cls, path, output_dir): if not path.startswith('/Root'): path = os.path.join('/Root', path) -======= - @classmethod - def run_tools_dump(cls, path, output_dir): ->>>>>>> c4120382b23 (add Full cycle local backup restore) _, tail = os.path.split(path) out_subdir = os.path.join(output_dir, tail) if os.path.exists(out_subdir): @@ -193,12 +164,9 @@ def run_tools_dump(cls, path, output_dir): @classmethod def run_tools_restore_import(cls, input_dir, collection_path): -<<<<<<< HEAD if not collection_path.startswith('/Root'): collection_path = os.path.join('/Root', collection_path) -======= ->>>>>>> c4120382b23 (add Full cycle local backup restore) cmd = [ backup_bin(), "--verbose", @@ -208,15 +176,9 @@ def run_tools_restore_import(cls, input_dir, collection_path): cls.root_dir, "tools", "restore", -<<<<<<< HEAD "--path", collection_path, "--input", -======= - "-p", - collection_path, - "-i", ->>>>>>> c4120382b23 (add Full cycle local backup restore) input_dir, ] return yatest.common.execute(cmd, check_exit_code=False) @@ -246,8 +208,11 @@ def create_users(self): self.create_user("bob") self.create_user("eve") +<<<<<<< HEAD <<<<<<< HEAD >>>>>>> 958a7211813 (add Full cycle local backup restore) +======= +>>>>>>> d2ba25d3a53 (Few changes) def collection_scheme_path(self, collection_name: str) -> str: return os.path.join(self.root_dir, ".backups", "collections", collection_name) @@ -474,118 +439,6 @@ def _modify_and_backup(self, collection_src): def _export_backups(self, collection_src): export_dir = output_path(self.test_name, collection_src) -======= - -def create_table_with_data(session, path, not_null=False): - full_path = "/Root/" + path - session.create_table( - full_path, - ydb.TableDescription() - .with_column( - ydb.Column( - "id", - ydb.PrimitiveType.Uint32 if not_null else ydb.OptionalType(ydb.PrimitiveType.Uint32), - ) - ) - .with_column(ydb.Column("number", ydb.OptionalType(ydb.PrimitiveType.Uint64))) - .with_column(ydb.Column("txt", ydb.OptionalType(ydb.PrimitiveType.String))) - .with_primary_keys("id"), - ) - - path_prefix, table = os.path.split(full_path) - session.transaction().execute( - ( - f'PRAGMA TablePathPrefix("{path_prefix}"); ' - f'UPSERT INTO {table} (id, number, txt) VALUES ' - f'(1, 10, "one"), (2, 20, "two"), (3, 30, "three");' - ), - commit_tx=True, - ) - - -class TestFullCycleLocalBackupRestore(BaseTestBackupInFiles): - def test_full_cycle_local_backup_restore(self): - collection_src = f"coll_src_{int(time.time())}" - collection_restore_v1 = f"coll_restore_v1_{int(time.time())}" - collection_restore_v2 = f"coll_restore_v2_{int(time.time())}" - export_dir = output_path(self.test_name, collection_src) - - t1 = "orders" - t2 = "products" - full_t1 = f"/Root/{t1}" - full_t2 = f"/Root/{t2}" - - session = self.driver.table_client.session().create() - create_table_with_data(session, t1) - create_table_with_data(session, t2) - - yql_cmd = ('PRAGMA TablePathPrefix("/Root"); ' f"SELECT id, number, txt FROM {t1} ORDER BY id;") - execution = yatest.common.execute( - [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", "/Root", "yql", "--script", yql_cmd], - check_exit_code=False, - ) - assert execution.exit_code == 0 - - create_collection_sql = ( - f"CREATE BACKUP COLLECTION `{collection_src}`\n" - f" ( TABLE `{full_t1}`\n" - f" , TABLE `{full_t2}`\n" - f" )\n" - "WITH\n" - " ( STORAGE = 'cluster'\n" - " , INCREMENTAL_BACKUP_ENABLED = 'false'\n" - " );\n" - ) - create_res = yatest.common.execute( - [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", "/Root", "yql", "--script", create_collection_sql], - check_exit_code=False, - ) - - stderr_out = "" - if create_res.std_err: - stderr_out += create_res.std_err.decode("utf-8") - if create_res.std_out: - stderr_out += create_res.std_out.decode("utf-8") - assert create_res.exit_code == 0, f"CREATE BACKUP COLLECTION failed: {stderr_out}" - - backup_res1 = yatest.common.execute( - [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", "/Root", "yql", "--script", f"BACKUP `{collection_src}`;"], - check_exit_code=False, - ) - assert backup_res1.exit_code == 0, "BACKUP (1) failed" - - snap1_res = yatest.common.execute( - [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", "/Root", "yql", "--script", yql_cmd], - check_exit_code=False, - ) - out_t1_after_full1_s = snap1_res.std_out.decode("utf-8") if snap1_res.std_out else "" - out_t1_after_full1_rows = parse_yql_table(out_t1_after_full1_s) - - create_table_with_data(session, "extra_table_1") - session.transaction().execute( - 'PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES (11, 111, "added1");', - commit_tx=True, - ) - backup_res2 = yatest.common.execute( - [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", "/Root", "yql", "--script", f"BACKUP `{collection_src}`;"], - check_exit_code=False, - ) - assert backup_res2.exit_code == 0, "BACKUP (2) failed" - - snap2_res = yatest.common.execute( - [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", "/Root", "yql", "--script", yql_cmd], - check_exit_code=False, - ) - out_t1_after_full2_s = snap2_res.std_out.decode("utf-8") if snap2_res.std_out else "" - out_t1_after_full2_rows = parse_yql_table(out_t1_after_full2_s) - ->>>>>>> c4120382b23 (add Full cycle local backup restore) if os.path.exists(export_dir): shutil.rmtree(export_dir) os.makedirs(export_dir, exist_ok=True) @@ -596,7 +449,6 @@ def test_full_cycle_local_backup_restore(self): "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, "--database", -<<<<<<< HEAD self.root_dir, "tools", "dump", @@ -649,83 +501,20 @@ def _verify_restore(self, export_info, snapshot1, snapshot2): self.wait_for_collection(collection_restore_v1, timeout_s=30) self.wait_for_collection(collection_restore_v2, timeout_s=30) -======= - "/Root", - "tools", - "dump", - "-p", - f".backups/collections/{collection_src}", - "-o", - export_dir, - ] - dump_res = yatest.common.execute(dump_cmd, check_exit_code=False) - assert dump_res.exit_code == 0, "tools dump failed" - - exported_items = sorted([name for name in os.listdir(export_dir) - if os.path.isdir(os.path.join(export_dir, name))]) - assert len(exported_items) >= 2, "Expected at least 2 exported backups for this test" - - create_restore_v1_sql = ( - f"CREATE BACKUP COLLECTION `{collection_restore_v1}`\n" - f" ( TABLE `{full_t1}`\n" - f" , TABLE `{full_t2}`\n" - f" )\n" - "WITH ( STORAGE = 'cluster' );\n" - ) - create_restore_v2_sql = ( - f"CREATE BACKUP COLLECTION `{collection_restore_v2}`\n" - f" ( TABLE `{full_t1}`\n" - f" , TABLE `{full_t2}`\n" - f" )\n" - "WITH ( STORAGE = 'cluster' );\n" - ) - - res_v1 = yatest.common.execute( - [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", "/Root", "yql", "--script", create_restore_v1_sql], - check_exit_code=False, - ) - stderr_out = "" - if res_v1.std_err: - stderr_out += res_v1.std_err.decode("utf-8") - if res_v1.std_out: - stderr_out += res_v1.std_out.decode("utf-8") - assert res_v1.exit_code == 0, f"CREATE restore collection v1 failed: {stderr_out}" - - res_v2 = yatest.common.execute( - [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", "/Root", "yql", "--script", create_restore_v2_sql], - check_exit_code=False, - ) - stderr_out = "" - if res_v2.std_err: - stderr_out += res_v2.std_err.decode("utf-8") - if res_v2.std_out: - stderr_out += res_v2.std_out.decode("utf-8") - assert res_v2.exit_code == 0, f"CREATE restore collection v2 failed: {stderr_out}" ->>>>>>> c4120382b23 (add Full cycle local backup restore) bdir_v1 = os.path.join(export_dir, exported_items[0]) r1 = yatest.common.execute( [backup_bin(), "--verbose", "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, -<<<<<<< HEAD "--database", self.root_dir, "tools", "restore", "--path", f"/Root/.backups/collections/{collection_restore_v1}", "--input", bdir_v1], check_exit_code=False, ) assert r1.exit_code == 0, f"tools restore import v1 failed: {r1.std_err}" -======= - "--database", "/Root", "tools", "restore", "-p", f".backups/collections/{collection_restore_v1}", "-i", bdir_v1], - check_exit_code=False, - ) - assert r1.exit_code == 0, f"tools restore import v1 failed for {bdir_v1}" ->>>>>>> c4120382b23 (add Full cycle local backup restore) bdir_v2 = os.path.join(export_dir, exported_items[1]) r2 = yatest.common.execute( [backup_bin(), "--verbose", "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, -<<<<<<< HEAD "--database", self.root_dir, "tools", "restore", "--path", f"/Root/.backups/collections/{collection_restore_v2}", "--input", bdir_v2], @@ -783,6 +572,7 @@ def test_full_cycle_local_backup_restore(self): # Restore and verify self._verify_restore(export_info, snapshot1, snapshot2) <<<<<<< HEAD +<<<<<<< HEAD class TestFullCycleLocalBackupRestoreWIncr(TestFullCycleLocalBackupRestore): @@ -1190,63 +980,462 @@ def record_last_snapshot(): check_exit_code=False, ) assert r2.exit_code == 0, f"tools restore import v2 failed for {bdir_v2}" +======= +>>>>>>> d2ba25d3a53 (Few changes) - rest_call_v1 = yatest.common.execute( - [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", "/Root", "yql", "--script", f"RESTORE `{collection_restore_v1}`;"], - check_exit_code=False, - ) - assert rest_call_v1.exit_code != 0, "Expected RESTORE v1 to fail when target tables already exist" - session.execute_scheme(f"DROP TABLE `{full_t1}`;") - session.execute_scheme(f"DROP TABLE `{full_t2}`;") - try: - session.execute_scheme('DROP TABLE `/Root/extra_table_1`;') - except Exception: - pass +class TestFullCycleLocalBackupRestoreWIncr(BaseTestBackupInFiles): + def _capture_snapshot(self, table): + with self.session_scope() as session: + return sdk_select_table_rows(session, table) - restore_exec_v1 = yatest.common.execute( - [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", "/Root", "yql", "--script", f"RESTORE `{collection_restore_v1}`;"], - check_exit_code=False, - ) - assert restore_exec_v1.exit_code == 0, "RESTORE v1 failed" + def _export_backups(self, collection_src): + export_dir = output_path(self.test_name, collection_src) + if os.path.exists(export_dir): + shutil.rmtree(export_dir) + os.makedirs(export_dir, exist_ok=True) - select_cmd = [ + dump_cmd = [ backup_bin(), + "--verbose", "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, "--database", - "/Root", + self.root_dir, + "tools", + "dump", + "--path", + f"/Root/.backups/collections/{collection_src}", + "--output", + export_dir, + ] + dump_res = yatest.common.execute(dump_cmd, check_exit_code=False) + if dump_res.exit_code != 0: + raise AssertionError(f"tools dump failed: {dump_res.std_err}") + + exported_items = sorted([name for name in os.listdir(export_dir) + if os.path.isdir(os.path.join(export_dir, name))]) + assert len(exported_items) >= 2, f"Expected at least 2 exported backups, got: {exported_items}" + + return export_dir, exported_items + + def _execute_yql(self, script, verbose=False): + cmd = [backup_bin()] + if verbose: + cmd.append("--verbose") + cmd += [ + "--endpoint", + f"grpc://localhost:{self.cluster.nodes[1].grpc_port}", + "--database", + self.root_dir, "yql", "--script", - f'PRAGMA TablePathPrefix("/Root"); SELECT id, number, txt FROM {t1} ORDER BY id;', + script, ] - select_res1 = yatest.common.execute(select_cmd, check_exit_code=False) - select_text1 = select_res1.std_out.decode("utf-8") if select_res1.std_out else "" - select_rows1 = parse_yql_table(select_text1) - assert select_rows1 == out_t1_after_full1_rows, "Restored data (v1) does not match snapshot after full1" + return yatest.common.execute(cmd, check_exit_code=False) - session.execute_scheme(f"DROP TABLE `{full_t1}`;") - session.execute_scheme(f"DROP TABLE `{full_t2}`;") - try: - session.execute_scheme('DROP TABLE `/Root/extra_table_1`;') - except Exception: - pass + def _setup_test_collections(self): + collection_src = f"coll_src_{int(time.time())}" + t1 = "orders" + t2 = "products" - restore_exec_v2 = yatest.common.execute( - [backup_bin(), "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", "/Root", "yql", "--script", f"RESTORE `{collection_restore_v2}`;"], - check_exit_code=False, - ) - assert restore_exec_v2.exit_code == 0, "RESTORE v2 failed" + with self.session_scope() as session: + create_table_with_data(session, t1) + create_table_with_data(session, t2) - select_res2 = yatest.common.execute(select_cmd, check_exit_code=False) - select_text2 = select_res2.std_out.decode("utf-8") if select_res2.std_out else "" - select_rows2 = parse_yql_table(select_text2) - assert select_rows2 == out_t1_after_full2_rows, "Restored data (v2) does not match snapshot after full2" + return collection_src, t1, t2 - if os.path.exists(export_dir): - shutil.rmtree(export_dir) ->>>>>>> c4120382b23 (add Full cycle local backup restore) ->>>>>>> 958a7211813 (add Full cycle local backup restore) + def _modify_data_add_and_remove(self, add_rows: List[tuple] = None, remove_ids: List[int] = None): + add_rows = add_rows or [] + remove_ids = remove_ids or [] + + with self.session_scope() as session: + # Adds + if add_rows: + values = ", ".join(f"({i},{n},\"{t}\")" for i, n, t in add_rows) + session.transaction().execute( + f'PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES {values};', + commit_tx=True, + ) + # Removes + for rid in remove_ids: + session.transaction().execute( + f'PRAGMA TablePathPrefix("/Root"); DELETE FROM orders WHERE id = {rid};', + commit_tx=True, + ) + + def _add_more_tables(self, prefix: str, count: int = 1): + created = [] + for i in range(1, count + 1): + name = f"{prefix}_{i}_{int(time.time()) % 10000}" + with self.session_scope() as session: + create_table_with_data(session, name) + created.append(f"/Root/{name}") + return created + + def _remove_tables(self, table_paths: List[str]): + with self.session_scope() as session: + for tp in table_paths: + try: + session.execute_scheme(f"DROP TABLE `{tp}`;") + except Exception: + logger.debug(f"Failed to drop table {tp} (maybe not exists)") + + # extract timestamp prefix from exported folder name + name_re = re.compile(r"^([0-9]{8}T[0-9]{6}Z?)_(full|incremental)") + + def extract_ts(self, name): + m = self.name_re.match(name) + if m: + return m.group(1) + return name.split("_", 1)[0] + + def import_exported_up_to_timestamp(self, target_collection, target_ts, export_dir, full_orders, full_products, timeout_s=60): + create_sql = f""" + CREATE BACKUP COLLECTION `{target_collection}` + ( TABLE `{full_orders}`, TABLE `{full_products}` ) + WITH ( STORAGE = 'cluster' ); + """ + res = self._execute_yql(create_sql) + assert res.exit_code == 0, f"CREATE {target_collection} failed: {getattr(res, 'std_err', None)}" + self.wait_for_collection(target_collection, timeout_s=30) + + all_dirs = sorted([d for d in os.listdir(export_dir) if os.path.isdir(os.path.join(export_dir, d))]) + chosen = [d for d in all_dirs if self.extract_ts(d) <= target_ts] + assert chosen, f"No exported snapshots with ts <= {target_ts} found in {export_dir}: {all_dirs}" + + logger.info(f"Will import into {target_collection} these snapshots (in order): {chosen}") + + for name in chosen: + src = os.path.join(export_dir, name) + dest_path = f"/Root/.backups/collections/{target_collection}/{name}" + logger.info(f"Importing {name} (ts={self.extract_ts(name)}) -> {dest_path}") + r = yatest.common.execute( + [ + backup_bin(), + "--verbose", + "--endpoint", + "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", + self.root_dir, + "tools", + "restore", + "--path", + dest_path, + "--input", + src, + ], + check_exit_code=False, + ) + out = (r.std_out or b"").decode("utf-8", "ignore") + err = (r.std_err or b"").decode("utf-8", "ignore") + if r.exit_code != 0: + logger.error(f"tools restore import failed for {name}: exit={r.exit_code} stdout={out} stderr={err}") + assert r.exit_code == 0, f"tools restore import failed for {name}: stdout={out} stderr={err}" + + # --- WAIT until imported snapshots are visible in scheme (registered) --- + deadline = time.time() + timeout_s + expected = set(chosen) + while time.time() < deadline: + try: + kids = set(self.get_collection_children(target_collection)) + if expected.issubset(kids): + logger.info(f"All imported snapshots are registered in collection {target_collection}") + break + else: + missing = expected - kids + logger.info(f"Waiting for registered snapshots in {target_collection}, missing: {missing}") + except Exception as e: + logger.debug(f"While waiting for imported snapshots: {e}") + time.sleep(5) + else: + # timeout + try: + kids = sorted(self.get_collection_children(target_collection)) + except Exception: + kids = "" + raise AssertionError(f"Imported snapshots did not appear in collection {target_collection} within {timeout_s}s. Expected: {sorted(chosen)}. Present: {kids}") + + # small safety pause to ensure system is stable before RESTORE + time.sleep(5) + + def normalize_rows(self, rows): + header = rows[0] + body = rows[1:] + + def norm_val(v): + return v.decode() if isinstance(v, (bytes, bytearray)) else str(v) + sorted_body = sorted([tuple(norm_val(x) for x in r) for r in body]) + return (tuple(header), tuple(sorted_body)) + + def test_full_cycle_local_backup_restore_with_incrementals(self): + # setup + collection_src, t_orders, t_products = self._setup_test_collections() + full_orders = f"/Root/{t_orders}" + full_products = f"/Root/{t_products}" + + # Keep created snapshot names + created_snapshots: List[str] = [] + snapshot_rows = {} # name -> captured rows + + # Create collection with incremental enabled + create_collection_sql = f""" + CREATE BACKUP COLLECTION `{collection_src}` + ( TABLE `{full_orders}`, TABLE `{full_products}` ) + WITH ( STORAGE = 'cluster', INCREMENTAL_BACKUP_ENABLED = 'true' ); + """ + create_res = self._execute_yql(create_collection_sql) + assert create_res.exit_code == 0, "CREATE BACKUP COLLECTION failed" + self.wait_for_collection(collection_src, timeout_s=30) + + def record_last_snapshot(): + kids = sorted(self.get_collection_children(collection_src)) + assert kids, "No snapshots found after backup" + last = kids[-1] + created_snapshots.append(last) + return last + + # Add/remove data + self._modify_data_add_and_remove(add_rows=[(10, 1000, "a1")], remove_ids=[2]) + + # Add more tables (1) + extras = [] + extras += self._add_more_tables("extra1", 1) + + # Create full backup 1 + time.sleep(1.1) + res = self._execute_yql(f"BACKUP `{collection_src}`;") + assert res.exit_code == 0, f"FULL BACKUP 1 failed: {getattr(res, 'std_err', None)}" + self.wait_for_collection_has_snapshot(collection_src, timeout_s=30) + snap_full1 = record_last_snapshot() + snapshot_rows[snap_full1] = self._capture_snapshot(t_orders) + + # Add/remove data + self._modify_data_add_and_remove(add_rows=[(20, 2000, "b1")], remove_ids=[1]) + + # Add more tables, remove some tables from step 4 + extras += self._add_more_tables("extra2", 1) + if extras: + self._remove_tables([extras[0]]) + + # Create incremental backup 1 + time.sleep(1.1) + res = self._execute_yql(f"BACKUP `{collection_src}` INCREMENTAL;") + assert res.exit_code == 0, "INCREMENTAL 1 failed" + snap_inc1 = record_last_snapshot() + snapshot_rows[snap_inc1] = self._capture_snapshot(t_orders) + + # Add/remove + self._modify_data_add_and_remove(add_rows=[(30, 3000, "c1")], remove_ids=[10]) + + # Add more tables, Remove some tables from step 5 + extras += self._add_more_tables("extra3", 1) + if len(extras) >= 2: + self._remove_tables([extras[1]]) + + # Create incremental backup 2 + time.sleep(1.1) + res = self._execute_yql(f"BACKUP `{collection_src}` INCREMENTAL;") + assert res.exit_code == 0, "INCREMENTAL 2 failed" + snap_inc2 = record_last_snapshot() + snapshot_rows[snap_inc2] = self._capture_snapshot(t_orders) + + # Add more tables, remove some tables from step 5 + extras += self._add_more_tables("extra4", 1) + if len(extras) >= 3: + self._remove_tables([extras[2]]) + + # Add/remove + self._modify_data_add_and_remove(add_rows=[(40, 4000, "d1")], remove_ids=[20]) + + # Create full backup 2 + time.sleep(1.1) + res = self._execute_yql(f"BACKUP `{collection_src}`;") + assert res.exit_code == 0, "FULL BACKUP 2 failed" + snap_full2 = record_last_snapshot() + snapshot_rows[snap_full2] = self._capture_snapshot(t_orders) + + # Create incremental backup 3 + time.sleep(1.1) + res = self._execute_yql(f"BACKUP `{collection_src}` INCREMENTAL;") + assert res.exit_code == 0, "INCREMENTAL 3 failed" + snap_inc3 = record_last_snapshot() + snapshot_rows[snap_inc3] = self._capture_snapshot(t_orders) + + # Add more tables, remove some tables from step 5 + extras += self._add_more_tables("extra5", 1) + if len(extras) >= 4: + self._remove_tables([extras[3]]) + + # Add/remove + self._modify_data_add_and_remove(add_rows=[(50, 5000, "e1")], remove_ids=[30]) + + # Create incremental backup 4 + time.sleep(1.1) + res = self._execute_yql(f"BACKUP `{collection_src}` INCREMENTAL;") + assert res.exit_code == 0, "INCREMENTAL 4 failed" + snap_inc4 = record_last_snapshot() + snapshot_rows[snap_inc4] = self._capture_snapshot(t_orders) + + # Export backups + export_dir, exported_items = self._export_backups(collection_src) + exported_dirs = sorted([d for d in os.listdir(export_dir) if os.path.isdir(os.path.join(export_dir, d))]) + + # all recorded snapshots should be exported + for s in created_snapshots: + assert s in exported_dirs, f"Recorded snapshot {s} not in exported dirs {exported_dirs}" + + # Try to restore and get error that tables already exist + restore_all_col = f"restore_all_{int(time.time())}" + # import all snapshots up to the latest snapshot + latest_ts = self.extract_ts(created_snapshots[-1]) + self.import_exported_up_to_timestamp(restore_all_col, latest_ts, export_dir, full_orders, full_products) + rest_all = self._execute_yql(f"RESTORE `{restore_all_col}`;") + assert rest_all.exit_code != 0, "Expected RESTORE to fail when tables already exist" + + # Remove all tables + self._remove_tables([full_orders, full_products] + extras) + + # Restore to full backup 1 + col_full1 = f"restore_full1_{int(time.time())}" + ts_full1 = self.extract_ts(snap_full1) + self.import_exported_up_to_timestamp(col_full1, ts_full1, export_dir, full_orders, full_products) + rest_full1 = self._execute_yql(f"RESTORE `{col_full1}`;") + assert rest_full1.exit_code == 0, f"RESTORE full1 failed: {rest_full1.std_err}" + restored_rows = self._capture_snapshot(t_orders) + assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_full1]), "Verify data in backup (1) failed" + + # Restore to incremental 1 (full1 + inc1) + col_inc1 = f"restore_inc1_{int(time.time())}" + ts_inc1 = self.extract_ts(snap_inc1) + self.import_exported_up_to_timestamp(col_inc1, ts_inc1, export_dir, full_orders, full_products) + # ensure target tables absent + self._remove_tables([full_orders, full_products]) + rest_inc1 = self._execute_yql(f"RESTORE `{col_inc1}`;") + assert rest_inc1.exit_code == 0, f"RESTORE inc1 failed: {rest_inc1.std_err}" + restored_rows = self._capture_snapshot(t_orders) + assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_inc1]), "Verify data in backup (2) failed" + + # Restore to incremental 2 (full1 + inc1 + inc2) + col_inc2 = f"restore_inc2_{int(time.time())}" + ts_inc2 = self.extract_ts(snap_inc2) + self.import_exported_up_to_timestamp(col_inc2, ts_inc2, export_dir, full_orders, full_products) + self._remove_tables([full_orders, full_products]) + rest_inc2 = self._execute_yql(f"RESTORE `{col_inc2}`;") + time.sleep(1.1) + assert rest_inc2.exit_code == 0, f"RESTORE inc2 failed: {rest_inc2.std_err}" + restored_rows = self._capture_snapshot(t_orders) + assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_inc2]), "Verify data in backup (3) failed" + + # Remove all tables (2) + self._remove_tables([full_orders, full_products] + extras) + + # Try to restore incremental-only (no base full) -> expect fail + inc_only_col = f"inc_only_{int(time.time())}" + # pick incrementals strictly after full1 + idx_full1 = created_snapshots.index(snap_full1) + incs_after_full1 = [s for s in created_snapshots if "_incremental" in s and created_snapshots.index(s) > idx_full1] + if incs_after_full1: + # import incrementals only (should fail on restore) + create_sql = f""" + CREATE BACKUP COLLECTION `{inc_only_col}` + ( TABLE `{full_orders}`, TABLE `{full_products}` ) + WITH ( STORAGE = 'cluster' ); + """ + res = self._execute_yql(create_sql) + assert res.exit_code == 0 + self.wait_for_collection(inc_only_col, timeout_s=30) + for s in incs_after_full1: + src = os.path.join(export_dir, s) + dest_path = f"/Root/.backups/collections/{inc_only_col}/{s}" + r = yatest.common.execute( + [ + backup_bin(), + "--verbose", + "--endpoint", + "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", + self.root_dir, + "tools", + "restore", + "--path", + dest_path, + "--input", + src, + ], + check_exit_code=False, + ) + assert r.exit_code == 0, f"tools restore import for inc-only failed: {r.std_err}" + rest_inc_only = self._execute_yql(f"RESTORE `{inc_only_col}`;") + assert rest_inc_only.exit_code != 0, "Expected restore of incrementals-only (no base) to fail" + else: + logger.info("No incrementals after full1 — skipping incremental-only restore check") + + # Restore to full backup 2 and verify + col_full2 = f"restore_full2_{int(time.time())}" + ts_full2 = self.extract_ts(snap_full2) + # import all snapshots up to full2 + self.import_exported_up_to_timestamp(col_full2, ts_full2, export_dir, full_orders, full_products) + self._remove_tables([full_orders, full_products]) + rest_full2 = self._execute_yql(f"RESTORE `{col_full2}`;") + assert rest_full2.exit_code == 0, f"RESTORE full2 failed: {rest_full2.std_err}" + restored_rows = self._capture_snapshot(t_orders) + assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_full2]), "Verify data in backup (4) failed" + + # Restore to most-relevant incremental after full2 + chosen_inc_after_full2 = None + for cand in (snap_inc3, snap_inc4): + if cand in created_snapshots and created_snapshots.index(cand) > created_snapshots.index(snap_full2): + chosen_inc_after_full2 = cand + break + + if chosen_inc_after_full2: + col_post_full2 = f"restore_postfull2_{int(time.time())}" + idx_chosen = created_snapshots.index(chosen_inc_after_full2) + snaps_for_post = created_snapshots[: idx_chosen + 1] + # import required snapshots + create_sql = f""" + CREATE BACKUP COLLECTION `{col_post_full2}` + ( TABLE `{full_orders}`, TABLE `{full_products}` ) + WITH ( STORAGE = 'cluster' ); + """ + res = self._execute_yql(create_sql) + assert res.exit_code == 0 + self.wait_for_collection(col_post_full2, timeout_s=30) + for s in snaps_for_post: + src = os.path.join(export_dir, s) + dest_path = f"/Root/.backups/collections/{col_post_full2}/{s}" + r = yatest.common.execute( + [ + backup_bin(), + "--verbose", + "--endpoint", + "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", + self.root_dir, + "tools", + "restore", + "--path", + dest_path, + "--input", + src, + ], + check_exit_code=False, + ) + assert r.exit_code == 0, f"tools restore import failed for {s}: {r.std_err}" + # drop and restore + self._remove_tables([full_orders, full_products]) + rest_post = self._execute_yql(f"RESTORE `{col_post_full2}`;") + assert rest_post.exit_code == 0, f"RESTORE post-full2 failed: {rest_post.std_err}" + restored_rows = self._capture_snapshot(t_orders) + expected_rows = snapshot_rows[chosen_inc_after_full2] + assert self.normalize_rows(restored_rows) == self.normalize_rows(expected_rows), "Verify data in backup (5) failed" + + # cleanup + if os.path.exists(export_dir): + shutil.rmtree(export_dir) +<<<<<<< HEAD +>>>>>>> c4120382b23 (add Full cycle local backup restore) +>>>>>>> 958a7211813 (add Full cycle local backup restore) +======= +>>>>>>> d2ba25d3a53 (Few changes) From 6357d1bdfb185ac172b39e5f209dc505e67f2aaf Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Sat, 11 Oct 2025 13:11:57 +0000 Subject: [PATCH 03/18] add full cycle local backup restore with incrementals --- .../backup_collection/basic_user_scenarios.py | 500 +----------------- 1 file changed, 1 insertion(+), 499 deletions(-) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index f36d1acf82b7..5554e574e810 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -14,11 +14,6 @@ from ydb.tests.oss.ydb_sdk_import import ydb from contextlib import contextmanager -<<<<<<< HEAD ->>>>>>> 958a7211813 (add Full cycle local backup restore) -======= - ->>>>>>> d2ba25d3a53 (Few changes) logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -187,32 +182,6 @@ def scheme_listdir(self, path): return [child.name for child in self.driver.scheme_client.list_directory(path).children if not is_system_object(child)] -<<<<<<< HEAD -======= - def create_user(self, user, password="password"): - cmd = [ - backup_bin(), - "--verbose", - "--endpoint", - "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", - self.root_dir, - "yql", - "--script", - f"CREATE USER {user} PASSWORD '{password}'", - ] - yatest.common.execute(cmd) - - def create_users(self): - self.create_user("alice") - self.create_user("bob") - self.create_user("eve") - -<<<<<<< HEAD -<<<<<<< HEAD ->>>>>>> 958a7211813 (add Full cycle local backup restore) -======= ->>>>>>> d2ba25d3a53 (Few changes) def collection_scheme_path(self, collection_name: str) -> str: return os.path.join(self.root_dir, ".backups", "collections", collection_name) @@ -571,8 +540,6 @@ def test_full_cycle_local_backup_restore(self): # Restore and verify self._verify_restore(export_info, snapshot1, snapshot2) -<<<<<<< HEAD -<<<<<<< HEAD class TestFullCycleLocalBackupRestoreWIncr(TestFullCycleLocalBackupRestore): @@ -973,469 +940,4 @@ def record_last_snapshot(): # cleanup if os.path.exists(export_dir): - shutil.rmtree(export_dir) -======= -======= - "--database", "/Root", "tools", "restore", "-p", f".backups/collections/{collection_restore_v2}", "-i", bdir_v2], - check_exit_code=False, - ) - assert r2.exit_code == 0, f"tools restore import v2 failed for {bdir_v2}" -======= ->>>>>>> d2ba25d3a53 (Few changes) - - -class TestFullCycleLocalBackupRestoreWIncr(BaseTestBackupInFiles): - def _capture_snapshot(self, table): - with self.session_scope() as session: - return sdk_select_table_rows(session, table) - - def _export_backups(self, collection_src): - export_dir = output_path(self.test_name, collection_src) - if os.path.exists(export_dir): - shutil.rmtree(export_dir) - os.makedirs(export_dir, exist_ok=True) - - dump_cmd = [ - backup_bin(), - "--verbose", - "--endpoint", - "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", - self.root_dir, - "tools", - "dump", - "--path", - f"/Root/.backups/collections/{collection_src}", - "--output", - export_dir, - ] - dump_res = yatest.common.execute(dump_cmd, check_exit_code=False) - if dump_res.exit_code != 0: - raise AssertionError(f"tools dump failed: {dump_res.std_err}") - - exported_items = sorted([name for name in os.listdir(export_dir) - if os.path.isdir(os.path.join(export_dir, name))]) - assert len(exported_items) >= 2, f"Expected at least 2 exported backups, got: {exported_items}" - - return export_dir, exported_items - - def _execute_yql(self, script, verbose=False): - cmd = [backup_bin()] - if verbose: - cmd.append("--verbose") - cmd += [ - "--endpoint", - f"grpc://localhost:{self.cluster.nodes[1].grpc_port}", - "--database", - self.root_dir, - "yql", - "--script", - script, - ] - return yatest.common.execute(cmd, check_exit_code=False) - - def _setup_test_collections(self): - collection_src = f"coll_src_{int(time.time())}" - t1 = "orders" - t2 = "products" - - with self.session_scope() as session: - create_table_with_data(session, t1) - create_table_with_data(session, t2) - - return collection_src, t1, t2 - - def _modify_data_add_and_remove(self, add_rows: List[tuple] = None, remove_ids: List[int] = None): - add_rows = add_rows or [] - remove_ids = remove_ids or [] - - with self.session_scope() as session: - # Adds - if add_rows: - values = ", ".join(f"({i},{n},\"{t}\")" for i, n, t in add_rows) - session.transaction().execute( - f'PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES {values};', - commit_tx=True, - ) - # Removes - for rid in remove_ids: - session.transaction().execute( - f'PRAGMA TablePathPrefix("/Root"); DELETE FROM orders WHERE id = {rid};', - commit_tx=True, - ) - - def _add_more_tables(self, prefix: str, count: int = 1): - created = [] - for i in range(1, count + 1): - name = f"{prefix}_{i}_{int(time.time()) % 10000}" - with self.session_scope() as session: - create_table_with_data(session, name) - created.append(f"/Root/{name}") - return created - - def _remove_tables(self, table_paths: List[str]): - with self.session_scope() as session: - for tp in table_paths: - try: - session.execute_scheme(f"DROP TABLE `{tp}`;") - except Exception: - logger.debug(f"Failed to drop table {tp} (maybe not exists)") - - # extract timestamp prefix from exported folder name - name_re = re.compile(r"^([0-9]{8}T[0-9]{6}Z?)_(full|incremental)") - - def extract_ts(self, name): - m = self.name_re.match(name) - if m: - return m.group(1) - return name.split("_", 1)[0] - - def import_exported_up_to_timestamp(self, target_collection, target_ts, export_dir, full_orders, full_products, timeout_s=60): - create_sql = f""" - CREATE BACKUP COLLECTION `{target_collection}` - ( TABLE `{full_orders}`, TABLE `{full_products}` ) - WITH ( STORAGE = 'cluster' ); - """ - res = self._execute_yql(create_sql) - assert res.exit_code == 0, f"CREATE {target_collection} failed: {getattr(res, 'std_err', None)}" - self.wait_for_collection(target_collection, timeout_s=30) - - all_dirs = sorted([d for d in os.listdir(export_dir) if os.path.isdir(os.path.join(export_dir, d))]) - chosen = [d for d in all_dirs if self.extract_ts(d) <= target_ts] - assert chosen, f"No exported snapshots with ts <= {target_ts} found in {export_dir}: {all_dirs}" - - logger.info(f"Will import into {target_collection} these snapshots (in order): {chosen}") - - for name in chosen: - src = os.path.join(export_dir, name) - dest_path = f"/Root/.backups/collections/{target_collection}/{name}" - logger.info(f"Importing {name} (ts={self.extract_ts(name)}) -> {dest_path}") - r = yatest.common.execute( - [ - backup_bin(), - "--verbose", - "--endpoint", - "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", - self.root_dir, - "tools", - "restore", - "--path", - dest_path, - "--input", - src, - ], - check_exit_code=False, - ) - out = (r.std_out or b"").decode("utf-8", "ignore") - err = (r.std_err or b"").decode("utf-8", "ignore") - if r.exit_code != 0: - logger.error(f"tools restore import failed for {name}: exit={r.exit_code} stdout={out} stderr={err}") - assert r.exit_code == 0, f"tools restore import failed for {name}: stdout={out} stderr={err}" - - # --- WAIT until imported snapshots are visible in scheme (registered) --- - deadline = time.time() + timeout_s - expected = set(chosen) - while time.time() < deadline: - try: - kids = set(self.get_collection_children(target_collection)) - if expected.issubset(kids): - logger.info(f"All imported snapshots are registered in collection {target_collection}") - break - else: - missing = expected - kids - logger.info(f"Waiting for registered snapshots in {target_collection}, missing: {missing}") - except Exception as e: - logger.debug(f"While waiting for imported snapshots: {e}") - time.sleep(5) - else: - # timeout - try: - kids = sorted(self.get_collection_children(target_collection)) - except Exception: - kids = "" - raise AssertionError(f"Imported snapshots did not appear in collection {target_collection} within {timeout_s}s. Expected: {sorted(chosen)}. Present: {kids}") - - # small safety pause to ensure system is stable before RESTORE - time.sleep(5) - - def normalize_rows(self, rows): - header = rows[0] - body = rows[1:] - - def norm_val(v): - return v.decode() if isinstance(v, (bytes, bytearray)) else str(v) - sorted_body = sorted([tuple(norm_val(x) for x in r) for r in body]) - return (tuple(header), tuple(sorted_body)) - - def test_full_cycle_local_backup_restore_with_incrementals(self): - # setup - collection_src, t_orders, t_products = self._setup_test_collections() - full_orders = f"/Root/{t_orders}" - full_products = f"/Root/{t_products}" - - # Keep created snapshot names - created_snapshots: List[str] = [] - snapshot_rows = {} # name -> captured rows - - # Create collection with incremental enabled - create_collection_sql = f""" - CREATE BACKUP COLLECTION `{collection_src}` - ( TABLE `{full_orders}`, TABLE `{full_products}` ) - WITH ( STORAGE = 'cluster', INCREMENTAL_BACKUP_ENABLED = 'true' ); - """ - create_res = self._execute_yql(create_collection_sql) - assert create_res.exit_code == 0, "CREATE BACKUP COLLECTION failed" - self.wait_for_collection(collection_src, timeout_s=30) - - def record_last_snapshot(): - kids = sorted(self.get_collection_children(collection_src)) - assert kids, "No snapshots found after backup" - last = kids[-1] - created_snapshots.append(last) - return last - - # Add/remove data - self._modify_data_add_and_remove(add_rows=[(10, 1000, "a1")], remove_ids=[2]) - - # Add more tables (1) - extras = [] - extras += self._add_more_tables("extra1", 1) - - # Create full backup 1 - time.sleep(1.1) - res = self._execute_yql(f"BACKUP `{collection_src}`;") - assert res.exit_code == 0, f"FULL BACKUP 1 failed: {getattr(res, 'std_err', None)}" - self.wait_for_collection_has_snapshot(collection_src, timeout_s=30) - snap_full1 = record_last_snapshot() - snapshot_rows[snap_full1] = self._capture_snapshot(t_orders) - - # Add/remove data - self._modify_data_add_and_remove(add_rows=[(20, 2000, "b1")], remove_ids=[1]) - - # Add more tables, remove some tables from step 4 - extras += self._add_more_tables("extra2", 1) - if extras: - self._remove_tables([extras[0]]) - - # Create incremental backup 1 - time.sleep(1.1) - res = self._execute_yql(f"BACKUP `{collection_src}` INCREMENTAL;") - assert res.exit_code == 0, "INCREMENTAL 1 failed" - snap_inc1 = record_last_snapshot() - snapshot_rows[snap_inc1] = self._capture_snapshot(t_orders) - - # Add/remove - self._modify_data_add_and_remove(add_rows=[(30, 3000, "c1")], remove_ids=[10]) - - # Add more tables, Remove some tables from step 5 - extras += self._add_more_tables("extra3", 1) - if len(extras) >= 2: - self._remove_tables([extras[1]]) - - # Create incremental backup 2 - time.sleep(1.1) - res = self._execute_yql(f"BACKUP `{collection_src}` INCREMENTAL;") - assert res.exit_code == 0, "INCREMENTAL 2 failed" - snap_inc2 = record_last_snapshot() - snapshot_rows[snap_inc2] = self._capture_snapshot(t_orders) - - # Add more tables, remove some tables from step 5 - extras += self._add_more_tables("extra4", 1) - if len(extras) >= 3: - self._remove_tables([extras[2]]) - - # Add/remove - self._modify_data_add_and_remove(add_rows=[(40, 4000, "d1")], remove_ids=[20]) - - # Create full backup 2 - time.sleep(1.1) - res = self._execute_yql(f"BACKUP `{collection_src}`;") - assert res.exit_code == 0, "FULL BACKUP 2 failed" - snap_full2 = record_last_snapshot() - snapshot_rows[snap_full2] = self._capture_snapshot(t_orders) - - # Create incremental backup 3 - time.sleep(1.1) - res = self._execute_yql(f"BACKUP `{collection_src}` INCREMENTAL;") - assert res.exit_code == 0, "INCREMENTAL 3 failed" - snap_inc3 = record_last_snapshot() - snapshot_rows[snap_inc3] = self._capture_snapshot(t_orders) - - # Add more tables, remove some tables from step 5 - extras += self._add_more_tables("extra5", 1) - if len(extras) >= 4: - self._remove_tables([extras[3]]) - - # Add/remove - self._modify_data_add_and_remove(add_rows=[(50, 5000, "e1")], remove_ids=[30]) - - # Create incremental backup 4 - time.sleep(1.1) - res = self._execute_yql(f"BACKUP `{collection_src}` INCREMENTAL;") - assert res.exit_code == 0, "INCREMENTAL 4 failed" - snap_inc4 = record_last_snapshot() - snapshot_rows[snap_inc4] = self._capture_snapshot(t_orders) - - # Export backups - export_dir, exported_items = self._export_backups(collection_src) - exported_dirs = sorted([d for d in os.listdir(export_dir) if os.path.isdir(os.path.join(export_dir, d))]) - - # all recorded snapshots should be exported - for s in created_snapshots: - assert s in exported_dirs, f"Recorded snapshot {s} not in exported dirs {exported_dirs}" - - # Try to restore and get error that tables already exist - restore_all_col = f"restore_all_{int(time.time())}" - # import all snapshots up to the latest snapshot - latest_ts = self.extract_ts(created_snapshots[-1]) - self.import_exported_up_to_timestamp(restore_all_col, latest_ts, export_dir, full_orders, full_products) - rest_all = self._execute_yql(f"RESTORE `{restore_all_col}`;") - assert rest_all.exit_code != 0, "Expected RESTORE to fail when tables already exist" - - # Remove all tables - self._remove_tables([full_orders, full_products] + extras) - - # Restore to full backup 1 - col_full1 = f"restore_full1_{int(time.time())}" - ts_full1 = self.extract_ts(snap_full1) - self.import_exported_up_to_timestamp(col_full1, ts_full1, export_dir, full_orders, full_products) - rest_full1 = self._execute_yql(f"RESTORE `{col_full1}`;") - assert rest_full1.exit_code == 0, f"RESTORE full1 failed: {rest_full1.std_err}" - restored_rows = self._capture_snapshot(t_orders) - assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_full1]), "Verify data in backup (1) failed" - - # Restore to incremental 1 (full1 + inc1) - col_inc1 = f"restore_inc1_{int(time.time())}" - ts_inc1 = self.extract_ts(snap_inc1) - self.import_exported_up_to_timestamp(col_inc1, ts_inc1, export_dir, full_orders, full_products) - # ensure target tables absent - self._remove_tables([full_orders, full_products]) - rest_inc1 = self._execute_yql(f"RESTORE `{col_inc1}`;") - assert rest_inc1.exit_code == 0, f"RESTORE inc1 failed: {rest_inc1.std_err}" - restored_rows = self._capture_snapshot(t_orders) - assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_inc1]), "Verify data in backup (2) failed" - - # Restore to incremental 2 (full1 + inc1 + inc2) - col_inc2 = f"restore_inc2_{int(time.time())}" - ts_inc2 = self.extract_ts(snap_inc2) - self.import_exported_up_to_timestamp(col_inc2, ts_inc2, export_dir, full_orders, full_products) - self._remove_tables([full_orders, full_products]) - rest_inc2 = self._execute_yql(f"RESTORE `{col_inc2}`;") - time.sleep(1.1) - assert rest_inc2.exit_code == 0, f"RESTORE inc2 failed: {rest_inc2.std_err}" - restored_rows = self._capture_snapshot(t_orders) - assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_inc2]), "Verify data in backup (3) failed" - - # Remove all tables (2) - self._remove_tables([full_orders, full_products] + extras) - - # Try to restore incremental-only (no base full) -> expect fail - inc_only_col = f"inc_only_{int(time.time())}" - # pick incrementals strictly after full1 - idx_full1 = created_snapshots.index(snap_full1) - incs_after_full1 = [s for s in created_snapshots if "_incremental" in s and created_snapshots.index(s) > idx_full1] - if incs_after_full1: - # import incrementals only (should fail on restore) - create_sql = f""" - CREATE BACKUP COLLECTION `{inc_only_col}` - ( TABLE `{full_orders}`, TABLE `{full_products}` ) - WITH ( STORAGE = 'cluster' ); - """ - res = self._execute_yql(create_sql) - assert res.exit_code == 0 - self.wait_for_collection(inc_only_col, timeout_s=30) - for s in incs_after_full1: - src = os.path.join(export_dir, s) - dest_path = f"/Root/.backups/collections/{inc_only_col}/{s}" - r = yatest.common.execute( - [ - backup_bin(), - "--verbose", - "--endpoint", - "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", - self.root_dir, - "tools", - "restore", - "--path", - dest_path, - "--input", - src, - ], - check_exit_code=False, - ) - assert r.exit_code == 0, f"tools restore import for inc-only failed: {r.std_err}" - rest_inc_only = self._execute_yql(f"RESTORE `{inc_only_col}`;") - assert rest_inc_only.exit_code != 0, "Expected restore of incrementals-only (no base) to fail" - else: - logger.info("No incrementals after full1 — skipping incremental-only restore check") - - # Restore to full backup 2 and verify - col_full2 = f"restore_full2_{int(time.time())}" - ts_full2 = self.extract_ts(snap_full2) - # import all snapshots up to full2 - self.import_exported_up_to_timestamp(col_full2, ts_full2, export_dir, full_orders, full_products) - self._remove_tables([full_orders, full_products]) - rest_full2 = self._execute_yql(f"RESTORE `{col_full2}`;") - assert rest_full2.exit_code == 0, f"RESTORE full2 failed: {rest_full2.std_err}" - restored_rows = self._capture_snapshot(t_orders) - assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_full2]), "Verify data in backup (4) failed" - - # Restore to most-relevant incremental after full2 - chosen_inc_after_full2 = None - for cand in (snap_inc3, snap_inc4): - if cand in created_snapshots and created_snapshots.index(cand) > created_snapshots.index(snap_full2): - chosen_inc_after_full2 = cand - break - - if chosen_inc_after_full2: - col_post_full2 = f"restore_postfull2_{int(time.time())}" - idx_chosen = created_snapshots.index(chosen_inc_after_full2) - snaps_for_post = created_snapshots[: idx_chosen + 1] - # import required snapshots - create_sql = f""" - CREATE BACKUP COLLECTION `{col_post_full2}` - ( TABLE `{full_orders}`, TABLE `{full_products}` ) - WITH ( STORAGE = 'cluster' ); - """ - res = self._execute_yql(create_sql) - assert res.exit_code == 0 - self.wait_for_collection(col_post_full2, timeout_s=30) - for s in snaps_for_post: - src = os.path.join(export_dir, s) - dest_path = f"/Root/.backups/collections/{col_post_full2}/{s}" - r = yatest.common.execute( - [ - backup_bin(), - "--verbose", - "--endpoint", - "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", - self.root_dir, - "tools", - "restore", - "--path", - dest_path, - "--input", - src, - ], - check_exit_code=False, - ) - assert r.exit_code == 0, f"tools restore import failed for {s}: {r.std_err}" - # drop and restore - self._remove_tables([full_orders, full_products]) - rest_post = self._execute_yql(f"RESTORE `{col_post_full2}`;") - assert rest_post.exit_code == 0, f"RESTORE post-full2 failed: {rest_post.std_err}" - restored_rows = self._capture_snapshot(t_orders) - expected_rows = snapshot_rows[chosen_inc_after_full2] - assert self.normalize_rows(restored_rows) == self.normalize_rows(expected_rows), "Verify data in backup (5) failed" - - # cleanup - if os.path.exists(export_dir): - shutil.rmtree(export_dir) -<<<<<<< HEAD ->>>>>>> c4120382b23 (add Full cycle local backup restore) ->>>>>>> 958a7211813 (add Full cycle local backup restore) -======= ->>>>>>> d2ba25d3a53 (Few changes) + shutil.rmtree(export_dir) \ No newline at end of file From 6a45612adf73217073428c1c5ab716c0ade5ae5c Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Sat, 11 Oct 2025 18:18:16 +0000 Subject: [PATCH 04/18] changed time --- .../functional/backup_collection/basic_user_scenarios.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index 5554e574e810..e036252fc393 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -808,14 +808,18 @@ def record_last_snapshot(): col_inc1 = f"restore_inc1_{int(time.time())}" time.sleep(1.1) ts_inc1 = self.extract_ts(snap_inc1) + time.sleep(1.1) self.import_exported_up_to_timestamp(col_inc1, ts_inc1, export_dir, full_orders, full_products) + time.sleep(1.1) # ensure target tables absent self._remove_tables([full_orders, full_products]) time.sleep(1.1) rest_inc1 = self._execute_yql(f"RESTORE `{col_inc1}`;") time.sleep(1.1) assert rest_inc1.exit_code == 0, f"RESTORE inc1 failed: {rest_inc1.std_err}" + time.sleep(1.1) restored_rows = self._capture_snapshot(t_orders) + time.sleep(1.1) assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_inc1]), "Verify data in backup (2) failed" time.sleep(1.1) From 94865204b1e637230d316a49eb2507d9e48f87a8 Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Sat, 11 Oct 2025 20:58:02 +0000 Subject: [PATCH 05/18] changed time2 --- .../functional/backup_collection/basic_user_scenarios.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index e036252fc393..2cd89c863947 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -808,16 +808,11 @@ def record_last_snapshot(): col_inc1 = f"restore_inc1_{int(time.time())}" time.sleep(1.1) ts_inc1 = self.extract_ts(snap_inc1) - time.sleep(1.1) self.import_exported_up_to_timestamp(col_inc1, ts_inc1, export_dir, full_orders, full_products) - time.sleep(1.1) # ensure target tables absent self._remove_tables([full_orders, full_products]) - time.sleep(1.1) rest_inc1 = self._execute_yql(f"RESTORE `{col_inc1}`;") - time.sleep(1.1) assert rest_inc1.exit_code == 0, f"RESTORE inc1 failed: {rest_inc1.std_err}" - time.sleep(1.1) restored_rows = self._capture_snapshot(t_orders) time.sleep(1.1) assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_inc1]), "Verify data in backup (2) failed" @@ -828,11 +823,8 @@ def record_last_snapshot(): col_inc2 = f"restore_inc2_{int(time.time())}" ts_inc2 = self.extract_ts(snap_inc2) self.import_exported_up_to_timestamp(col_inc2, ts_inc2, export_dir, full_orders, full_products) - time.sleep(1.1) self._remove_tables([full_orders, full_products]) - time.sleep(1.1) rest_inc2 = self._execute_yql(f"RESTORE `{col_inc2}`;") - time.sleep(1.1) assert rest_inc2.exit_code == 0, f"RESTORE inc2 failed: {rest_inc2.std_err}" restored_rows = self._capture_snapshot(t_orders) time.sleep(1.1) From 0d07f9341d579be2afea706a67b07ae68c8eb6ef Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Sat, 11 Oct 2025 21:18:49 +0000 Subject: [PATCH 06/18] changed time2 --- ydb/tests/functional/backup_collection/basic_user_scenarios.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index 2cd89c863947..fe1312d2ddc0 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -812,9 +812,9 @@ def record_last_snapshot(): # ensure target tables absent self._remove_tables([full_orders, full_products]) rest_inc1 = self._execute_yql(f"RESTORE `{col_inc1}`;") + time.sleep(1.1) assert rest_inc1.exit_code == 0, f"RESTORE inc1 failed: {rest_inc1.std_err}" restored_rows = self._capture_snapshot(t_orders) - time.sleep(1.1) assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_inc1]), "Verify data in backup (2) failed" time.sleep(1.1) @@ -827,7 +827,6 @@ def record_last_snapshot(): rest_inc2 = self._execute_yql(f"RESTORE `{col_inc2}`;") assert rest_inc2.exit_code == 0, f"RESTORE inc2 failed: {rest_inc2.std_err}" restored_rows = self._capture_snapshot(t_orders) - time.sleep(1.1) assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_inc2]), "Verify data in backup (3) failed" # Remove all tables (2) From dd985185ede553993767e1617ab0538995db1d3a Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Sat, 11 Oct 2025 21:35:58 +0000 Subject: [PATCH 07/18] changed time3 --- ydb/tests/functional/backup_collection/basic_user_scenarios.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index fe1312d2ddc0..cee641135f33 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -811,6 +811,7 @@ def record_last_snapshot(): self.import_exported_up_to_timestamp(col_inc1, ts_inc1, export_dir, full_orders, full_products) # ensure target tables absent self._remove_tables([full_orders, full_products]) + time.sleep(1.1) rest_inc1 = self._execute_yql(f"RESTORE `{col_inc1}`;") time.sleep(1.1) assert rest_inc1.exit_code == 0, f"RESTORE inc1 failed: {rest_inc1.std_err}" From 1f2638d88a0efe7afa9c85854b4d10d998e9fba5 Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Sat, 11 Oct 2025 21:51:46 +0000 Subject: [PATCH 08/18] changed time4 --- ydb/tests/functional/backup_collection/basic_user_scenarios.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index cee641135f33..2ae3187f01e5 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -828,6 +828,7 @@ def record_last_snapshot(): rest_inc2 = self._execute_yql(f"RESTORE `{col_inc2}`;") assert rest_inc2.exit_code == 0, f"RESTORE inc2 failed: {rest_inc2.std_err}" restored_rows = self._capture_snapshot(t_orders) + time.sleep(1.1) assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_inc2]), "Verify data in backup (3) failed" # Remove all tables (2) From f1935d288e1059b5f1c0a2d331c02c2993db7a22 Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Sat, 11 Oct 2025 22:09:41 +0000 Subject: [PATCH 09/18] changed time5 --- ydb/tests/functional/backup_collection/basic_user_scenarios.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index 2ae3187f01e5..5554e574e810 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -824,8 +824,11 @@ def record_last_snapshot(): col_inc2 = f"restore_inc2_{int(time.time())}" ts_inc2 = self.extract_ts(snap_inc2) self.import_exported_up_to_timestamp(col_inc2, ts_inc2, export_dir, full_orders, full_products) + time.sleep(1.1) self._remove_tables([full_orders, full_products]) + time.sleep(1.1) rest_inc2 = self._execute_yql(f"RESTORE `{col_inc2}`;") + time.sleep(1.1) assert rest_inc2.exit_code == 0, f"RESTORE inc2 failed: {rest_inc2.std_err}" restored_rows = self._capture_snapshot(t_orders) time.sleep(1.1) From 1cbda674a4dd8567ab98d907d672f0eabe493f92 Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Mon, 13 Oct 2025 12:34:43 +0000 Subject: [PATCH 10/18] add pollin-function --- .../backup_collection/basic_user_scenarios.py | 50 +++++++++++++------ 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index 5554e574e810..3495be59d49b 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -324,6 +324,36 @@ def assert_collection_contains_tables(self, collection_name: str, expected_table except Exception: pass + def wait_for_table_rows(self, + table: str, + expected_rows, + timeout_s: int = 60, + poll_interval: float = 0.5): + # helper to get current rows safely + deadline = time.time() + timeout_s + last_exc = None + + while time.time() < deadline: + try: + cur_rows = None + try: + cur_rows = self._capture_snapshot(table) + except Exception as e: + # table might not exist yet or select could fail — record and retry + last_exc = e + time.sleep(poll_interval) + continue + + if cur_rows == expected_rows: + return cur_rows + + except Exception as e: + last_exc = e + + time.sleep(poll_interval) + + raise AssertionError(f"Timeout waiting for table '{table}' rows to match expected (timeout {timeout_s}s). Last error: {last_exc}") + class TestFullCycleLocalBackupRestore(BaseTestBackupInFiles): def _execute_yql(self, script, verbose=False): @@ -800,22 +830,18 @@ def record_last_snapshot(): self.import_exported_up_to_timestamp(col_full1, ts_full1, export_dir, full_orders, full_products) rest_full1 = self._execute_yql(f"RESTORE `{col_full1}`;") assert rest_full1.exit_code == 0, f"RESTORE full1 failed: {rest_full1.std_err}" - restored_rows = self._capture_snapshot(t_orders) - time.sleep(1.1) + restored_rows = self.wait_for_table_rows(t_orders, snapshot_rows[snap_full1], timeout_s=90) assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_full1]), "Verify data in backup (1) failed" # Restore to incremental 1 (full1 + inc1) col_inc1 = f"restore_inc1_{int(time.time())}" - time.sleep(1.1) ts_inc1 = self.extract_ts(snap_inc1) self.import_exported_up_to_timestamp(col_inc1, ts_inc1, export_dir, full_orders, full_products) # ensure target tables absent self._remove_tables([full_orders, full_products]) - time.sleep(1.1) rest_inc1 = self._execute_yql(f"RESTORE `{col_inc1}`;") - time.sleep(1.1) assert rest_inc1.exit_code == 0, f"RESTORE inc1 failed: {rest_inc1.std_err}" - restored_rows = self._capture_snapshot(t_orders) + restored_rows = self.wait_for_table_rows(t_orders, snapshot_rows[snap_inc1], timeout_s=90) assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_inc1]), "Verify data in backup (2) failed" time.sleep(1.1) @@ -824,14 +850,10 @@ def record_last_snapshot(): col_inc2 = f"restore_inc2_{int(time.time())}" ts_inc2 = self.extract_ts(snap_inc2) self.import_exported_up_to_timestamp(col_inc2, ts_inc2, export_dir, full_orders, full_products) - time.sleep(1.1) self._remove_tables([full_orders, full_products]) - time.sleep(1.1) rest_inc2 = self._execute_yql(f"RESTORE `{col_inc2}`;") - time.sleep(1.1) assert rest_inc2.exit_code == 0, f"RESTORE inc2 failed: {rest_inc2.std_err}" - restored_rows = self._capture_snapshot(t_orders) - time.sleep(1.1) + restored_rows = self.wait_for_table_rows(t_orders, snapshot_rows[snap_inc2], timeout_s=90) assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_inc2]), "Verify data in backup (3) failed" # Remove all tables (2) @@ -887,6 +909,7 @@ def record_last_snapshot(): rest_full2 = self._execute_yql(f"RESTORE `{col_full2}`;") assert rest_full2.exit_code == 0, f"RESTORE full2 failed: {rest_full2.std_err}" restored_rows = self._capture_snapshot(t_orders) + restored_rows = self.wait_for_table_rows(t_orders, snapshot_rows[snap_full2], timeout_s=90) assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_full2]), "Verify data in backup (4) failed" # Restore to most-relevant incremental after full2 @@ -934,9 +957,8 @@ def record_last_snapshot(): self._remove_tables([full_orders, full_products]) rest_post = self._execute_yql(f"RESTORE `{col_post_full2}`;") assert rest_post.exit_code == 0, f"RESTORE post-full2 failed: {rest_post.std_err}" - restored_rows = self._capture_snapshot(t_orders) - expected_rows = snapshot_rows[chosen_inc_after_full2] - assert self.normalize_rows(restored_rows) == self.normalize_rows(expected_rows), "Verify data in backup (5) failed" + restored_rows = self.wait_for_table_rows(t_orders, snapshot_rows[chosen_inc_after_full2], timeout_s=90) + assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[chosen_inc_after_full2]), "Verify data in backup (5) failed" # cleanup if os.path.exists(export_dir): From 9078ea76c35bb48693b3754b654d1df3647414ae Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Mon, 13 Oct 2025 12:53:16 +0000 Subject: [PATCH 11/18] refresh --- ydb/tests/functional/backup_collection/basic_user_scenarios.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index 3495be59d49b..89e3b1ba4f8f 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -844,8 +844,6 @@ def record_last_snapshot(): restored_rows = self.wait_for_table_rows(t_orders, snapshot_rows[snap_inc1], timeout_s=90) assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_inc1]), "Verify data in backup (2) failed" - time.sleep(1.1) - # Restore to incremental 2 (full1 + inc1 + inc2) col_inc2 = f"restore_inc2_{int(time.time())}" ts_inc2 = self.extract_ts(snap_inc2) @@ -908,7 +906,6 @@ def record_last_snapshot(): self._remove_tables([full_orders, full_products]) rest_full2 = self._execute_yql(f"RESTORE `{col_full2}`;") assert rest_full2.exit_code == 0, f"RESTORE full2 failed: {rest_full2.std_err}" - restored_rows = self._capture_snapshot(t_orders) restored_rows = self.wait_for_table_rows(t_orders, snapshot_rows[snap_full2], timeout_s=90) assert self.normalize_rows(restored_rows) == self.normalize_rows(snapshot_rows[snap_full2]), "Verify data in backup (4) failed" From 66eae93a922cefcc683e7165b4dfe2b888df08ea Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Mon, 13 Oct 2025 13:18:29 +0000 Subject: [PATCH 12/18] schema-change --- .../backup_collection/basic_user_scenarios.py | 365 +++++++++++++++++- 1 file changed, 364 insertions(+), 1 deletion(-) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index 89e3b1ba4f8f..baf5a8c2b734 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -959,4 +959,367 @@ def record_last_snapshot(): # cleanup if os.path.exists(export_dir): - shutil.rmtree(export_dir) \ No newline at end of file + shutil.rmtree(export_dir) + + +class TestFullCycleLocalBackupRestoreWSchemaChange(TestFullCycleLocalBackupRestore): + def _create_backup_collection(self, collection_src, tables: List[str]): + # create backup collection referencing given table full paths + table_entries = ",\n".join([f"TABLE `/Root/{t}`" for t in tables]) + sql = f""" + CREATE BACKUP COLLECTION `{collection_src}` + ( {table_entries} ) + WITH ( STORAGE = 'cluster', INCREMENTAL_BACKUP_ENABLED = 'false' ); + """ + res = self._execute_yql(sql) + stderr_out = "" + if getattr(res, 'std_err', None): + stderr_out += res.std_err.decode('utf-8', errors='ignore') + if getattr(res, 'std_out', None): + stderr_out += res.std_out.decode('utf-8', errors='ignore') + assert res.exit_code == 0, f"CREATE BACKUP COLLECTION failed: {stderr_out}" + self.wait_for_collection(collection_src, timeout_s=30) + + def _backup_now(self, collection_src): + time.sleep(1.1) + res = self._execute_yql(f"BACKUP `{collection_src}`;") + if res.exit_code != 0: + out = (res.std_out or b"").decode('utf-8', 'ignore') + err = (res.std_err or b"").decode('utf-8', 'ignore') + raise AssertionError(f"BACKUP failed: code={res.exit_code} STDOUT: {out} STDERR: {err}") + + def _restore_import(self, export_dir, exported_item, collection_restore): + bdir = os.path.join(export_dir, exported_item) + r = yatest.common.execute( + [backup_bin(), "--verbose", "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", self.root_dir, "tools", "restore", + "--path", f"/Root/.backups/collections/{collection_restore}", + "--input", bdir], + check_exit_code=False, + ) + assert r.exit_code == 0, f"tools restore import failed: {r.std_err}" + + def _verify_restored_table_data(self, table, expected_rows): + # rows = self._capture_snapshot(table) + rows = self.wait_for_table_rows(table, expected_rows, timeout_s=90) + assert rows == expected_rows, f"Restored data for {table} doesn't match expected.\nExpected: {expected_rows}\nGot: {rows}" + + def _capture_schema(self, table_path: str): + desc = self.driver.scheme_client.describe_path(table_path) + cols = self._get_columns_from_scheme_entry(desc, path_hint=table_path) + return cols + + def _capture_acl(self, table_path: str): + # Attempt to capture owner/grants/acl in a readable form. + try: + desc = self.driver.scheme_client.describe_path(table_path) + except Exception: + return None + + acl_info = {} + owner = getattr(desc, "owner", None) + if owner: + acl_info["owner"] = owner + + for cand in ("acl", "grants", "effective_acl", "permission", "permissions"): + if hasattr(desc, cand): + try: + val = getattr(desc, cand) + acl_info[cand] = val + except Exception: + acl_info[cand] = "" + + # Fallback: try SHOW GRANTS via YQL and capture stdout + try: + res = self._execute_yql(f"SHOW GRANTS ON '{table_path}';") + out = (res.std_out or b"").decode('utf-8', 'ignore') + if out: + acl_info["show_grants"] = out + except Exception: + pass + + return acl_info + + # --- Helpers for extended checks (schema and ACL capture) --- + def _get_columns_from_scheme_entry(self, desc, path_hint: str = None): + # Reuse original robust approach: try multiple candidate attributes + try: + table_obj = getattr(desc, "table", None) + if table_obj is not None: + cols = getattr(table_obj, "columns", None) + if cols: + return [c.name for c in cols] + + cols = getattr(desc, "columns", None) + if cols: + try: + return [c.name for c in cols] + except Exception: + return [str(c) for c in cols] + + for attr in ("schema", "entry", "path"): + nested = getattr(desc, attr, None) + if nested is not None: + table_obj = getattr(nested, "table", None) + cols = getattr(table_obj, "columns", None) if table_obj is not None else None + if cols: + return [c.name for c in cols] + except Exception: + pass + + if getattr(desc, "is_table", False) or getattr(desc, "is_row_table", False) or getattr(desc, "is_column_table", False): + if path_hint: + table_path = path_hint + else: + name = getattr(desc, "name", None) + assert name, f"SchemeEntry has no name, can't form path. desc repr: {repr(desc)}" + table_path = name if name.startswith("/Root") else os.path.join(self.root_dir, name) + + try: + tc = getattr(self.driver, "table_client", None) + if tc is not None and hasattr(tc, "describe_table"): + desc_tbl = tc.describe_table(table_path) + cols = getattr(desc_tbl, "columns", None) or getattr(desc_tbl, "Columns", None) + if cols: + try: + return [c.name for c in cols] + except Exception: + return [str(c) for c in cols] + except Exception: + pass + + try: + with self.session_scope() as session: + if hasattr(session, "describe_table"): + desc_tbl = session.describe_table(table_path) + cols = getattr(desc_tbl, "columns", None) or getattr(desc_tbl, "Columns", None) + if cols: + try: + return [c.name for c in cols] + except Exception: + return [str(c) for c in cols] + except Exception: + pass + + diagnostics = ["Failed to find columns via known candidates.\n"] + try: + diagnostics.append("dir(desc):\n" + ", ".join(dir(desc)) + "\n") + except Exception as e: + diagnostics.append(f"dir(desc) raised: {e}\n") + + readable = [] + for attr in sorted(set(dir(desc))): + if attr.startswith("_"): + continue + if len(readable) >= 40: + break + try: + val = getattr(desc, attr) + if callable(val): + continue + s = repr(val) + if len(s) > 300: + s = s[:300] + "...(truncated)" + readable.append(f"{attr} = {s}") + except Exception as e: + readable.append(f"{attr} = ") + + diagnostics.append("Sample attributes (truncated):\n" + "\n".join(readable) + "\n") + + raise AssertionError( + "describe_path returned SchemeEntry in unexpected shape. Cannot locate columns.\n\nDiagnostic dump:\n\n" + + "\n".join(diagnostics) + ) + + def _drop_tables(self, tables: List[str]): + with self.session_scope() as session: + for t in tables: + full = f"/Root/{t}" + try: + session.execute_scheme(f"DROP TABLE `{full}`;") + except Exception: + # ignore failures + pass + + def test_full_cycle_local_backup_restore_with_schema_changes(self): + collection_src, t1, t2 = self._setup_test_collections() + + # Create backup collection (will reference the initial tables) + self._create_backup_collection(collection_src, [t1, t2]) + + # === Step 4: Add/remove data, change ACLs (1), add more tables (1) === + # perform first wave of modifications that must be captured by full backup 1 + with self.session_scope() as session: + # add & remove data + session.transaction().execute('PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES (10, 100, "one-wave");', commit_tx=True) + session.transaction().execute('PRAGMA TablePathPrefix("/Root"); DELETE FROM products WHERE id = 1;', commit_tx=True) + + # change ACLs: try multiple grant syntaxes until success + desc_for_acl = self.driver.scheme_client.describe_path("/Root/orders") + owner_role = getattr(desc_for_acl, "owner", None) or "root@builtin" + + def q(role: str) -> str: + return "`" + role.replace("`", "") + "`" + + role_candidates = [owner_role, "public", "everyone", "root"] + grant_variants = [] + for r in role_candidates: + role_quoted = q(r) + grant_variants.extend([ + f"GRANT ALL ON `/Root/orders` TO {role_quoted};", + f"GRANT SELECT ON `/Root/orders` TO {role_quoted};", + f"GRANT 'ydb.generic.read' ON `/Root/orders` TO {role_quoted};", + ]) + grant_variants.append(f"GRANT ALL ON `/Root/orders` TO {q(owner_role)};") + + acl_applied = False + for cmd in grant_variants: + res = self._execute_yql(cmd) + if res.exit_code == 0: + acl_applied = True + break + assert acl_applied, "Failed to apply any GRANT variant in step (1)" + + # add more tables (1) + create_table_with_data(session, "extra_table_1") + + # capture state after wave 1 + snapshot_wave1_t1 = self._capture_snapshot(t1) + snapshot_wave1_t2 = self._capture_snapshot(t2) + schema_wave1_t1 = self._capture_schema(f"/Root/{t1}") + schema_wave1_t2 = self._capture_schema(f"/Root/{t2}") + acl_wave1_t1 = self._capture_acl(f"/Root/{t1}") + acl_wave1_t2 = self._capture_acl(f"/Root/{t2}") + + # === Create full backup 1 === + self._backup_now(collection_src) + self.wait_for_collection_has_snapshot(collection_src, timeout_s=30) + + # === Step 7..12: modifications (2) include add/remove data, add more tables (2), remove some tables from step5, + # add/alter/drop columns, change ACLs === + with self.session_scope() as session: + # data modifications + session.transaction().execute('PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES (11, 111, "two-wave");', commit_tx=True) + session.transaction().execute('PRAGMA TablePathPrefix("/Root"); DELETE FROM orders WHERE id = 2;', commit_tx=True) + + # add more tables (2) + create_table_with_data(session, "extra_table_2") + + # remove some tables from step5: drop extra_table_1 + try: + session.execute_scheme('DROP TABLE `/Root/extra_table_1`;') + except Exception: + pass + + # add columns to initial tables + try: + session.execute_scheme('ALTER TABLE `/Root/orders` ADD COLUMN new_col Uint32;') + except Exception: + # if not supported, ignore but record + logger.info('ALTER TABLE ADD COLUMN not supported in this build') + + # attempt alter column (if supported) - many builds do not support complex ALTER, so guard with try + try: + session.execute_scheme('ALTER TABLE `/Root/orders` ALTER COLUMN number SET NOT NULL;') + except Exception: + logger.info('ALTER COLUMN ... SET NOT NULL not supported, skipping') + + # drop columns + try: + session.execute_scheme('ALTER TABLE `/Root/orders` DROP COLUMN new_col;') + except Exception: + logger.info('DROP COLUMN new_col not supported or already dropped') + + # change ACLs again for initial tables + desc_for_acl2 = self.driver.scheme_client.describe_path("/Root/orders") + owner_role2 = getattr(desc_for_acl2, "owner", None) or "root@builtin" + owner_quoted = owner_role2.replace('`', '') + cmd = f"GRANT SELECT ON `/Root/orders` TO `{owner_quoted}`;" + res = self._execute_yql(cmd) + assert res.exit_code == 0, "Failed to apply GRANT in wave 2" + + # capture state after wave 2 + snapshot_wave2_t1 = self._capture_snapshot(t1) + snapshot_wave2_t2 = self._capture_snapshot(t2) + schema_wave2_t1 = self._capture_schema(f"/Root/{t1}") + schema_wave2_t2 = self._capture_schema(f"/Root/{t2}") + acl_wave2_t1 = self._capture_acl(f"/Root/{t1}") + acl_wave2_t2 = self._capture_acl(f"/Root/{t2}") + + # === Create full backup 2 === + self._backup_now(collection_src) + self.wait_for_collection_has_snapshot(collection_src, timeout_s=30) + + # === Export backups so we can import snapshots into separate collections for restore verification === + export_dir, exported_items = self._export_backups(collection_src) + # expect at least two exported snapshots (backup1 and backup2) + assert len(exported_items) >= 2, "Expected at least 2 exported snapshots for verification" + + # === Attempt to import exported backup into new collection and RESTORE while tables exist => expect fail === + # create restore collections + coll_restore_1 = f"coll_restore_v1_{int(time.time())}" + coll_restore_2 = f"coll_restore_v2_{int(time.time())}" + self._create_backup_collection(coll_restore_1, [t1, t2]) + self._create_backup_collection(coll_restore_2, [t1, t2]) + + # import exported snapshots into restore collections + # imported_items are directories in exported_items; we'll import both + self._restore_import(export_dir, exported_items[0], coll_restore_1) + self._restore_import(export_dir, exported_items[1], coll_restore_2) + + # try RESTORE when tables already exist -> should fail + res_restore_when_exists = self._execute_yql(f"RESTORE `{coll_restore_1}`;") + assert res_restore_when_exists.exit_code != 0, "Expected RESTORE to fail when target tables already exist" + + # === Remove all tables from DB (orders, products, extras) === + self._drop_tables([t1, t2, "extra_table_1", "extra_table_2"]) # ignore errors + + # === Now RESTORE coll_restore_1 (which corresponds to backup1) === + res_restore1 = self._execute_yql(f"RESTORE `{coll_restore_1}`;") + assert res_restore1.exit_code == 0, f"RESTORE v1 failed: {res_restore1.std_err or res_restore1.std_out}" + + # verify schema/data/acl for backup1 + # verify data + self._verify_restored_table_data(t1, snapshot_wave1_t1) + self._verify_restored_table_data(t2, snapshot_wave1_t2) + + # verify schema + restored_schema_t1 = self._capture_schema(f"/Root/{t1}") + restored_schema_t2 = self._capture_schema(f"/Root/{t2}") + assert restored_schema_t1 == schema_wave1_t1, f"Schema for {t1} after restore v1 differs: expected {schema_wave1_t1}, got {restored_schema_t1}" + assert restored_schema_t2 == schema_wave1_t2, f"Schema for {t2} after restore v1 differs: expected {schema_wave1_t2}, got {restored_schema_t2}" + + # verify acl + restored_acl_t1 = self._capture_acl(f"/Root/{t1}") + restored_acl_t2 = self._capture_acl(f"/Root/{t2}") + # We compare that SHOW GRANTS output contains previously stored show_grants if present + if 'show_grants' in (acl_wave1_t1 or {}): + assert 'show_grants' in (restored_acl_t1 or {}) and acl_wave1_t1['show_grants'] in restored_acl_t1['show_grants'] + if 'show_grants' in (acl_wave1_t2 or {}): + assert 'show_grants' in (restored_acl_t2 or {}) and acl_wave1_t2['show_grants'] in restored_acl_t2['show_grants'] + + # === Remove all tables again and restore backup2 === + self._drop_tables([t1, t2, "extra_table_1", "extra_table_2"]) # ignore errors + + res_restore2 = self._execute_yql(f"RESTORE `{coll_restore_2}`;") + assert res_restore2.exit_code == 0, f"RESTORE v2 failed: {res_restore2.std_err or res_restore2.std_out}" + + # verify data/schema/acl for backup2 + self._verify_restored_table_data(t1, snapshot_wave2_t1) + self._verify_restored_table_data(t2, snapshot_wave2_t2) + + restored_schema2_t1 = self._capture_schema(f"/Root/{t1}") + restored_schema2_t2 = self._capture_schema(f"/Root/{t2}") + assert restored_schema2_t1 == schema_wave2_t1, f"Schema for {t1} after restore v2 differs: expected {schema_wave2_t1}, got {restored_schema2_t1}" + assert restored_schema2_t2 == schema_wave2_t2, f"Schema for {t2} after restore v2 differs: expected {schema_wave2_t2}, got {restored_schema2_t2}" + + restored_acl2_t1 = self._capture_acl(f"/Root/{t1}") + restored_acl2_t2 = self._capture_acl(f"/Root/{t2}") + if 'show_grants' in (acl_wave2_t1 or {}): + assert 'show_grants' in (restored_acl2_t1 or {}) and acl_wave2_t1['show_grants'] in restored_acl2_t1['show_grants'] + if 'show_grants' in (acl_wave2_t2 or {}): + assert 'show_grants' in (restored_acl2_t2 or {}) and acl_wave2_t2['show_grants'] in restored_acl2_t2['show_grants'] + + # cleanup exported data + if os.path.exists(export_dir): + shutil.rmtree(export_dir) From 75e891385ca243ebe43e0e2afa6733d5d9cd61d2 Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Tue, 14 Oct 2025 08:46:09 +0000 Subject: [PATCH 13/18] last --- .../backup_collection/basic_user_scenarios.py | 363 ------------------ 1 file changed, 363 deletions(-) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index baf5a8c2b734..01b47199a426 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -960,366 +960,3 @@ def record_last_snapshot(): # cleanup if os.path.exists(export_dir): shutil.rmtree(export_dir) - - -class TestFullCycleLocalBackupRestoreWSchemaChange(TestFullCycleLocalBackupRestore): - def _create_backup_collection(self, collection_src, tables: List[str]): - # create backup collection referencing given table full paths - table_entries = ",\n".join([f"TABLE `/Root/{t}`" for t in tables]) - sql = f""" - CREATE BACKUP COLLECTION `{collection_src}` - ( {table_entries} ) - WITH ( STORAGE = 'cluster', INCREMENTAL_BACKUP_ENABLED = 'false' ); - """ - res = self._execute_yql(sql) - stderr_out = "" - if getattr(res, 'std_err', None): - stderr_out += res.std_err.decode('utf-8', errors='ignore') - if getattr(res, 'std_out', None): - stderr_out += res.std_out.decode('utf-8', errors='ignore') - assert res.exit_code == 0, f"CREATE BACKUP COLLECTION failed: {stderr_out}" - self.wait_for_collection(collection_src, timeout_s=30) - - def _backup_now(self, collection_src): - time.sleep(1.1) - res = self._execute_yql(f"BACKUP `{collection_src}`;") - if res.exit_code != 0: - out = (res.std_out or b"").decode('utf-8', 'ignore') - err = (res.std_err or b"").decode('utf-8', 'ignore') - raise AssertionError(f"BACKUP failed: code={res.exit_code} STDOUT: {out} STDERR: {err}") - - def _restore_import(self, export_dir, exported_item, collection_restore): - bdir = os.path.join(export_dir, exported_item) - r = yatest.common.execute( - [backup_bin(), "--verbose", "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", self.root_dir, "tools", "restore", - "--path", f"/Root/.backups/collections/{collection_restore}", - "--input", bdir], - check_exit_code=False, - ) - assert r.exit_code == 0, f"tools restore import failed: {r.std_err}" - - def _verify_restored_table_data(self, table, expected_rows): - # rows = self._capture_snapshot(table) - rows = self.wait_for_table_rows(table, expected_rows, timeout_s=90) - assert rows == expected_rows, f"Restored data for {table} doesn't match expected.\nExpected: {expected_rows}\nGot: {rows}" - - def _capture_schema(self, table_path: str): - desc = self.driver.scheme_client.describe_path(table_path) - cols = self._get_columns_from_scheme_entry(desc, path_hint=table_path) - return cols - - def _capture_acl(self, table_path: str): - # Attempt to capture owner/grants/acl in a readable form. - try: - desc = self.driver.scheme_client.describe_path(table_path) - except Exception: - return None - - acl_info = {} - owner = getattr(desc, "owner", None) - if owner: - acl_info["owner"] = owner - - for cand in ("acl", "grants", "effective_acl", "permission", "permissions"): - if hasattr(desc, cand): - try: - val = getattr(desc, cand) - acl_info[cand] = val - except Exception: - acl_info[cand] = "" - - # Fallback: try SHOW GRANTS via YQL and capture stdout - try: - res = self._execute_yql(f"SHOW GRANTS ON '{table_path}';") - out = (res.std_out or b"").decode('utf-8', 'ignore') - if out: - acl_info["show_grants"] = out - except Exception: - pass - - return acl_info - - # --- Helpers for extended checks (schema and ACL capture) --- - def _get_columns_from_scheme_entry(self, desc, path_hint: str = None): - # Reuse original robust approach: try multiple candidate attributes - try: - table_obj = getattr(desc, "table", None) - if table_obj is not None: - cols = getattr(table_obj, "columns", None) - if cols: - return [c.name for c in cols] - - cols = getattr(desc, "columns", None) - if cols: - try: - return [c.name for c in cols] - except Exception: - return [str(c) for c in cols] - - for attr in ("schema", "entry", "path"): - nested = getattr(desc, attr, None) - if nested is not None: - table_obj = getattr(nested, "table", None) - cols = getattr(table_obj, "columns", None) if table_obj is not None else None - if cols: - return [c.name for c in cols] - except Exception: - pass - - if getattr(desc, "is_table", False) or getattr(desc, "is_row_table", False) or getattr(desc, "is_column_table", False): - if path_hint: - table_path = path_hint - else: - name = getattr(desc, "name", None) - assert name, f"SchemeEntry has no name, can't form path. desc repr: {repr(desc)}" - table_path = name if name.startswith("/Root") else os.path.join(self.root_dir, name) - - try: - tc = getattr(self.driver, "table_client", None) - if tc is not None and hasattr(tc, "describe_table"): - desc_tbl = tc.describe_table(table_path) - cols = getattr(desc_tbl, "columns", None) or getattr(desc_tbl, "Columns", None) - if cols: - try: - return [c.name for c in cols] - except Exception: - return [str(c) for c in cols] - except Exception: - pass - - try: - with self.session_scope() as session: - if hasattr(session, "describe_table"): - desc_tbl = session.describe_table(table_path) - cols = getattr(desc_tbl, "columns", None) or getattr(desc_tbl, "Columns", None) - if cols: - try: - return [c.name for c in cols] - except Exception: - return [str(c) for c in cols] - except Exception: - pass - - diagnostics = ["Failed to find columns via known candidates.\n"] - try: - diagnostics.append("dir(desc):\n" + ", ".join(dir(desc)) + "\n") - except Exception as e: - diagnostics.append(f"dir(desc) raised: {e}\n") - - readable = [] - for attr in sorted(set(dir(desc))): - if attr.startswith("_"): - continue - if len(readable) >= 40: - break - try: - val = getattr(desc, attr) - if callable(val): - continue - s = repr(val) - if len(s) > 300: - s = s[:300] + "...(truncated)" - readable.append(f"{attr} = {s}") - except Exception as e: - readable.append(f"{attr} = ") - - diagnostics.append("Sample attributes (truncated):\n" + "\n".join(readable) + "\n") - - raise AssertionError( - "describe_path returned SchemeEntry in unexpected shape. Cannot locate columns.\n\nDiagnostic dump:\n\n" - + "\n".join(diagnostics) - ) - - def _drop_tables(self, tables: List[str]): - with self.session_scope() as session: - for t in tables: - full = f"/Root/{t}" - try: - session.execute_scheme(f"DROP TABLE `{full}`;") - except Exception: - # ignore failures - pass - - def test_full_cycle_local_backup_restore_with_schema_changes(self): - collection_src, t1, t2 = self._setup_test_collections() - - # Create backup collection (will reference the initial tables) - self._create_backup_collection(collection_src, [t1, t2]) - - # === Step 4: Add/remove data, change ACLs (1), add more tables (1) === - # perform first wave of modifications that must be captured by full backup 1 - with self.session_scope() as session: - # add & remove data - session.transaction().execute('PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES (10, 100, "one-wave");', commit_tx=True) - session.transaction().execute('PRAGMA TablePathPrefix("/Root"); DELETE FROM products WHERE id = 1;', commit_tx=True) - - # change ACLs: try multiple grant syntaxes until success - desc_for_acl = self.driver.scheme_client.describe_path("/Root/orders") - owner_role = getattr(desc_for_acl, "owner", None) or "root@builtin" - - def q(role: str) -> str: - return "`" + role.replace("`", "") + "`" - - role_candidates = [owner_role, "public", "everyone", "root"] - grant_variants = [] - for r in role_candidates: - role_quoted = q(r) - grant_variants.extend([ - f"GRANT ALL ON `/Root/orders` TO {role_quoted};", - f"GRANT SELECT ON `/Root/orders` TO {role_quoted};", - f"GRANT 'ydb.generic.read' ON `/Root/orders` TO {role_quoted};", - ]) - grant_variants.append(f"GRANT ALL ON `/Root/orders` TO {q(owner_role)};") - - acl_applied = False - for cmd in grant_variants: - res = self._execute_yql(cmd) - if res.exit_code == 0: - acl_applied = True - break - assert acl_applied, "Failed to apply any GRANT variant in step (1)" - - # add more tables (1) - create_table_with_data(session, "extra_table_1") - - # capture state after wave 1 - snapshot_wave1_t1 = self._capture_snapshot(t1) - snapshot_wave1_t2 = self._capture_snapshot(t2) - schema_wave1_t1 = self._capture_schema(f"/Root/{t1}") - schema_wave1_t2 = self._capture_schema(f"/Root/{t2}") - acl_wave1_t1 = self._capture_acl(f"/Root/{t1}") - acl_wave1_t2 = self._capture_acl(f"/Root/{t2}") - - # === Create full backup 1 === - self._backup_now(collection_src) - self.wait_for_collection_has_snapshot(collection_src, timeout_s=30) - - # === Step 7..12: modifications (2) include add/remove data, add more tables (2), remove some tables from step5, - # add/alter/drop columns, change ACLs === - with self.session_scope() as session: - # data modifications - session.transaction().execute('PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES (11, 111, "two-wave");', commit_tx=True) - session.transaction().execute('PRAGMA TablePathPrefix("/Root"); DELETE FROM orders WHERE id = 2;', commit_tx=True) - - # add more tables (2) - create_table_with_data(session, "extra_table_2") - - # remove some tables from step5: drop extra_table_1 - try: - session.execute_scheme('DROP TABLE `/Root/extra_table_1`;') - except Exception: - pass - - # add columns to initial tables - try: - session.execute_scheme('ALTER TABLE `/Root/orders` ADD COLUMN new_col Uint32;') - except Exception: - # if not supported, ignore but record - logger.info('ALTER TABLE ADD COLUMN not supported in this build') - - # attempt alter column (if supported) - many builds do not support complex ALTER, so guard with try - try: - session.execute_scheme('ALTER TABLE `/Root/orders` ALTER COLUMN number SET NOT NULL;') - except Exception: - logger.info('ALTER COLUMN ... SET NOT NULL not supported, skipping') - - # drop columns - try: - session.execute_scheme('ALTER TABLE `/Root/orders` DROP COLUMN new_col;') - except Exception: - logger.info('DROP COLUMN new_col not supported or already dropped') - - # change ACLs again for initial tables - desc_for_acl2 = self.driver.scheme_client.describe_path("/Root/orders") - owner_role2 = getattr(desc_for_acl2, "owner", None) or "root@builtin" - owner_quoted = owner_role2.replace('`', '') - cmd = f"GRANT SELECT ON `/Root/orders` TO `{owner_quoted}`;" - res = self._execute_yql(cmd) - assert res.exit_code == 0, "Failed to apply GRANT in wave 2" - - # capture state after wave 2 - snapshot_wave2_t1 = self._capture_snapshot(t1) - snapshot_wave2_t2 = self._capture_snapshot(t2) - schema_wave2_t1 = self._capture_schema(f"/Root/{t1}") - schema_wave2_t2 = self._capture_schema(f"/Root/{t2}") - acl_wave2_t1 = self._capture_acl(f"/Root/{t1}") - acl_wave2_t2 = self._capture_acl(f"/Root/{t2}") - - # === Create full backup 2 === - self._backup_now(collection_src) - self.wait_for_collection_has_snapshot(collection_src, timeout_s=30) - - # === Export backups so we can import snapshots into separate collections for restore verification === - export_dir, exported_items = self._export_backups(collection_src) - # expect at least two exported snapshots (backup1 and backup2) - assert len(exported_items) >= 2, "Expected at least 2 exported snapshots for verification" - - # === Attempt to import exported backup into new collection and RESTORE while tables exist => expect fail === - # create restore collections - coll_restore_1 = f"coll_restore_v1_{int(time.time())}" - coll_restore_2 = f"coll_restore_v2_{int(time.time())}" - self._create_backup_collection(coll_restore_1, [t1, t2]) - self._create_backup_collection(coll_restore_2, [t1, t2]) - - # import exported snapshots into restore collections - # imported_items are directories in exported_items; we'll import both - self._restore_import(export_dir, exported_items[0], coll_restore_1) - self._restore_import(export_dir, exported_items[1], coll_restore_2) - - # try RESTORE when tables already exist -> should fail - res_restore_when_exists = self._execute_yql(f"RESTORE `{coll_restore_1}`;") - assert res_restore_when_exists.exit_code != 0, "Expected RESTORE to fail when target tables already exist" - - # === Remove all tables from DB (orders, products, extras) === - self._drop_tables([t1, t2, "extra_table_1", "extra_table_2"]) # ignore errors - - # === Now RESTORE coll_restore_1 (which corresponds to backup1) === - res_restore1 = self._execute_yql(f"RESTORE `{coll_restore_1}`;") - assert res_restore1.exit_code == 0, f"RESTORE v1 failed: {res_restore1.std_err or res_restore1.std_out}" - - # verify schema/data/acl for backup1 - # verify data - self._verify_restored_table_data(t1, snapshot_wave1_t1) - self._verify_restored_table_data(t2, snapshot_wave1_t2) - - # verify schema - restored_schema_t1 = self._capture_schema(f"/Root/{t1}") - restored_schema_t2 = self._capture_schema(f"/Root/{t2}") - assert restored_schema_t1 == schema_wave1_t1, f"Schema for {t1} after restore v1 differs: expected {schema_wave1_t1}, got {restored_schema_t1}" - assert restored_schema_t2 == schema_wave1_t2, f"Schema for {t2} after restore v1 differs: expected {schema_wave1_t2}, got {restored_schema_t2}" - - # verify acl - restored_acl_t1 = self._capture_acl(f"/Root/{t1}") - restored_acl_t2 = self._capture_acl(f"/Root/{t2}") - # We compare that SHOW GRANTS output contains previously stored show_grants if present - if 'show_grants' in (acl_wave1_t1 or {}): - assert 'show_grants' in (restored_acl_t1 or {}) and acl_wave1_t1['show_grants'] in restored_acl_t1['show_grants'] - if 'show_grants' in (acl_wave1_t2 or {}): - assert 'show_grants' in (restored_acl_t2 or {}) and acl_wave1_t2['show_grants'] in restored_acl_t2['show_grants'] - - # === Remove all tables again and restore backup2 === - self._drop_tables([t1, t2, "extra_table_1", "extra_table_2"]) # ignore errors - - res_restore2 = self._execute_yql(f"RESTORE `{coll_restore_2}`;") - assert res_restore2.exit_code == 0, f"RESTORE v2 failed: {res_restore2.std_err or res_restore2.std_out}" - - # verify data/schema/acl for backup2 - self._verify_restored_table_data(t1, snapshot_wave2_t1) - self._verify_restored_table_data(t2, snapshot_wave2_t2) - - restored_schema2_t1 = self._capture_schema(f"/Root/{t1}") - restored_schema2_t2 = self._capture_schema(f"/Root/{t2}") - assert restored_schema2_t1 == schema_wave2_t1, f"Schema for {t1} after restore v2 differs: expected {schema_wave2_t1}, got {restored_schema2_t1}" - assert restored_schema2_t2 == schema_wave2_t2, f"Schema for {t2} after restore v2 differs: expected {schema_wave2_t2}, got {restored_schema2_t2}" - - restored_acl2_t1 = self._capture_acl(f"/Root/{t1}") - restored_acl2_t2 = self._capture_acl(f"/Root/{t2}") - if 'show_grants' in (acl_wave2_t1 or {}): - assert 'show_grants' in (restored_acl2_t1 or {}) and acl_wave2_t1['show_grants'] in restored_acl2_t1['show_grants'] - if 'show_grants' in (acl_wave2_t2 or {}): - assert 'show_grants' in (restored_acl2_t2 or {}) and acl_wave2_t2['show_grants'] in restored_acl2_t2['show_grants'] - - # cleanup exported data - if os.path.exists(export_dir): - shutil.rmtree(export_dir) From 28fbb21d0a8961c30b888d4531ee47b3bb6650d7 Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Tue, 14 Oct 2025 09:30:41 +0000 Subject: [PATCH 14/18] last --- ydb/tests/functional/backup_collection/basic_user_scenarios.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index 01b47199a426..396134c75e8a 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -718,7 +718,7 @@ def test_full_cycle_local_backup_restore_with_incrementals(self): def record_last_snapshot(): kids = sorted(self.get_collection_children(collection_src)) - assert kids, "No snapshots found after backup" + assert kids, "no snapshots found after backup" last = kids[-1] created_snapshots.append(last) return last From fd3bb0434b301e6eeca682d9d8a2a9b87f4e5763 Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Tue, 14 Oct 2025 10:45:56 +0000 Subject: [PATCH 15/18] init --- .../backup_collection/basic_user_scenarios.py | 365 +++++++++++++++++- 1 file changed, 364 insertions(+), 1 deletion(-) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index 396134c75e8a..baf5a8c2b734 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -718,7 +718,7 @@ def test_full_cycle_local_backup_restore_with_incrementals(self): def record_last_snapshot(): kids = sorted(self.get_collection_children(collection_src)) - assert kids, "no snapshots found after backup" + assert kids, "No snapshots found after backup" last = kids[-1] created_snapshots.append(last) return last @@ -960,3 +960,366 @@ def record_last_snapshot(): # cleanup if os.path.exists(export_dir): shutil.rmtree(export_dir) + + +class TestFullCycleLocalBackupRestoreWSchemaChange(TestFullCycleLocalBackupRestore): + def _create_backup_collection(self, collection_src, tables: List[str]): + # create backup collection referencing given table full paths + table_entries = ",\n".join([f"TABLE `/Root/{t}`" for t in tables]) + sql = f""" + CREATE BACKUP COLLECTION `{collection_src}` + ( {table_entries} ) + WITH ( STORAGE = 'cluster', INCREMENTAL_BACKUP_ENABLED = 'false' ); + """ + res = self._execute_yql(sql) + stderr_out = "" + if getattr(res, 'std_err', None): + stderr_out += res.std_err.decode('utf-8', errors='ignore') + if getattr(res, 'std_out', None): + stderr_out += res.std_out.decode('utf-8', errors='ignore') + assert res.exit_code == 0, f"CREATE BACKUP COLLECTION failed: {stderr_out}" + self.wait_for_collection(collection_src, timeout_s=30) + + def _backup_now(self, collection_src): + time.sleep(1.1) + res = self._execute_yql(f"BACKUP `{collection_src}`;") + if res.exit_code != 0: + out = (res.std_out or b"").decode('utf-8', 'ignore') + err = (res.std_err or b"").decode('utf-8', 'ignore') + raise AssertionError(f"BACKUP failed: code={res.exit_code} STDOUT: {out} STDERR: {err}") + + def _restore_import(self, export_dir, exported_item, collection_restore): + bdir = os.path.join(export_dir, exported_item) + r = yatest.common.execute( + [backup_bin(), "--verbose", "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", self.root_dir, "tools", "restore", + "--path", f"/Root/.backups/collections/{collection_restore}", + "--input", bdir], + check_exit_code=False, + ) + assert r.exit_code == 0, f"tools restore import failed: {r.std_err}" + + def _verify_restored_table_data(self, table, expected_rows): + # rows = self._capture_snapshot(table) + rows = self.wait_for_table_rows(table, expected_rows, timeout_s=90) + assert rows == expected_rows, f"Restored data for {table} doesn't match expected.\nExpected: {expected_rows}\nGot: {rows}" + + def _capture_schema(self, table_path: str): + desc = self.driver.scheme_client.describe_path(table_path) + cols = self._get_columns_from_scheme_entry(desc, path_hint=table_path) + return cols + + def _capture_acl(self, table_path: str): + # Attempt to capture owner/grants/acl in a readable form. + try: + desc = self.driver.scheme_client.describe_path(table_path) + except Exception: + return None + + acl_info = {} + owner = getattr(desc, "owner", None) + if owner: + acl_info["owner"] = owner + + for cand in ("acl", "grants", "effective_acl", "permission", "permissions"): + if hasattr(desc, cand): + try: + val = getattr(desc, cand) + acl_info[cand] = val + except Exception: + acl_info[cand] = "" + + # Fallback: try SHOW GRANTS via YQL and capture stdout + try: + res = self._execute_yql(f"SHOW GRANTS ON '{table_path}';") + out = (res.std_out or b"").decode('utf-8', 'ignore') + if out: + acl_info["show_grants"] = out + except Exception: + pass + + return acl_info + + # --- Helpers for extended checks (schema and ACL capture) --- + def _get_columns_from_scheme_entry(self, desc, path_hint: str = None): + # Reuse original robust approach: try multiple candidate attributes + try: + table_obj = getattr(desc, "table", None) + if table_obj is not None: + cols = getattr(table_obj, "columns", None) + if cols: + return [c.name for c in cols] + + cols = getattr(desc, "columns", None) + if cols: + try: + return [c.name for c in cols] + except Exception: + return [str(c) for c in cols] + + for attr in ("schema", "entry", "path"): + nested = getattr(desc, attr, None) + if nested is not None: + table_obj = getattr(nested, "table", None) + cols = getattr(table_obj, "columns", None) if table_obj is not None else None + if cols: + return [c.name for c in cols] + except Exception: + pass + + if getattr(desc, "is_table", False) or getattr(desc, "is_row_table", False) or getattr(desc, "is_column_table", False): + if path_hint: + table_path = path_hint + else: + name = getattr(desc, "name", None) + assert name, f"SchemeEntry has no name, can't form path. desc repr: {repr(desc)}" + table_path = name if name.startswith("/Root") else os.path.join(self.root_dir, name) + + try: + tc = getattr(self.driver, "table_client", None) + if tc is not None and hasattr(tc, "describe_table"): + desc_tbl = tc.describe_table(table_path) + cols = getattr(desc_tbl, "columns", None) or getattr(desc_tbl, "Columns", None) + if cols: + try: + return [c.name for c in cols] + except Exception: + return [str(c) for c in cols] + except Exception: + pass + + try: + with self.session_scope() as session: + if hasattr(session, "describe_table"): + desc_tbl = session.describe_table(table_path) + cols = getattr(desc_tbl, "columns", None) or getattr(desc_tbl, "Columns", None) + if cols: + try: + return [c.name for c in cols] + except Exception: + return [str(c) for c in cols] + except Exception: + pass + + diagnostics = ["Failed to find columns via known candidates.\n"] + try: + diagnostics.append("dir(desc):\n" + ", ".join(dir(desc)) + "\n") + except Exception as e: + diagnostics.append(f"dir(desc) raised: {e}\n") + + readable = [] + for attr in sorted(set(dir(desc))): + if attr.startswith("_"): + continue + if len(readable) >= 40: + break + try: + val = getattr(desc, attr) + if callable(val): + continue + s = repr(val) + if len(s) > 300: + s = s[:300] + "...(truncated)" + readable.append(f"{attr} = {s}") + except Exception as e: + readable.append(f"{attr} = ") + + diagnostics.append("Sample attributes (truncated):\n" + "\n".join(readable) + "\n") + + raise AssertionError( + "describe_path returned SchemeEntry in unexpected shape. Cannot locate columns.\n\nDiagnostic dump:\n\n" + + "\n".join(diagnostics) + ) + + def _drop_tables(self, tables: List[str]): + with self.session_scope() as session: + for t in tables: + full = f"/Root/{t}" + try: + session.execute_scheme(f"DROP TABLE `{full}`;") + except Exception: + # ignore failures + pass + + def test_full_cycle_local_backup_restore_with_schema_changes(self): + collection_src, t1, t2 = self._setup_test_collections() + + # Create backup collection (will reference the initial tables) + self._create_backup_collection(collection_src, [t1, t2]) + + # === Step 4: Add/remove data, change ACLs (1), add more tables (1) === + # perform first wave of modifications that must be captured by full backup 1 + with self.session_scope() as session: + # add & remove data + session.transaction().execute('PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES (10, 100, "one-wave");', commit_tx=True) + session.transaction().execute('PRAGMA TablePathPrefix("/Root"); DELETE FROM products WHERE id = 1;', commit_tx=True) + + # change ACLs: try multiple grant syntaxes until success + desc_for_acl = self.driver.scheme_client.describe_path("/Root/orders") + owner_role = getattr(desc_for_acl, "owner", None) or "root@builtin" + + def q(role: str) -> str: + return "`" + role.replace("`", "") + "`" + + role_candidates = [owner_role, "public", "everyone", "root"] + grant_variants = [] + for r in role_candidates: + role_quoted = q(r) + grant_variants.extend([ + f"GRANT ALL ON `/Root/orders` TO {role_quoted};", + f"GRANT SELECT ON `/Root/orders` TO {role_quoted};", + f"GRANT 'ydb.generic.read' ON `/Root/orders` TO {role_quoted};", + ]) + grant_variants.append(f"GRANT ALL ON `/Root/orders` TO {q(owner_role)};") + + acl_applied = False + for cmd in grant_variants: + res = self._execute_yql(cmd) + if res.exit_code == 0: + acl_applied = True + break + assert acl_applied, "Failed to apply any GRANT variant in step (1)" + + # add more tables (1) + create_table_with_data(session, "extra_table_1") + + # capture state after wave 1 + snapshot_wave1_t1 = self._capture_snapshot(t1) + snapshot_wave1_t2 = self._capture_snapshot(t2) + schema_wave1_t1 = self._capture_schema(f"/Root/{t1}") + schema_wave1_t2 = self._capture_schema(f"/Root/{t2}") + acl_wave1_t1 = self._capture_acl(f"/Root/{t1}") + acl_wave1_t2 = self._capture_acl(f"/Root/{t2}") + + # === Create full backup 1 === + self._backup_now(collection_src) + self.wait_for_collection_has_snapshot(collection_src, timeout_s=30) + + # === Step 7..12: modifications (2) include add/remove data, add more tables (2), remove some tables from step5, + # add/alter/drop columns, change ACLs === + with self.session_scope() as session: + # data modifications + session.transaction().execute('PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES (11, 111, "two-wave");', commit_tx=True) + session.transaction().execute('PRAGMA TablePathPrefix("/Root"); DELETE FROM orders WHERE id = 2;', commit_tx=True) + + # add more tables (2) + create_table_with_data(session, "extra_table_2") + + # remove some tables from step5: drop extra_table_1 + try: + session.execute_scheme('DROP TABLE `/Root/extra_table_1`;') + except Exception: + pass + + # add columns to initial tables + try: + session.execute_scheme('ALTER TABLE `/Root/orders` ADD COLUMN new_col Uint32;') + except Exception: + # if not supported, ignore but record + logger.info('ALTER TABLE ADD COLUMN not supported in this build') + + # attempt alter column (if supported) - many builds do not support complex ALTER, so guard with try + try: + session.execute_scheme('ALTER TABLE `/Root/orders` ALTER COLUMN number SET NOT NULL;') + except Exception: + logger.info('ALTER COLUMN ... SET NOT NULL not supported, skipping') + + # drop columns + try: + session.execute_scheme('ALTER TABLE `/Root/orders` DROP COLUMN new_col;') + except Exception: + logger.info('DROP COLUMN new_col not supported or already dropped') + + # change ACLs again for initial tables + desc_for_acl2 = self.driver.scheme_client.describe_path("/Root/orders") + owner_role2 = getattr(desc_for_acl2, "owner", None) or "root@builtin" + owner_quoted = owner_role2.replace('`', '') + cmd = f"GRANT SELECT ON `/Root/orders` TO `{owner_quoted}`;" + res = self._execute_yql(cmd) + assert res.exit_code == 0, "Failed to apply GRANT in wave 2" + + # capture state after wave 2 + snapshot_wave2_t1 = self._capture_snapshot(t1) + snapshot_wave2_t2 = self._capture_snapshot(t2) + schema_wave2_t1 = self._capture_schema(f"/Root/{t1}") + schema_wave2_t2 = self._capture_schema(f"/Root/{t2}") + acl_wave2_t1 = self._capture_acl(f"/Root/{t1}") + acl_wave2_t2 = self._capture_acl(f"/Root/{t2}") + + # === Create full backup 2 === + self._backup_now(collection_src) + self.wait_for_collection_has_snapshot(collection_src, timeout_s=30) + + # === Export backups so we can import snapshots into separate collections for restore verification === + export_dir, exported_items = self._export_backups(collection_src) + # expect at least two exported snapshots (backup1 and backup2) + assert len(exported_items) >= 2, "Expected at least 2 exported snapshots for verification" + + # === Attempt to import exported backup into new collection and RESTORE while tables exist => expect fail === + # create restore collections + coll_restore_1 = f"coll_restore_v1_{int(time.time())}" + coll_restore_2 = f"coll_restore_v2_{int(time.time())}" + self._create_backup_collection(coll_restore_1, [t1, t2]) + self._create_backup_collection(coll_restore_2, [t1, t2]) + + # import exported snapshots into restore collections + # imported_items are directories in exported_items; we'll import both + self._restore_import(export_dir, exported_items[0], coll_restore_1) + self._restore_import(export_dir, exported_items[1], coll_restore_2) + + # try RESTORE when tables already exist -> should fail + res_restore_when_exists = self._execute_yql(f"RESTORE `{coll_restore_1}`;") + assert res_restore_when_exists.exit_code != 0, "Expected RESTORE to fail when target tables already exist" + + # === Remove all tables from DB (orders, products, extras) === + self._drop_tables([t1, t2, "extra_table_1", "extra_table_2"]) # ignore errors + + # === Now RESTORE coll_restore_1 (which corresponds to backup1) === + res_restore1 = self._execute_yql(f"RESTORE `{coll_restore_1}`;") + assert res_restore1.exit_code == 0, f"RESTORE v1 failed: {res_restore1.std_err or res_restore1.std_out}" + + # verify schema/data/acl for backup1 + # verify data + self._verify_restored_table_data(t1, snapshot_wave1_t1) + self._verify_restored_table_data(t2, snapshot_wave1_t2) + + # verify schema + restored_schema_t1 = self._capture_schema(f"/Root/{t1}") + restored_schema_t2 = self._capture_schema(f"/Root/{t2}") + assert restored_schema_t1 == schema_wave1_t1, f"Schema for {t1} after restore v1 differs: expected {schema_wave1_t1}, got {restored_schema_t1}" + assert restored_schema_t2 == schema_wave1_t2, f"Schema for {t2} after restore v1 differs: expected {schema_wave1_t2}, got {restored_schema_t2}" + + # verify acl + restored_acl_t1 = self._capture_acl(f"/Root/{t1}") + restored_acl_t2 = self._capture_acl(f"/Root/{t2}") + # We compare that SHOW GRANTS output contains previously stored show_grants if present + if 'show_grants' in (acl_wave1_t1 or {}): + assert 'show_grants' in (restored_acl_t1 or {}) and acl_wave1_t1['show_grants'] in restored_acl_t1['show_grants'] + if 'show_grants' in (acl_wave1_t2 or {}): + assert 'show_grants' in (restored_acl_t2 or {}) and acl_wave1_t2['show_grants'] in restored_acl_t2['show_grants'] + + # === Remove all tables again and restore backup2 === + self._drop_tables([t1, t2, "extra_table_1", "extra_table_2"]) # ignore errors + + res_restore2 = self._execute_yql(f"RESTORE `{coll_restore_2}`;") + assert res_restore2.exit_code == 0, f"RESTORE v2 failed: {res_restore2.std_err or res_restore2.std_out}" + + # verify data/schema/acl for backup2 + self._verify_restored_table_data(t1, snapshot_wave2_t1) + self._verify_restored_table_data(t2, snapshot_wave2_t2) + + restored_schema2_t1 = self._capture_schema(f"/Root/{t1}") + restored_schema2_t2 = self._capture_schema(f"/Root/{t2}") + assert restored_schema2_t1 == schema_wave2_t1, f"Schema for {t1} after restore v2 differs: expected {schema_wave2_t1}, got {restored_schema2_t1}" + assert restored_schema2_t2 == schema_wave2_t2, f"Schema for {t2} after restore v2 differs: expected {schema_wave2_t2}, got {restored_schema2_t2}" + + restored_acl2_t1 = self._capture_acl(f"/Root/{t1}") + restored_acl2_t2 = self._capture_acl(f"/Root/{t2}") + if 'show_grants' in (acl_wave2_t1 or {}): + assert 'show_grants' in (restored_acl2_t1 or {}) and acl_wave2_t1['show_grants'] in restored_acl2_t1['show_grants'] + if 'show_grants' in (acl_wave2_t2 or {}): + assert 'show_grants' in (restored_acl2_t2 or {}) and acl_wave2_t2['show_grants'] in restored_acl2_t2['show_grants'] + + # cleanup exported data + if os.path.exists(export_dir): + shutil.rmtree(export_dir) From 5ee9127387f5a6d1c8b880c92ea0faa301efc8b9 Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Wed, 15 Oct 2025 12:55:44 +0000 Subject: [PATCH 16/18] some changes --- .../backup_collection/basic_user_scenarios.py | 312 ++++++++++-------- 1 file changed, 173 insertions(+), 139 deletions(-) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index baf5a8c2b734..b66790ec41eb 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -354,6 +354,86 @@ def wait_for_table_rows(self, raise AssertionError(f"Timeout waiting for table '{table}' rows to match expected (timeout {timeout_s}s). Last error: {last_exc}") + def _create_backup_collection(self, collection_src, tables: List[str]): + # create backup collection referencing given table full paths + table_entries = ",\n".join([f"TABLE `/Root/{t}`" for t in tables]) + sql = f""" + CREATE BACKUP COLLECTION `{collection_src}` + ( {table_entries} ) + WITH ( STORAGE = 'cluster', INCREMENTAL_BACKUP_ENABLED = 'false' ); + """ + res = self._execute_yql(sql) + stderr_out = "" + if getattr(res, 'std_err', None): + stderr_out += res.std_err.decode('utf-8', errors='ignore') + if getattr(res, 'std_out', None): + stderr_out += res.std_out.decode('utf-8', errors='ignore') + assert res.exit_code == 0, f"CREATE BACKUP COLLECTION failed: {stderr_out}" + self.wait_for_collection(collection_src, timeout_s=30) + + def _backup_now(self, collection_src): + time.sleep(1.1) + res = self._execute_yql(f"BACKUP `{collection_src}`;") + if res.exit_code != 0: + out = (res.std_out or b"").decode('utf-8', 'ignore') + err = (res.std_err or b"").decode('utf-8', 'ignore') + raise AssertionError(f"BACKUP failed: code={res.exit_code} STDOUT: {out} STDERR: {err}") + + def _restore_import(self, export_dir, exported_item, collection_restore): + bdir = os.path.join(export_dir, exported_item) + r = yatest.common.execute( + [backup_bin(), "--verbose", "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, + "--database", self.root_dir, "tools", "restore", + "--path", f"/Root/.backups/collections/{collection_restore}", + "--input", bdir], + check_exit_code=False, + ) + assert r.exit_code == 0, f"tools restore import failed: {r.std_err}" + + def _verify_restored_table_data(self, table, expected_rows): + rows = self.wait_for_table_rows(table, expected_rows, timeout_s=90) + assert rows == expected_rows, f"Restored data for {table} doesn't match expected.\nExpected: {expected_rows}\nGot: {rows}" + + def _capture_acl(self, table_path: str): + # Attempt to capture owner/grants/acl in a readable form. + try: + desc = self.driver.scheme_client.describe_path(table_path) + except Exception: + return None + + acl_info = {} + owner = getattr(desc, "owner", None) + if owner: + acl_info["owner"] = owner + + for cand in ("acl", "grants", "effective_acl", "permission", "permissions"): + if hasattr(desc, cand): + try: + val = getattr(desc, cand) + acl_info[cand] = val + except Exception: + acl_info[cand] = "" + + # Fallback: try SHOW GRANTS via YQL and capture stdout + try: + res = self._execute_yql(f"SHOW GRANTS ON '{table_path}';") + out = (res.std_out or b"").decode('utf-8', 'ignore') + if out: + acl_info["show_grants"] = out + except Exception: + pass + + return acl_info + + def _drop_tables(self, tables: List[str]): + with self.session_scope() as session: + for t in tables: + full = f"/Root/{t}" + try: + session.execute_scheme(f"DROP TABLE `{full}`;") + except Exception: + raise AssertionError("Drop failed") + class TestFullCycleLocalBackupRestore(BaseTestBackupInFiles): def _execute_yql(self, script, verbose=False): @@ -963,84 +1043,6 @@ def record_last_snapshot(): class TestFullCycleLocalBackupRestoreWSchemaChange(TestFullCycleLocalBackupRestore): - def _create_backup_collection(self, collection_src, tables: List[str]): - # create backup collection referencing given table full paths - table_entries = ",\n".join([f"TABLE `/Root/{t}`" for t in tables]) - sql = f""" - CREATE BACKUP COLLECTION `{collection_src}` - ( {table_entries} ) - WITH ( STORAGE = 'cluster', INCREMENTAL_BACKUP_ENABLED = 'false' ); - """ - res = self._execute_yql(sql) - stderr_out = "" - if getattr(res, 'std_err', None): - stderr_out += res.std_err.decode('utf-8', errors='ignore') - if getattr(res, 'std_out', None): - stderr_out += res.std_out.decode('utf-8', errors='ignore') - assert res.exit_code == 0, f"CREATE BACKUP COLLECTION failed: {stderr_out}" - self.wait_for_collection(collection_src, timeout_s=30) - - def _backup_now(self, collection_src): - time.sleep(1.1) - res = self._execute_yql(f"BACKUP `{collection_src}`;") - if res.exit_code != 0: - out = (res.std_out or b"").decode('utf-8', 'ignore') - err = (res.std_err or b"").decode('utf-8', 'ignore') - raise AssertionError(f"BACKUP failed: code={res.exit_code} STDOUT: {out} STDERR: {err}") - - def _restore_import(self, export_dir, exported_item, collection_restore): - bdir = os.path.join(export_dir, exported_item) - r = yatest.common.execute( - [backup_bin(), "--verbose", "--endpoint", "grpc://localhost:%d" % self.cluster.nodes[1].grpc_port, - "--database", self.root_dir, "tools", "restore", - "--path", f"/Root/.backups/collections/{collection_restore}", - "--input", bdir], - check_exit_code=False, - ) - assert r.exit_code == 0, f"tools restore import failed: {r.std_err}" - - def _verify_restored_table_data(self, table, expected_rows): - # rows = self._capture_snapshot(table) - rows = self.wait_for_table_rows(table, expected_rows, timeout_s=90) - assert rows == expected_rows, f"Restored data for {table} doesn't match expected.\nExpected: {expected_rows}\nGot: {rows}" - - def _capture_schema(self, table_path: str): - desc = self.driver.scheme_client.describe_path(table_path) - cols = self._get_columns_from_scheme_entry(desc, path_hint=table_path) - return cols - - def _capture_acl(self, table_path: str): - # Attempt to capture owner/grants/acl in a readable form. - try: - desc = self.driver.scheme_client.describe_path(table_path) - except Exception: - return None - - acl_info = {} - owner = getattr(desc, "owner", None) - if owner: - acl_info["owner"] = owner - - for cand in ("acl", "grants", "effective_acl", "permission", "permissions"): - if hasattr(desc, cand): - try: - val = getattr(desc, cand) - acl_info[cand] = val - except Exception: - acl_info[cand] = "" - - # Fallback: try SHOW GRANTS via YQL and capture stdout - try: - res = self._execute_yql(f"SHOW GRANTS ON '{table_path}';") - out = (res.std_out or b"").decode('utf-8', 'ignore') - if out: - acl_info["show_grants"] = out - except Exception: - pass - - return acl_info - - # --- Helpers for extended checks (schema and ACL capture) --- def _get_columns_from_scheme_entry(self, desc, path_hint: str = None): # Reuse original robust approach: try multiple candidate attributes try: @@ -1131,15 +1133,48 @@ def _get_columns_from_scheme_entry(self, desc, path_hint: str = None): + "\n".join(diagnostics) ) - def _drop_tables(self, tables: List[str]): + def _capture_schema(self, table_path: str): + desc = self.driver.scheme_client.describe_path(table_path) + cols = self._get_columns_from_scheme_entry(desc, path_hint=table_path) + return cols + + def _create_table_with_data(self, session, path, not_null=False): + full_path = "/Root/" + path + session.create_table( + full_path, + ydb.TableDescription() + .with_column( + ydb.Column( + "id", + ydb.PrimitiveType.Uint32 if not_null else ydb.OptionalType(ydb.PrimitiveType.Uint32), + ) + ) + .with_column(ydb.Column("number", ydb.OptionalType(ydb.PrimitiveType.Uint64))) + .with_column(ydb.Column("txt", ydb.OptionalType(ydb.PrimitiveType.String))) + .with_column(ydb.Column("expire_at", ydb.OptionalType(ydb.PrimitiveType.Timestamp))) + .with_primary_keys("id"), + ) + + path_prefix, table = os.path.split(full_path) + session.transaction().execute( + ( + f'PRAGMA TablePathPrefix("{path_prefix}"); ' + f'UPSERT INTO {table} (id, number, txt, expire_at) VALUES ' + f'(1, 10, "one", CurrentUtcTimestamp()), (2, 20, "two", CurrentUtcTimestamp()), (3, 30, "three", CurrentUtcTimestamp());' + ), + commit_tx=True, + ) + + def _setup_test_collections(self): + collection_src = f"coll_src_{int(time.time())}" + t1 = "orders" + t2 = "products" + with self.session_scope() as session: - for t in tables: - full = f"/Root/{t}" - try: - session.execute_scheme(f"DROP TABLE `{full}`;") - except Exception: - # ignore failures - pass + self._create_table_with_data(session, t1) + self._create_table_with_data(session, t2) + + return collection_src, t1, t2 def test_full_cycle_local_backup_restore_with_schema_changes(self): collection_src, t1, t2 = self._setup_test_collections() @@ -1147,11 +1182,11 @@ def test_full_cycle_local_backup_restore_with_schema_changes(self): # Create backup collection (will reference the initial tables) self._create_backup_collection(collection_src, [t1, t2]) - # === Step 4: Add/remove data, change ACLs (1), add more tables (1) === - # perform first wave of modifications that must be captured by full backup 1 + # Add/remove data, change ACLs, add more tables + # perform first stage of modifications that must be captured by full backup 1 with self.session_scope() as session: # add & remove data - session.transaction().execute('PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES (10, 100, "one-wave");', commit_tx=True) + session.transaction().execute('PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES (10, 100, "one-stage");', commit_tx=True) session.transaction().execute('PRAGMA TablePathPrefix("/Root"); DELETE FROM products WHERE id = 1;', commit_tx=True) # change ACLs: try multiple grant syntaxes until success @@ -1180,55 +1215,54 @@ def q(role: str) -> str: break assert acl_applied, "Failed to apply any GRANT variant in step (1)" - # add more tables (1) + # add more tables create_table_with_data(session, "extra_table_1") - # capture state after wave 1 - snapshot_wave1_t1 = self._capture_snapshot(t1) - snapshot_wave1_t2 = self._capture_snapshot(t2) - schema_wave1_t1 = self._capture_schema(f"/Root/{t1}") - schema_wave1_t2 = self._capture_schema(f"/Root/{t2}") - acl_wave1_t1 = self._capture_acl(f"/Root/{t1}") - acl_wave1_t2 = self._capture_acl(f"/Root/{t2}") + # capture state after stage 1 + snapshot_stage1_t1 = self._capture_snapshot(t1) + snapshot_stage1_t2 = self._capture_snapshot(t2) + schema_stage1_t1 = self._capture_schema(f"/Root/{t1}") + schema_stage1_t2 = self._capture_schema(f"/Root/{t2}") + acl_stage1_t1 = self._capture_acl(f"/Root/{t1}") + acl_stage1_t2 = self._capture_acl(f"/Root/{t2}") - # === Create full backup 1 === + # Create full backup 1 self._backup_now(collection_src) self.wait_for_collection_has_snapshot(collection_src, timeout_s=30) - # === Step 7..12: modifications (2) include add/remove data, add more tables (2), remove some tables from step5, - # add/alter/drop columns, change ACLs === + # modifications include add/remove data, add more tables, remove some tables, + # add/alter/drop columns, change ACLs with self.session_scope() as session: # data modifications - session.transaction().execute('PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES (11, 111, "two-wave");', commit_tx=True) + session.transaction().execute('PRAGMA TablePathPrefix("/Root"); UPSERT INTO orders (id, number, txt) VALUES (11, 111, "two-stage");', commit_tx=True) session.transaction().execute('PRAGMA TablePathPrefix("/Root"); DELETE FROM orders WHERE id = 2;', commit_tx=True) - # add more tables (2) + # add more tables create_table_with_data(session, "extra_table_2") # remove some tables from step5: drop extra_table_1 try: session.execute_scheme('DROP TABLE `/Root/extra_table_1`;') except Exception: - pass + raise AssertionError("DROP failed") # add columns to initial tables try: session.execute_scheme('ALTER TABLE `/Root/orders` ADD COLUMN new_col Uint32;') except Exception: - # if not supported, ignore but record - logger.info('ALTER TABLE ADD COLUMN not supported in this build') + raise AssertionError("ADD COLUMN failed") # attempt alter column (if supported) - many builds do not support complex ALTER, so guard with try try: - session.execute_scheme('ALTER TABLE `/Root/orders` ALTER COLUMN number SET NOT NULL;') + session.execute_scheme('ALTER TABLE `/Root/orders` SET (TTL = Interval("PT0S") ON expire_at);') except Exception: - logger.info('ALTER COLUMN ... SET NOT NULL not supported, skipping') + raise AssertionError("SET TTL failed") # drop columns try: session.execute_scheme('ALTER TABLE `/Root/orders` DROP COLUMN new_col;') except Exception: - logger.info('DROP COLUMN new_col not supported or already dropped') + raise AssertionError("DROP COLUMN failed") # change ACLs again for initial tables desc_for_acl2 = self.driver.scheme_client.describe_path("/Root/orders") @@ -1236,26 +1270,26 @@ def q(role: str) -> str: owner_quoted = owner_role2.replace('`', '') cmd = f"GRANT SELECT ON `/Root/orders` TO `{owner_quoted}`;" res = self._execute_yql(cmd) - assert res.exit_code == 0, "Failed to apply GRANT in wave 2" + assert res.exit_code == 0, "Failed to apply GRANT in stage 2" - # capture state after wave 2 - snapshot_wave2_t1 = self._capture_snapshot(t1) - snapshot_wave2_t2 = self._capture_snapshot(t2) - schema_wave2_t1 = self._capture_schema(f"/Root/{t1}") - schema_wave2_t2 = self._capture_schema(f"/Root/{t2}") - acl_wave2_t1 = self._capture_acl(f"/Root/{t1}") - acl_wave2_t2 = self._capture_acl(f"/Root/{t2}") + # capture state after stage 2 + snapshot_stage2_t1 = self._capture_snapshot(t1) + snapshot_stage2_t2 = self._capture_snapshot(t2) + schema_stage2_t1 = self._capture_schema(f"/Root/{t1}") + schema_stage2_t2 = self._capture_schema(f"/Root/{t2}") + acl_stage2_t1 = self._capture_acl(f"/Root/{t1}") + acl_stage2_t2 = self._capture_acl(f"/Root/{t2}") - # === Create full backup 2 === + # Create full backup 2 self._backup_now(collection_src) self.wait_for_collection_has_snapshot(collection_src, timeout_s=30) - # === Export backups so we can import snapshots into separate collections for restore verification === + # Export backups so we can import snapshots into separate collections for restore verification export_dir, exported_items = self._export_backups(collection_src) # expect at least two exported snapshots (backup1 and backup2) assert len(exported_items) >= 2, "Expected at least 2 exported snapshots for verification" - # === Attempt to import exported backup into new collection and RESTORE while tables exist => expect fail === + # Attempt to import exported backup into new collection and RESTORE while tables exist -> expect fail # create restore collections coll_restore_1 = f"coll_restore_v1_{int(time.time())}" coll_restore_2 = f"coll_restore_v2_{int(time.time())}" @@ -1271,54 +1305,54 @@ def q(role: str) -> str: res_restore_when_exists = self._execute_yql(f"RESTORE `{coll_restore_1}`;") assert res_restore_when_exists.exit_code != 0, "Expected RESTORE to fail when target tables already exist" - # === Remove all tables from DB (orders, products, extras) === - self._drop_tables([t1, t2, "extra_table_1", "extra_table_2"]) # ignore errors + # Remove all tables from DB (orders, products, extras) + self._drop_tables([t1, t2, "extra_table_2"]) - # === Now RESTORE coll_restore_1 (which corresponds to backup1) === + # Now RESTORE coll_restore_1 (which corresponds to backup1) res_restore1 = self._execute_yql(f"RESTORE `{coll_restore_1}`;") assert res_restore1.exit_code == 0, f"RESTORE v1 failed: {res_restore1.std_err or res_restore1.std_out}" # verify schema/data/acl for backup1 # verify data - self._verify_restored_table_data(t1, snapshot_wave1_t1) - self._verify_restored_table_data(t2, snapshot_wave1_t2) + self._verify_restored_table_data(t1, snapshot_stage1_t1) + self._verify_restored_table_data(t2, snapshot_stage1_t2) # verify schema restored_schema_t1 = self._capture_schema(f"/Root/{t1}") restored_schema_t2 = self._capture_schema(f"/Root/{t2}") - assert restored_schema_t1 == schema_wave1_t1, f"Schema for {t1} after restore v1 differs: expected {schema_wave1_t1}, got {restored_schema_t1}" - assert restored_schema_t2 == schema_wave1_t2, f"Schema for {t2} after restore v1 differs: expected {schema_wave1_t2}, got {restored_schema_t2}" + assert restored_schema_t1 == schema_stage1_t1, f"Schema for {t1} after restore v1 differs: expected {schema_stage1_t1}, got {restored_schema_t1}" + assert restored_schema_t2 == schema_stage1_t2, f"Schema for {t2} after restore v1 differs: expected {schema_stage1_t2}, got {restored_schema_t2}" # verify acl restored_acl_t1 = self._capture_acl(f"/Root/{t1}") restored_acl_t2 = self._capture_acl(f"/Root/{t2}") # We compare that SHOW GRANTS output contains previously stored show_grants if present - if 'show_grants' in (acl_wave1_t1 or {}): - assert 'show_grants' in (restored_acl_t1 or {}) and acl_wave1_t1['show_grants'] in restored_acl_t1['show_grants'] - if 'show_grants' in (acl_wave1_t2 or {}): - assert 'show_grants' in (restored_acl_t2 or {}) and acl_wave1_t2['show_grants'] in restored_acl_t2['show_grants'] + if 'show_grants' in (acl_stage1_t1 or {}): + assert 'show_grants' in (restored_acl_t1 or {}) and acl_stage1_t1['show_grants'] in restored_acl_t1['show_grants'] + if 'show_grants' in (acl_stage1_t2 or {}): + assert 'show_grants' in (restored_acl_t2 or {}) and acl_stage1_t2['show_grants'] in restored_acl_t2['show_grants'] # === Remove all tables again and restore backup2 === - self._drop_tables([t1, t2, "extra_table_1", "extra_table_2"]) # ignore errors + self._drop_tables([t1, t2]) # ignore errors res_restore2 = self._execute_yql(f"RESTORE `{coll_restore_2}`;") assert res_restore2.exit_code == 0, f"RESTORE v2 failed: {res_restore2.std_err or res_restore2.std_out}" # verify data/schema/acl for backup2 - self._verify_restored_table_data(t1, snapshot_wave2_t1) - self._verify_restored_table_data(t2, snapshot_wave2_t2) + self._verify_restored_table_data(t1, snapshot_stage2_t1) + self._verify_restored_table_data(t2, snapshot_stage2_t2) restored_schema2_t1 = self._capture_schema(f"/Root/{t1}") restored_schema2_t2 = self._capture_schema(f"/Root/{t2}") - assert restored_schema2_t1 == schema_wave2_t1, f"Schema for {t1} after restore v2 differs: expected {schema_wave2_t1}, got {restored_schema2_t1}" - assert restored_schema2_t2 == schema_wave2_t2, f"Schema for {t2} after restore v2 differs: expected {schema_wave2_t2}, got {restored_schema2_t2}" + assert restored_schema2_t1 == schema_stage2_t1, f"Schema for {t1} after restore v2 differs: expected {schema_stage2_t1}, got {restored_schema2_t1}" + assert restored_schema2_t2 == schema_stage2_t2, f"Schema for {t2} after restore v2 differs: expected {schema_stage2_t2}, got {restored_schema2_t2}" restored_acl2_t1 = self._capture_acl(f"/Root/{t1}") restored_acl2_t2 = self._capture_acl(f"/Root/{t2}") - if 'show_grants' in (acl_wave2_t1 or {}): - assert 'show_grants' in (restored_acl2_t1 or {}) and acl_wave2_t1['show_grants'] in restored_acl2_t1['show_grants'] - if 'show_grants' in (acl_wave2_t2 or {}): - assert 'show_grants' in (restored_acl2_t2 or {}) and acl_wave2_t2['show_grants'] in restored_acl2_t2['show_grants'] + if 'show_grants' in (acl_stage2_t1 or {}): + assert 'show_grants' in (restored_acl2_t1 or {}) and acl_stage2_t1['show_grants'] in restored_acl2_t1['show_grants'] + if 'show_grants' in (acl_stage2_t2 or {}): + assert 'show_grants' in (restored_acl2_t2 or {}) and acl_stage2_t2['show_grants'] in restored_acl2_t2['show_grants'] # cleanup exported data if os.path.exists(export_dir): From fa7763d2dc47ca8ff0c28f460a43f212c01fe2cf Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Wed, 15 Oct 2025 19:23:29 +0000 Subject: [PATCH 17/18] add wrong behavior --- .../backup_collection/basic_user_scenarios.py | 128 +++++++++++++++--- 1 file changed, 107 insertions(+), 21 deletions(-) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index b66790ec41eb..5fc20906c194 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -35,31 +35,108 @@ def is_system_object(obj): def sdk_select_table_rows(session, table, path_prefix="/Root"): - sql = f'PRAGMA TablePathPrefix("{path_prefix}"); SELECT id, number, txt FROM {table} ORDER BY id;' + if table.startswith("/"): + full_path = table + base_name = os.path.basename(table) + table_for_sql = base_name + pp = os.path.dirname(full_path) or path_prefix + else: + base_name = table + full_path = os.path.join(path_prefix, base_name) + table_for_sql = base_name + pp = path_prefix + + cols = None + primary_keys = None + try: + if hasattr(session, "describe_table"): + desc = session.describe_table(full_path) + else: + tc = getattr(getattr(session, "driver", None), "table_client", None) + if tc is not None and hasattr(tc, "describe_table"): + desc = tc.describe_table(full_path) + else: + desc = None + + if desc is not None: + raw_cols = getattr(desc, "columns", None) or getattr(desc, "Columns", None) + if raw_cols: + try: + cols = [c.name for c in raw_cols] + except Exception: + cols = [str(c) for c in raw_cols] + + pk = getattr(desc, "primary_key", None) or getattr(desc, "primary_keys", None) or getattr(desc, "key_columns", None) + if pk: + try: + if isinstance(pk, (list, tuple)): + primary_keys = list(pk) + else: + primary_keys = [str(pk)] + except Exception: + primary_keys = None + except Exception: + cols = None + primary_keys = None + + if not cols: + try: + sql_try = f'PRAGMA TablePathPrefix("{pp}"); SELECT * FROM {table_for_sql} LIMIT 1;' + res_try = session.transaction().execute(sql_try, commit_tx=True) + rs0 = res_try[0] + try: + meta = getattr(rs0, "columns", None) or getattr(rs0, "Columns", None) + if meta: + cols = [c.name for c in meta] + except Exception: + cols = None + except Exception: + cols = None + + if not cols: + raise AssertionError(f"Не удалось получить список колонок для таблицы {full_path}") + + def q(n): + return "`" + n.replace("`", "``") + "`" + + select_list = ", ".join(q(c) for c in cols) + + order_clause = "" + if primary_keys: + pks = [p for p in primary_keys if p in cols] + if pks: + order_clause = " ORDER BY " + ", ".join(q(p) for p in pks) + + sql = f'PRAGMA TablePathPrefix("{pp}"); SELECT {select_list} FROM {table_for_sql}{order_clause};' + result_sets = session.transaction().execute(sql, commit_tx=True) rows = [] - rows.append(["id", "number", "txt"]) + rows.append(cols.copy()) for r in result_sets[0].rows: - try: - idv = r.id - except Exception: - idv = r[0] - try: - numv = r.number - except Exception: - numv = r[1] - try: - txtv = r.txt - except Exception: - txtv = r[2] + vals = [] + for i, col in enumerate(cols): + v = None + try: + v = getattr(r, col) + except Exception: + try: + v = r[i] + except Exception: + v = None - rows.append([ - str(idv) if idv is not None else "", - str(numv) if numv is not None else "", - txtv if txtv is not None else "", - ]) + if v is None: + vals.append("") + else: + try: + if isinstance(v, (bytes, bytearray)): + vals.append(v.decode("utf-8", "replace")) + else: + vals.append(str(v)) + except Exception: + vals.append(repr(v)) + rows.append(vals) return rows @@ -1240,6 +1317,16 @@ def q(role: str) -> str: # add more tables create_table_with_data(session, "extra_table_2") + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + # # remove table under backup collection -> except fail + # # but the current version allows you to delete + # # a table included in backup collection + # try: + # session.execute_scheme('DROP TABLE `/Root/products`;') + # except Exception: + # raise AssertionError("DROP failed") + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + # remove some tables from step5: drop extra_table_1 try: session.execute_scheme('DROP TABLE `/Root/extra_table_1`;') @@ -1252,7 +1339,6 @@ def q(role: str) -> str: except Exception: raise AssertionError("ADD COLUMN failed") - # attempt alter column (if supported) - many builds do not support complex ALTER, so guard with try try: session.execute_scheme('ALTER TABLE `/Root/orders` SET (TTL = Interval("PT0S") ON expire_at);') except Exception: @@ -1260,7 +1346,7 @@ def q(role: str) -> str: # drop columns try: - session.execute_scheme('ALTER TABLE `/Root/orders` DROP COLUMN new_col;') + session.execute_scheme('ALTER TABLE `/Root/orders` DROP COLUMN number;') except Exception: raise AssertionError("DROP COLUMN failed") From 3cf7d496d6a592278e21e629bd0b19e9d4feae4d Mon Sep 17 00:00:00 2001 From: Lebedev Oleg Date: Thu, 16 Oct 2025 10:50:38 +0000 Subject: [PATCH 18/18] wrong alter behavior --- .../backup_collection/basic_user_scenarios.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/ydb/tests/functional/backup_collection/basic_user_scenarios.py b/ydb/tests/functional/backup_collection/basic_user_scenarios.py index 5fc20906c194..159e67301660 100644 --- a/ydb/tests/functional/backup_collection/basic_user_scenarios.py +++ b/ydb/tests/functional/backup_collection/basic_user_scenarios.py @@ -1317,34 +1317,25 @@ def q(role: str) -> str: # add more tables create_table_with_data(session, "extra_table_2") - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # # remove table under backup collection -> except fail - # # but the current version allows you to delete - # # a table included in backup collection - # try: - # session.execute_scheme('DROP TABLE `/Root/products`;') - # except Exception: - # raise AssertionError("DROP failed") - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # remove some tables from step5: drop extra_table_1 try: session.execute_scheme('DROP TABLE `/Root/extra_table_1`;') except Exception: raise AssertionError("DROP failed") - # add columns to initial tables + # add columns to initial tables -> except fail try: session.execute_scheme('ALTER TABLE `/Root/orders` ADD COLUMN new_col Uint32;') except Exception: raise AssertionError("ADD COLUMN failed") + # ALTER SET -> except fail try: session.execute_scheme('ALTER TABLE `/Root/orders` SET (TTL = Interval("PT0S") ON expire_at);') except Exception: raise AssertionError("SET TTL failed") - # drop columns + # drop columns -> except fail try: session.execute_scheme('ALTER TABLE `/Root/orders` DROP COLUMN number;') except Exception: