From ed3dae3881ca6d3bc969a7cac0ddd5d0996d8e21 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 27 Oct 2025 08:29:44 +0100 Subject: [PATCH 1/3] use language attribute to detect multilanguage instead of nls table --- modelbaker/dbconnector/gpkg_connector.py | 16 ++++++++-------- modelbaker/dbconnector/mssql_connector.py | 10 +++++----- modelbaker/dbconnector/pg_connector.py | 10 +++++----- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/modelbaker/dbconnector/gpkg_connector.py b/modelbaker/dbconnector/gpkg_connector.py index 90781ff..5cfe944 100644 --- a/modelbaker/dbconnector/gpkg_connector.py +++ b/modelbaker/dbconnector/gpkg_connector.py @@ -1261,21 +1261,21 @@ def get_translation_handling(self) -> tuple[bool, str]: return self._table_exists(GPKG_NLS_TABLE) and self._lang != "", self._lang def get_available_languages(self, irrelevant_models=[]): - if not self._table_exists(GPKG_NLS_TABLE): + if not self._table_exists(GPKG_METAATTRS_TABLE): return [] cursor = self.conn.cursor() cursor.execute( """SELECT DISTINCT - lang - FROM "{t_ili2db_nls}" - WHERE - lang IS NOT NULL - AND - substr(iliElement, 0, instr(iliElement, '.')) NOT IN ({model_list}) + attr_value + FROM "{t_ili2db_meta_attrs}" + WHERE + attr_name = 'ili2db.ili.lang' + AND + ilielement NOT IN ({model_list}) ; """.format( - t_ili2db_nls=GPKG_NLS_TABLE, + t_ili2db_meta_attrs=GPKG_METAATTRS_TABLE, model_list=",".join( [f"'{modelname}'" for modelname in irrelevant_models] ), diff --git a/modelbaker/dbconnector/mssql_connector.py b/modelbaker/dbconnector/mssql_connector.py index cd61406..475d0b3 100644 --- a/modelbaker/dbconnector/mssql_connector.py +++ b/modelbaker/dbconnector/mssql_connector.py @@ -1261,17 +1261,17 @@ def get_translation_handling(self) -> tuple[bool, str]: return self._table_exists(NLS_TABLE) and self._lang != "", self._lang def get_available_languages(self, irrelevant_models=[]): - if self.schema and self._table_exists(NLS_TABLE): + if self.schema and self._table_exists(METAATTRS_TABLE): cur = self.conn.cursor() cur.execute( """ SELECT DISTINCT - lang - FROM {schema}.t_ili2db_nls + attr_value + FROM {schema}.t_ili2db_meta_attrs WHERE - lang IS NOT NULL + attr_name = 'ili2db.ili.lang' AND - left(iliElement, charindex('.', iliElement)-1) NOT IN ({model_list}) + ilielement NOT IN ({model_list}) """ ).format( schema=self.schema, diff --git a/modelbaker/dbconnector/pg_connector.py b/modelbaker/dbconnector/pg_connector.py index b22207a..2e7fef6 100644 --- a/modelbaker/dbconnector/pg_connector.py +++ b/modelbaker/dbconnector/pg_connector.py @@ -1397,18 +1397,18 @@ def get_translation_handling(self) -> tuple[bool, str]: return self._table_exists(PG_NLS_TABLE) and self._lang != "", self._lang def get_available_languages(self, irrelevant_models=[]): - if self.schema and self._table_exists(PG_NLS_TABLE): + if self.schema and self._table_exists(PG_METAATTRS_TABLE): cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) cur.execute( sql.SQL( """ SELECT DISTINCT - lang - FROM {schema}.t_ili2db_nls + attr_value + FROM {schema}.t_ili2db_meta_attrs WHERE - lang IS NOT NULL + attr_name = 'ili2db.ili.lang' AND - split_part(iliElement,'.',1) NOT IN ({model_list}) + ilielement NOT IN ({model_list}) """ ).format( schema=sql.Identifier(self.schema), From 2daf76421cd7587429c0f15cf8c86d3f933102d4 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 27 Oct 2025 09:56:35 +0100 Subject: [PATCH 2/3] test available models and translated models info request --- modelbaker/dbconnector/db_connector.py | 11 +++- modelbaker/dbconnector/gpkg_connector.py | 42 +++++++++++++-- modelbaker/dbconnector/mssql_connector.py | 40 ++++++++++++-- modelbaker/dbconnector/pg_connector.py | 45 +++++++++++++--- tests/test_translations.py | 65 +++++++++++++++++++++++ 5 files changed, 186 insertions(+), 17 deletions(-) diff --git a/modelbaker/dbconnector/db_connector.py b/modelbaker/dbconnector/db_connector.py index 1fef556..11259a8 100644 --- a/modelbaker/dbconnector/db_connector.py +++ b/modelbaker/dbconnector/db_connector.py @@ -449,9 +449,16 @@ def get_translation_handling(self) -> tuple[bool, str]: """ return False, "" - def get_available_languages(self, irrelevant_models: list[str]) -> list[str]: + def get_translation_models(self) -> list[str]: """ - Returns a list of available languages in the t_ili2db_nls table and ignores the values for the irrelevant models + Returns a list of models that are a TRANSLATION OF another model. + """ + return [] + + def get_available_languages(self, irrelevant_models: list[str], relevant_models: list[str]) -> list[str]: + """ + Returns a list of available languages in the t_ili2db_nls table and ignores the values for the irrelevant models. + If a list for relevant models is passed, only those are considered (otherwise all the others) """ return [] diff --git a/modelbaker/dbconnector/gpkg_connector.py b/modelbaker/dbconnector/gpkg_connector.py index 5cfe944..7d65511 100644 --- a/modelbaker/dbconnector/gpkg_connector.py +++ b/modelbaker/dbconnector/gpkg_connector.py @@ -1260,10 +1260,42 @@ def set_ili2db_sequence_value(self, value): def get_translation_handling(self) -> tuple[bool, str]: return self._table_exists(GPKG_NLS_TABLE) and self._lang != "", self._lang - def get_available_languages(self, irrelevant_models=[]): + def get_translation_models(self): if not self._table_exists(GPKG_METAATTRS_TABLE): return [] + cursor = self.conn.cursor() + cursor.execute( + """ + SELECT DISTINCT + ilielement + FROM "{t_ili2db_meta_attrs}" + WHERE + attr_name = 'ili2db.ili.translationOf' + ; + """.format( + t_ili2db_meta_attrs=GPKG_METAATTRS_TABLE, + ) + ) + records = cursor.fetchall() + cursor.close() + return [record["ilielement"] for record in records] + + def get_available_languages(self, irrelevant_models=[], relevant_models=[]): + if not self._table_exists(GPKG_METAATTRS_TABLE): + return [] + + white_list_restriction = '' + if len(relevant_models) > 0: + white_list_restriction = """ + AND + ilielement IN ({relevant_model_list}) + """.format( + relevant_model_list=",".join( + [f"'{modelname}'" for modelname in relevant_models] + ), + ) + cursor = self.conn.cursor() cursor.execute( """SELECT DISTINCT @@ -1272,18 +1304,20 @@ def get_available_languages(self, irrelevant_models=[]): WHERE attr_name = 'ili2db.ili.lang' AND - ilielement NOT IN ({model_list}) + ilielement NOT IN ({irrelevant_model_list}) + {white_list_restriction} ; """.format( t_ili2db_meta_attrs=GPKG_METAATTRS_TABLE, - model_list=",".join( + irrelevant_model_list=",".join( [f"'{modelname}'" for modelname in irrelevant_models] ), + white_list_restriction=white_list_restriction, ) ) records = cursor.fetchall() cursor.close() - return [record["lang"] for record in records] + return [record["attr_value"] for record in records] def get_domain_dispnames(self, tablename): if ( diff --git a/modelbaker/dbconnector/mssql_connector.py b/modelbaker/dbconnector/mssql_connector.py index 475d0b3..e109d31 100644 --- a/modelbaker/dbconnector/mssql_connector.py +++ b/modelbaker/dbconnector/mssql_connector.py @@ -1260,8 +1260,37 @@ def set_ili2db_sequence_value(self, value): def get_translation_handling(self) -> tuple[bool, str]: return self._table_exists(NLS_TABLE) and self._lang != "", self._lang - def get_available_languages(self, irrelevant_models=[]): + def get_translation_models(self): if self.schema and self._table_exists(METAATTRS_TABLE): + cur = self.conn.cursor() + cur.execute( + """ + SELECT DISTINCT + ilielement + FROM {schema}.t_ili2db_meta_attrs + WHERE + attr_name = 'ili2db.ili.translationOf' + """ + ).format( + schema=self.schema, + ) + return [row.ilielement for row in cur.fetchall()] + return [] + + def get_available_languages(self, irrelevant_models=[], relevant_models=[]): + if self.schema and self._table_exists(METAATTRS_TABLE): + + white_list_restriction = '' + if len(relevant_models) > 0: + white_list_restriction = """ + AND + ilielement IN ({relevant_model_list}) + """.format( + relevant_model_list=",".join( + [f"'{modelname}'" for modelname in relevant_models] + ), + ) + cur = self.conn.cursor() cur.execute( """ @@ -1271,14 +1300,15 @@ def get_available_languages(self, irrelevant_models=[]): WHERE attr_name = 'ili2db.ili.lang' AND - ilielement NOT IN ({model_list}) + ilielement NOT IN ({irrelevant_model_list}) + {white_list_restriction} """ ).format( schema=self.schema, - model_list=",".join( + irrelevant_model_list=",".join( [f"'{modelname}'" for modelname in irrelevant_models] ), + white_list_restriction=white_list_restriction, ) - - return [row.lang for row in cur.fetchall()] + return [row.attr_value for row in cur.fetchall()] return [] diff --git a/modelbaker/dbconnector/pg_connector.py b/modelbaker/dbconnector/pg_connector.py index 2e7fef6..1ada6b1 100644 --- a/modelbaker/dbconnector/pg_connector.py +++ b/modelbaker/dbconnector/pg_connector.py @@ -1396,8 +1396,40 @@ def get_all_schemas(self): def get_translation_handling(self) -> tuple[bool, str]: return self._table_exists(PG_NLS_TABLE) and self._lang != "", self._lang - def get_available_languages(self, irrelevant_models=[]): + def get_translation_models(self): if self.schema and self._table_exists(PG_METAATTRS_TABLE): + cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur.execute( + sql.SQL( + """ + SELECT DISTINCT + ilielement + FROM {schema}.t_ili2db_meta_attrs + WHERE + attr_name = 'ili2db.ili.translationOf' + """ + ).format( + schema=sql.Identifier(self.schema), + ) + ) + return [row["ilielement"] for row in cur.fetchall()] + return [] + + + def get_available_languages(self, irrelevant_models=[], relevant_models=[]): + if self.schema and self._table_exists(PG_METAATTRS_TABLE): + + white_list_restriction = '' + if len(relevant_models) > 0: + white_list_restriction = """ + AND + ilielement IN ({relevant_model_list}) + """.format( + relevant_model_list=",".join( + [f"'{modelname}'" for modelname in relevant_models] + ), + ) + cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) cur.execute( sql.SQL( @@ -1408,17 +1440,18 @@ def get_available_languages(self, irrelevant_models=[]): WHERE attr_name = 'ili2db.ili.lang' AND - ilielement NOT IN ({model_list}) + ilielement NOT IN ({irrelevant_model_list}) + {white_list_restriction} """ ).format( schema=sql.Identifier(self.schema), - model_list=sql.SQL(", ").join( + irrelevant_model_list=sql.SQL(", ").join( sql.Placeholder() * len(irrelevant_models) ), - ), - irrelevant_models, + white_list_restriction=white_list_restriction, + ) ) - return [row["lang"] for row in cur.fetchall()] + return [row["attr_value"] for row in cur.fetchall()] return [] def get_domain_dispnames(self, tablename): diff --git a/tests/test_translations.py b/tests/test_translations.py index 53d1de1..a7aae3d 100644 --- a/tests/test_translations.py +++ b/tests/test_translations.py @@ -27,6 +27,7 @@ from qgis.core import QgsProject from qgis.testing import start_app, unittest +import modelbaker.utils.db_utils as db_utils from modelbaker.dataobjects.project import Project from modelbaker.db_factory.gpkg_command_config_manager import GpkgCommandConfigManager from modelbaker.generator.generator import Generator @@ -230,6 +231,70 @@ def test_translated_db_objects_pg(self): == "Geometrie_Document_(Geometrie)_AffectationPrimaire_SurfaceDeZones_(t_id)" ) + def test_available_langs_gpkg(self): + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2gpkg + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilimodels = "PlansDAffectation_V1_2" + importer.configuration.dbfile = os.path.join( + self.basetestpath, "tmp_translated_gpkg.gpkg" + ) + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + assert importer.run() == iliimporter.Importer.SUCCESS + + db_connector = db_utils.get_db_connector(importer.configuration) + + # Translation handling is active + assert db_connector.get_translation_handling() + + # Get the translated models + assert {"PlansDAffectation_V1_2"} == set(db_connector.get_translation_models()) + + # Get all languages + assert {'en','de','fr'} == set(db_connector.get_available_languages()) + + # ... without irrelevant models + irrelevants = ["AdministrativeUnits_V1","AdministrativeUnitsCH_V1","Dictionaries_V1","DictionariesCH_V1"] + assert {'de','fr'} == set(db_connector.get_available_languages(irrelevants)) + + # ... and the language of the translated model only + assert {'fr'} == set(db_connector.get_available_languages([],["PlansDAffectation_V1_2"])) + + def test_translated_db_objects_pg(self): + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2pg + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilimodels = "PlansDAffectation_V1_2" + importer.configuration.dbschema = "tid_{:%Y%m%d%H%M%S%f}".format( + datetime.datetime.now() + ) + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + assert importer.run() == iliimporter.Importer.SUCCESS + + db_connector = db_utils.get_db_connector(importer.configuration) + + # Translation handling is active + assert db_connector.get_translation_handling() + + # Get the translated models + assert {"PlansDAffectation_V1_2"} == set(db_connector.get_translation_models()) + + # Get all languages + assert {'en','de','fr'} == set(db_connector.get_available_languages()) + + # ... without irrelevant models + irrelevants = ["AdministrativeUnits_V1","AdministrativeUnitsCH_V1","Dictionaries_V1","DictionariesCH_V1"] + assert {'de','fr'} == set(db_connector.get_available_languages(irrelevants)) + + # ... and the language of the translated model only + assert {'fr'} == set(db_connector.get_available_languages([],["PlansDAffectation_V1_2"])) + def print_info(self, text): logging.info(text) From e0ebf2f03bdb971f1b3433d3b922c9d480290570 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 27 Oct 2025 11:23:16 +0100 Subject: [PATCH 3/3] use sql placeholders properly --- modelbaker/dbconnector/gpkg_connector.py | 18 ++++++++---- modelbaker/dbconnector/mssql_connector.py | 18 ++++++++---- modelbaker/dbconnector/pg_connector.py | 36 +++++++++++++++-------- tests/test_translations.py | 12 ++++++++ 4 files changed, 59 insertions(+), 25 deletions(-) diff --git a/modelbaker/dbconnector/gpkg_connector.py b/modelbaker/dbconnector/gpkg_connector.py index 7d65511..f603a5c 100644 --- a/modelbaker/dbconnector/gpkg_connector.py +++ b/modelbaker/dbconnector/gpkg_connector.py @@ -1295,7 +1295,16 @@ def get_available_languages(self, irrelevant_models=[], relevant_models=[]): [f"'{modelname}'" for modelname in relevant_models] ), ) - + black_list_restriction = '' + if len(irrelevant_models) > 0: + black_list_restriction = """ + AND + ilielement NOT IN ({irrelevant_model_list}) + """.format( + irrelevant_model_list=",".join( + [f"'{modelname}'" for modelname in irrelevant_models] + ), + ) cursor = self.conn.cursor() cursor.execute( """SELECT DISTINCT @@ -1303,15 +1312,12 @@ def get_available_languages(self, irrelevant_models=[], relevant_models=[]): FROM "{t_ili2db_meta_attrs}" WHERE attr_name = 'ili2db.ili.lang' - AND - ilielement NOT IN ({irrelevant_model_list}) + {black_list_restriction} {white_list_restriction} ; """.format( t_ili2db_meta_attrs=GPKG_METAATTRS_TABLE, - irrelevant_model_list=",".join( - [f"'{modelname}'" for modelname in irrelevant_models] - ), + black_list_restriction=black_list_restriction, white_list_restriction=white_list_restriction, ) ) diff --git a/modelbaker/dbconnector/mssql_connector.py b/modelbaker/dbconnector/mssql_connector.py index e109d31..a4f15a1 100644 --- a/modelbaker/dbconnector/mssql_connector.py +++ b/modelbaker/dbconnector/mssql_connector.py @@ -1290,7 +1290,16 @@ def get_available_languages(self, irrelevant_models=[], relevant_models=[]): [f"'{modelname}'" for modelname in relevant_models] ), ) - + black_list_restriction = '' + if len(irrelevant_models) > 0: + black_list_restriction = """ + AND + ilielement NOT IN ({irrelevant_model_list}) + """.format( + irrelevant_model_list=",".join( + [f"'{modelname}'" for modelname in irrelevant_models] + ), + ) cur = self.conn.cursor() cur.execute( """ @@ -1299,15 +1308,12 @@ def get_available_languages(self, irrelevant_models=[], relevant_models=[]): FROM {schema}.t_ili2db_meta_attrs WHERE attr_name = 'ili2db.ili.lang' - AND - ilielement NOT IN ({irrelevant_model_list}) + {black_list_restriction} {white_list_restriction} """ ).format( schema=self.schema, - irrelevant_model_list=",".join( - [f"'{modelname}'" for modelname in irrelevant_models] - ), + black_list_restriction=black_list_restriction, white_list_restriction=white_list_restriction, ) return [row.attr_value for row in cur.fetchall()] diff --git a/modelbaker/dbconnector/pg_connector.py b/modelbaker/dbconnector/pg_connector.py index 1ada6b1..527dba9 100644 --- a/modelbaker/dbconnector/pg_connector.py +++ b/modelbaker/dbconnector/pg_connector.py @@ -1419,16 +1419,28 @@ def get_translation_models(self): def get_available_languages(self, irrelevant_models=[], relevant_models=[]): if self.schema and self._table_exists(PG_METAATTRS_TABLE): - white_list_restriction = '' + white_list_placeholders = sql.SQL('') if len(relevant_models) > 0: - white_list_restriction = """ + white_list_placeholders = sql.SQL(""" AND ilielement IN ({relevant_model_list}) - """.format( - relevant_model_list=",".join( - [f"'{modelname}'" for modelname in relevant_models] + """ + ).format( + relevant_model_list=sql.SQL(", ").join( + sql.Placeholder() * len(relevant_models) ), ) + black_list_placeholders = sql.SQL('') + if len(irrelevant_models) > 0: + black_list_placeholders = sql.SQL(""" + AND + ilielement NOT IN ({irrelevant_model_list}) + """ + ).format( + irrelevant_model_list=sql.SQL(", ").join( + sql.Placeholder() * len(irrelevant_models) + ) + ) cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) cur.execute( @@ -1439,17 +1451,15 @@ def get_available_languages(self, irrelevant_models=[], relevant_models=[]): FROM {schema}.t_ili2db_meta_attrs WHERE attr_name = 'ili2db.ili.lang' - AND - ilielement NOT IN ({irrelevant_model_list}) - {white_list_restriction} + {white_list_placeholders} + {black_list_placeholders} """ ).format( schema=sql.Identifier(self.schema), - irrelevant_model_list=sql.SQL(", ").join( - sql.Placeholder() * len(irrelevant_models) - ), - white_list_restriction=white_list_restriction, - ) + white_list_placeholders=white_list_placeholders, + black_list_placeholders=black_list_placeholders, + ), + relevant_models+irrelevant_models ) return [row["attr_value"] for row in cur.fetchall()] return [] diff --git a/tests/test_translations.py b/tests/test_translations.py index a7aae3d..432d26c 100644 --- a/tests/test_translations.py +++ b/tests/test_translations.py @@ -263,6 +263,12 @@ def test_available_langs_gpkg(self): # ... and the language of the translated model only assert {'fr'} == set(db_connector.get_available_languages([],["PlansDAffectation_V1_2"])) + # --- and nonsense use case for the validation, get only of an english model + assert {'en'} == set(db_connector.get_available_languages([],["AdministrativeUnits_V1"])) + + # ... as well as ignoring the translated models and alowing it again and the english one + assert {'de','en'} == set(db_connector.get_available_languages(["PlansDAffectation_V1_2"])) + def test_translated_db_objects_pg(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2pg @@ -295,6 +301,12 @@ def test_translated_db_objects_pg(self): # ... and the language of the translated model only assert {'fr'} == set(db_connector.get_available_languages([],["PlansDAffectation_V1_2"])) + # --- and nonsense use case for the validation, get only of an english model + assert {'en'} == set(db_connector.get_available_languages([],["AdministrativeUnits_V1"])) + + # ... as well as ignoring the translated models and alowing it again and the english one + assert {'de','en'} == set(db_connector.get_available_languages(["PlansDAffectation_V1_2"])) + def print_info(self, text): logging.info(text)