QGIS styles - set db and schema names

db/qgis-style/README.md

@@ -1 +1 @@
-Documentation moved to https://pgosm-flex.com
+Documentation moved to https://pgosm-flex.com/qgis-styles.html

docker/db.py

@@ -8,6 +8,7 @@
 import psycopg
 import sh
 
+import qgis_styles
 
 LOGGER = logging.getLogger('pgosm-flex')
 
@@ -181,14 +182,15 @@ def pg_isready():
     return True
 
 
-def prepare_pgosm_db(data_only, db_path, import_mode):
+def prepare_pgosm_db(data_only, db_path, import_mode, schema_name):
     """Runs through series of steps to prepare database for PgOSM.
 
     Parameters
     --------------------------
     data_only : bool
     db_path : str
     import_mode : import_mode.ImportMode
+    schema_name : str
     """
     if pg_conn_parts()['pg_host'] == 'localhost':
         drop_it = True
@@ -216,7 +218,9 @@ def prepare_pgosm_db(data_only, db_path, import_mode):
     if not data_only:
         LOGGER.info('Loading extras via Sqitch plus QGIS styles.')
         run_sqitch_prep(db_path)
-        load_qgis_styles(db_path)
+        qgis_styles.load_qgis_styles(db_path=db_path,
+                                     db_name=pg_conn_parts()['pg_db'],
+                                     schema_name=schema_name)
     else:
         LOGGER.info('Data only mode enabled, no Sqitch or QGIS styles.')
 
@@ -366,58 +370,6 @@ def run_sqitch_prep(db_path):
     return True
 
 
-def load_qgis_styles(db_path):
-    """Loads QGIS style data for easy formatting of most common layers.
-
-    Parameters
-    -------------------------
-    db_path : str
-    """
-    LOGGER.info('Load QGIS styles...')
-    # These two paths can easily be ran via psycopg
-    create_path = os.path.join(db_path,
-                               'qgis-style',
-                               'create_layer_styles.sql')
-    load_path = os.path.join(db_path,
-                             'qgis-style',
-                             '_load_layer_styles.sql')
-
-    with open(create_path, 'r') as file_in:
-        create_sql = file_in.read()
-
-    with open(load_path, 'r') as file_in:
-        load_sql = file_in.read()
-
-    with get_db_conn(conn_string=os.environ['PGOSM_CONN']) as conn:
-        cur = conn.cursor()
-        cur.execute(create_sql)
-    LOGGER.debug('QGIS Style table created')
-
-    # Loading layer_styles data is done from files created by pg_dump, using
-    # psql to reload is easiest
-    conn_string = os.environ['PGOSM_CONN']
-    cmds_populate = ['psql', '-d', conn_string,
-                     '-f', 'qgis-style/layer_styles.sql']
-
-    output = subprocess.run(cmds_populate,
-                            text=True,
-                            capture_output=True,
-                            cwd=db_path,
-                            check=False)
-
-    LOGGER.debug(f'Output from loading QGIS style data: {output.stdout}')
-
-    with get_db_conn(conn_string=os.environ['PGOSM_CONN']) as conn:
-        cur = conn.cursor()
-        cur.execute(load_sql)
-    LOGGER.info('QGIS Style table populated')
-
-    with get_db_conn(conn_string=os.environ['PGOSM_CONN']) as conn:
-        sql_clean = 'DELETE FROM public.layer_styles_staging;'
-        cur = conn.cursor()
-        cur.execute(sql_clean)
-        LOGGER.debug('QGIS Style staging table cleaned')
-
 
 def sqitch_db_string():
     """Returns DB string used for Sqitch.

docker/pgosm_flex.py

@@ -123,7 +123,8 @@ def run_pgosm_flex(ram, region, subregion, data_only, debug,
 
     db.prepare_pgosm_db(data_only=data_only,
                         db_path=paths['db_path'],
-                        import_mode=import_mode)
+                        import_mode=import_mode,
+                        schema_name=schema_name)
 
     if import_mode.replication_update:
         # If replication_update, a manual date is not valid.
@@ -141,7 +142,8 @@ def run_pgosm_flex(ram, region, subregion, data_only, debug,
                                          flex_path=paths['flex_path'],
                                          ram=ram,
                                          skip_nested=skip_nested,
-                                         import_mode=import_mode)
+                                         import_mode=import_mode,
+                                         debug=debug)
 
     if not success:
         msg = 'PgOSM Flex completed with errors. Details in output'
@@ -162,7 +164,7 @@ def run_pgosm_flex(ram, region, subregion, data_only, debug,
 
 
 def run_osm2pgsql_standard(input_file, out_path, flex_path, ram, skip_nested,
-                           import_mode):
+                           import_mode, debug):
     """Runs standard osm2pgsql command and optionally inits for replication
     (osm2pgsql-replication) mode.
 
@@ -174,6 +176,7 @@ def run_osm2pgsql_standard(input_file, out_path, flex_path, ram, skip_nested,
     ram : float
     skip_nested : boolean
     import_mode : import_mode.ImportMode
+    debug : boolean
 
     Returns
     ---------------------------
@@ -193,7 +196,8 @@ def run_osm2pgsql_standard(input_file, out_path, flex_path, ram, skip_nested,
                                                      out_path=out_path,
                                                      import_mode=import_mode)
 
-    run_osm2pgsql(osm2pgsql_command=osm2pgsql_command, flex_path=flex_path)
+    run_osm2pgsql(osm2pgsql_command=osm2pgsql_command, flex_path=flex_path,
+                  debug=debug)
 
     if not skip_nested:
         skip_nested = check_layerset_places(flex_path)
@@ -377,17 +381,23 @@ def get_export_full_path(out_path, export_filename):
     return export_path
 
 
-def run_osm2pgsql(osm2pgsql_command, flex_path):
+def run_osm2pgsql(osm2pgsql_command, flex_path, debug):
     """Runs the provided osm2pgsql command.
 
     Parameters
     ----------------------
     osm2pgsql_command : str
     flex_path : str
+    debug : boolean
     """
     logger = logging.getLogger('pgosm-flex')
     logger.info('Running osm2pgsql')
 
+    if debug:
+        print()
+        print(osm2pgsql_command)
+        print()
+
     returncode = helpers.run_command_via_subprocess(cmd=osm2pgsql_command.split(),
                                                     cwd=flex_path)
 

docker/qgis_styles.py

@@ -0,0 +1,130 @@
+"""PgOSM Flex module to handle loading QGIS styles to Postgres.
+"""
+import logging
+import os
+import subprocess
+
+import db
+
+
+LOGGER = logging.getLogger('pgosm-flex')
+
+
+def load_qgis_styles(db_path, db_name, schema_name):
+    """Loads QGIS style data for easy formatting of most common layers.
+
+    Parameters
+    -------------------------
+    db_path : str
+        Base path to pgosm-flex/db directory
+    db_name : str
+    schema_name : str
+    """
+    LOGGER.info(f'Load QGIS styles to database {db_name}...')
+    conn_string = os.environ['PGOSM_CONN']
+
+    create_layer_style_table(db_path=db_path, conn_string=conn_string)
+    populate_layer_style_staging(db_path=db_path, conn_string=conn_string)
+    update_styles_db_name(db_name=db_name, schema_name=schema_name,
+                          conn_string=conn_string)
+    load_staging_to_prod(db_path=db_path, conn_string=conn_string)
+
+
+def create_layer_style_table(db_path, conn_string):
+    """Ensures QGIS layer styles table exists.
+ 
+    Parameters
+    --------------------
+    db_path : str
+    conn_string : path
+    """
+    create_path = os.path.join(db_path,
+                               'qgis-style',
+                               'create_layer_styles.sql')
+
+    with open(create_path, 'r') as file_in:
+        create_sql = file_in.read()
+
+    with db.get_db_conn(conn_string=conn_string) as conn:
+        cur = conn.cursor()
+        cur.execute(create_sql)
+    LOGGER.debug('QGIS Style table created')
+
+
+def populate_layer_style_staging(db_path, conn_string):
+    """Loads data to public.layer_styles_staging using psql
+
+    Parameters
+    --------------------
+    db_path : str
+    conn_string : path
+    """
+    # Loading layer_styles data is done from files created by pg_dump, using
+    # psql to reload is easiest
+    cmds_populate = ['psql', '-d', conn_string,
+                     '-f', 'qgis-style/layer_styles.sql']
+
+    output = subprocess.run(cmds_populate,
+                            text=True,
+                            capture_output=True,
+                            cwd=db_path,
+                            check=False)
+
+    LOGGER.debug(f'Output from loading QGIS style data: {output.stdout}')
+
+
+def load_staging_to_prod(db_path, conn_string):
+    """Loads data from public.layer_styles_staging to public.layer_styles.
+
+    Parameters
+    --------------------
+    db_path : str
+    conn_string : path
+    """
+    load_path = os.path.join(db_path,
+                             'qgis-style',
+                             '_load_layer_styles.sql')
+
+    with open(load_path, 'r') as file_in:
+        load_sql = file_in.read()
+
+    with db.get_db_conn(conn_string=conn_string) as conn:
+        cur = conn.cursor()
+        cur.execute(load_sql)
+
+    LOGGER.info('QGIS Style table populated')
+
+    with db.get_db_conn(conn_string=conn_string) as conn:
+        sql_clean = 'DELETE FROM public.layer_styles_staging;'
+        cur = conn.cursor()
+        cur.execute(sql_clean)
+
+    LOGGER.debug('QGIS Style staging table cleaned')
+
+
+def update_styles_db_name(db_name, schema_name, conn_string):
+    """
+    Parameters
+    ----------------------
+    db_name : str
+    schema_name : str
+    conn_string : str
+    """
+    if db_name == 'pgosm' and schema_name == 'osm':
+        LOGGER.debug('Database name and schema name set to defaults. Update to layer styles not necessary')
+        return
+
+    sql_raw = """
+UPDATE public.layer_styles_staging
+    SET f_table_catalog = %(db_name)s ,
+        f_table_schema = %(schema_name)s
+;
+"""
+    params = {'db_name': db_name, 'schema_name': schema_name}
+    with db.get_db_conn(conn_string=conn_string) as conn:
+        cur = conn.cursor()
+        cur.execute(sql_raw, params=params)
+        conn.commit()
+
+    LOGGER.info(f'Updated QGIS layer styles for {db_name}.{schema_name}')
+

docker/tests/test_geofabrik.py

@@ -1,12 +1,15 @@
 """ Unit tests to cover the Geofabrik module."""
 import unittest
+import uuid
+
 import geofabrik, helpers
 
 REGION_US = 'north-america/us'
 SUBREGION_DC = 'district-of-columbia'
 LAYERSET = 'default'
 PGOSM_DATE = '2021-12-02'
 
+IMPORT_UUID = uuid.uuid4()
 
 class GeofabrikTests(unittest.TestCase):
 
@@ -18,7 +21,9 @@ def setUp(self):
                              pgosm_date=PGOSM_DATE,
                              layerset=LAYERSET,
                              layerset_path=None,
-                             sp_gist=False)
+                             sp_gist=False,
+                             replication=False,
+                             import_uuid=IMPORT_UUID)
 
 
     def tearDown(self):
@@ -39,7 +44,9 @@ def test_get_region_filename_returns_region_when_subregion_None(self):
                              pgosm_date=PGOSM_DATE,
                              layerset=LAYERSET,
                              layerset_path=None,
-                             sp_gist=False)
+                             sp_gist=False,
+                             replication=False,
+                             import_uuid=IMPORT_UUID)
 
         result = geofabrik.get_region_filename()
         expected = f'{REGION_US}-latest.osm.pbf'

docker/tests/test_pgosm_flex.py

@@ -1,12 +1,15 @@
 """ Unit tests to cover the DB module."""
 import unittest
+import uuid
+
 import pgosm_flex, helpers
 
+
 REGION_US = 'north-america/us'
 SUBREGION_DC = 'district-of-columbia'
 LAYERSET = 'default'
 PGOSM_DATE = '2021-12-02'
-
+IMPORT_UUID = uuid.uuid4()
 
 class PgOSMFlexTests(unittest.TestCase):
 
@@ -18,7 +21,9 @@ def setUp(self):
                              pgosm_date=PGOSM_DATE,
                              layerset=LAYERSET,
                              layerset_path=None,
-                             sp_gist=False)
+                             sp_gist=False,
+                             replication=False,
+                             import_uuid=IMPORT_UUID)
 
 
     def tearDown(self):
@@ -89,7 +94,9 @@ def test_get_export_filename_region_only(self):
                              pgosm_date=PGOSM_DATE,
                              layerset=LAYERSET,
                              layerset_path=None,
-                             sp_gist=False)
+                             sp_gist=False,
+                             replication=False,
+                             import_uuid=IMPORT_UUID)
 
         input_file = None
         result = pgosm_flex.get_export_filename(input_file)

docs/src/SUMMARY.md

@@ -29,4 +29,5 @@
 - [Build and Push Docker Images](./docker-build.md)
 - [Testing PgOSM Flex](./tests.md)
 - [Developing QGIS Styles](./qgis-styles-dev.md)
+- [Troubleshoot errors in osm2pgsql processing](./troubleshooting.md)