Merge pull request #371 from rustprooflabs/improve-version-checks

Add check against prior PgOSM Flex version with replication

.gitignore

@@ -4,3 +4,4 @@ output/
 **/__pycache__
 pgosm-data/*
 docs/book/*
+.vscode/*

docker/db.py

@@ -656,7 +656,8 @@ def get_prior_import(schema_name: str) -> dict:
 SELECT id, osm_date, region, layerset, import_status,
         import_mode ->> 'replication' AS replication,
         import_mode ->> 'update' AS use_update,
-        import_mode
+        import_mode,
+        split_part(pgosm_flex_version, '-', 1) AS pgosm_flex_version_no_hash
     FROM {schema_name}.pgosm_flex
     ORDER BY imported DESC
     LIMIT 1

docker/helpers.py

@@ -166,27 +166,45 @@ def get_region_combined(region: str, subregion: str) -> str:
     return pgosm_region
 
 
-def get_git_info() -> str:
+def get_git_info(tag_only: bool=False) -> str:
     """Provides git info in the form of the latest tag and most recent short sha
 
     Sends info to logger and returns string.
 
+    Parameters
+    ----------------------
+    tag_only : bool
+        When true, omits the short sha portion, only returning the tag.
+
     Returns
     ----------------------
     git_info : str
     """
     logger = logging.getLogger('pgosm-flex')
-    repo = git.Repo()
+
+    try:
+        repo = git.Repo()
+    except git.exc.InvalidGitRepositoryError:
+        # This error happens when running via make for some reason...
+        # This appears to fix it.
+        repo = git.Repo('../')
+
     try:
         sha = repo.head.object.hexsha
         short_sha = repo.git.rev_parse(sha, short=True)
         latest_tag = repo.git.describe('--abbrev=0', tags=True)
-        git_info = f'{latest_tag}-{short_sha}'
     except ValueError:
         git_info = 'Git info unavailable'
         logger.error('Unable to get git information.')
+        return '-- (version unknown) --'
+
+    if tag_only:
+        git_info = latest_tag
+    else:
+        git_info = f'{latest_tag}-{short_sha}'
+        # Logging only this full version, not the tag_only run
+        logger.info(f'PgOSM Flex version:  {git_info}')
 
-    logger.info(f'PgOSM Flex version:  {git_info}')
     return git_info
 
 

docker/import_mode.py

@@ -2,6 +2,9 @@
 """
 import logging
 import json
+from packaging.version import parse as parse_version
+
+import helpers
 
 
 class ImportMode():
@@ -83,6 +86,24 @@ def okay_to_run(self, prior_import: dict) -> bool:
 
         prior_replication = prior_import['replication']
 
+        # Check git version against latest.
+        # If current version is lower than prior version from latest import, stop.
+        prior_import_version = prior_import['pgosm_flex_version_no_hash']
+        git_tag = helpers.get_git_info(tag_only=True)
+
+        if git_tag == '-- (version unknown) --':
+            msg = 'Unable to detect PgOSM Flex version from Git.'
+            msg += ' Not enforcing version check against prior version.'
+            self.logger.warning(msg)
+        elif parse_version(git_tag) < parse_version(prior_import_version):
+            msg = f'PgOSM Flex version ({git_tag}) is lower than latest import'
+            msg += f' tracked in the pgosm_flex table ({prior_import_version}).'
+            msg += f' Use PgOSM Flex version {prior_import_version} or newer'
+            self.logger.error(msg)
+            return False
+        else:
+            self.logger.info(f'Prior import used PgOSM Flex: {prior_import_version}')
+
         if self.replication:
             if not prior_replication:
                 self.logger.error('Running w/ replication but prior import did not.  Requires --force to proceed.')

docker/tests/test_import_mode.py

@@ -146,7 +146,9 @@ def test_import_mode_okay_to_run_returns_false_when_prior_record_not_replication
         This should return False to avoid overwriting data.
         """
         replication = True
-        prior_import = {'replication': False}
+        prior_import = {'replication': False,
+                        'pgosm_flex_version_no_hash': '99.99.99'
+                        }
         replication_update = False
         update = None
         force = False
@@ -170,7 +172,9 @@ def test_import_mode_okay_to_run_returns_true_when_replication_prior_record_repl
         This should return True to allow replication to updated
         """
         replication = True
-        prior_import = {'replication': True}
+        prior_import = {'replication': True,
+                        'pgosm_flex_version_no_hash': '99.99.99'
+                        }
         replication_update = False
         update = None
         force = False
@@ -193,7 +197,9 @@ def test_import_mode_okay_to_run_returns_false_when_prior_import(self):
         This should return False to protect the data.
         """
         replication = False
-        prior_import = {'replication': False}
+        prior_import = {'replication': False,
+                        'pgosm_flex_version_no_hash': '99.99.99'
+                        }
         replication_update = False
         update = None
         force = False

requirements.txt

@@ -3,6 +3,7 @@ coverage>=6.4.1
 GitPython>=3.1.31
 osm2pgsql-tuner==0.0.6
 osmium>=3.4.1
+packaging>=23.0
 psycopg>=3.1
 psycopg-binary>=3.1
 sh>=1.14.2