Merge pull request #142 from GIScience/fix/package-upgrades

fix: upgrade dependencies

.github/workflows/run_tests.yml

@@ -1,10 +1,14 @@
-name: run tests
+name: CI Tests
 
-on: [push, pull_request]
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
 
 jobs:
   run_tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - name: Check out source
       uses: actions/checkout@v2

docker-compose.yml

@@ -17,7 +17,8 @@ services:
   ports:
    - "5000:5000"
   depends_on:
-    - db
+    db:
+      condition: service_healthy
   mem_limit: 28g
   networks:
    - poi_network
@@ -38,6 +39,11 @@ services:
    - ALLOW_IP_RANGE=0.0.0.0/0
   ports:
    - 5432:5432
+  healthcheck:
+      test: ["CMD-SHELL", "pg_isready", "-d", "${POSTGRES_DB}"]
+      interval: 10s
+      timeout: 10s
+      retries: 10
   restart: on-failure
   networks:
    - poi_network
@@ -56,7 +62,8 @@ services:
   - ./ops_settings_docker.yml:/deploy/app/openpoiservice/server/ops_settings.yml
   - ./categories_docker.yml:/deploy/app/openpoiservice/server/categories/categories.yml
   depends_on:
-    - db
+    db:
+      condition: service_healthy
   mem_limit: 28g
   networks:
    - poi_network
@@ -74,7 +81,8 @@ services:
   - ./ops_settings_docker.yml:/deploy/app/openpoiservice/server/ops_settings.yml
   - ./categories_docker.yml:/deploy/app/openpoiservice/server/categories/categories.yml
   depends_on:
-    - db
+    db:
+      condition: service_healthy
   mem_limit: 28g
   networks:
    - poi_network
@@ -92,7 +100,8 @@ services:
    - ./ops_settings_docker.yml:/deploy/app/openpoiservice/server/ops_settings.yml
    - ./categories_docker.yml:/deploy/app/openpoiservice/server/categories/categories.yml
   depends_on:
-   - db
+    db:
+      condition: service_healthy
   mem_limit: 8g
   networks:
    - poi_network

manage.py

@@ -89,7 +89,7 @@ def import_data():
 
     logger.info(f"Starting to import OSM data ({len(osm_files)} files in batch)")
     logger.debug(f"Files in import batch: {osm_files}")
-    parser.run_import(osm_files, import_log)
+    parser.run_import(osm_files, import_log, db)
 
     with open(logfile, "w") as f:
         json.dump(import_log, f, indent=4, sort_keys=True)

openpoiservice/server/api/views.py

@@ -237,6 +237,9 @@ def parse_geometries(geometry):
         # parse geom if valid
         geojson_obj = check_validity(g2)
 
+        if geojson_obj.is_empty:
+            raise api_exceptions.InvalidUsage(status_code=400, error_code=4007, message=str("geojson is empty"))
+
         if geojson_obj.geom_type == 'Point':
             check_for_buffer(geometry, ops_settings['maximum_search_radius_for_points'])
 

openpoiservice/server/db_import/parser.py

@@ -71,17 +71,13 @@ def parse_file(osm_file, osm_file_index=0, update_mode=False):
 
 
 @timeit
-def run_import(osm_files_to_import, import_log):
+def run_import(osm_files_to_import, import_log, db_con):
     try:
         update_mode = False
-        # run query on separate database connection, will conflict otherwise since parse_import runs in separate process
-        separate_db_con = SQLAlchemy()
-        prev_poi_count = separate_db_con.session.query(POIs.osm_type, POIs.osm_id).count()
+        prev_poi_count = db_con.session.query(POIs.osm_type, POIs.osm_id).count()
         if prev_poi_count > 0:
             update_mode = True
             logger.info("Data import running in UPDATE MODE")
-        separate_db_con.session.remove()
-        separate_db_con.engine.dispose()
     except sqlalchemy.exc.ProgrammingError:
         logger.error("Database has not been initialized! Existing.")
         return
@@ -92,35 +88,29 @@ def run_import(osm_files_to_import, import_log):
             continue
 
         if update_mode:
-            separate_db_con = SQLAlchemy()
-            prev_poi_count_file = separate_db_con.session.query(POIs.osm_type, POIs.osm_id).filter_by(src_index=osm_file_index).count()
+            prev_poi_count_file = db_con.session.query(POIs.osm_type, POIs.osm_id).filter_by(src_index=osm_file_index).count()
             logger.info(f"Setting flags on {prev_poi_count_file} POIs.")
-            separate_db_con.session.query(POIs).filter_by(src_index=osm_file_index).update({POIs.delflag: True})
-            separate_db_con.session.commit()
-            separate_db_con.session.remove()
-            separate_db_con.engine.dispose()
+            db_con.session.query(POIs).filter_by(src_index=osm_file_index).update({POIs.delflag: True})
+            db_con.session.commit()
 
         if parse_file(osm_file, osm_file_index, update_mode=update_mode) == 0:
             import_log[osm_file] = os.path.getmtime(osm_file)
         else:
             import_log[osm_file] = 0  # import failed, this file has to be inserted in the next update
 
     if update_mode:
-        delete_marked_entries()
+        delete_marked_entries(db_con)
 
     logger.info(f"Import complete.")
 
 
 @timeit
-def delete_marked_entries():
+def delete_marked_entries(db_con):
     logger.info(f"Updates complete, now performing delete operations...")
-    separate_db_con = SQLAlchemy()
-    to_delete = separate_db_con.session.query(POIs.osm_type, POIs.osm_id).filter_by(delflag=True).count()
+    to_delete = db_con.session.query(POIs.osm_type, POIs.osm_id).filter_by(delflag=True).count()
     if to_delete > 0:
         logger.info(f"{to_delete} POIs in the database have been removed from the OSM data, deleting...")
-        separate_db_con.session.query(POIs).filter_by(delflag=True).delete()
-        separate_db_con.session.commit()
+        db_con.session.query(POIs).filter_by(delflag=True).delete()
+        db_con.session.commit()
     else:
         logger.info(f"No POIs marked for deletion, nothing to do.")
-    separate_db_con.session.remove()
-    separate_db_con.engine.dispose()

openpoiservice/server/utils/decorators.py

@@ -100,7 +100,7 @@ def wrapper(*args, **kwargs):
 
 def timeit(method):
     def timed(*args, **kw):
-        if len(args) == 0:
+        if len(args) == 1:
             operation = "Delete operations"
         else:
             operation = "File import" if isinstance(args[0], str) else f"Batch of {len(args[0])} files"

openpoiservice/tests/base.py

@@ -20,7 +20,7 @@ def setUp(self):
         db.session.remove()
         db.engine.dispose()
         test_file = os.path.join(os.getcwd() + '/osm', 'bremen-tests.osm.pbf')
-        parser.parse_file(test_file)
+        parser.parse_file(test_file, 0, False)
 
     def tearDown(self):
         db.session.remove()

openpoiservice/tests/test_update.py

@@ -4,6 +4,7 @@
 import json
 from base import BaseTestCase
 from openpoiservice.server.db_import import parser
+from openpoiservice.server import db
 
 request_poi_bbox = dict(
     request='pois',
@@ -18,7 +19,7 @@ def test_import_update_mode(self):
 
         print("======== Perform database update =========")
         updated_test_file = os.path.join(os.getcwd() + '/osm_test', 'bremen-tests-mod.osm.pbf')
-        parser.run_import([updated_test_file], {})
+        parser.run_import([updated_test_file], {}, db)
 
         response = self.client.post('/pois', data=json.dumps(request_poi_bbox), content_type='application/json')
         self.assertEqual(response.status_code, 200)

requirements.txt

@@ -1,58 +1,58 @@
-attrs==20.3.0
-Automat==20.2.0
-blinker==1.4
-certifi==2022.12.7
-cffi==1.14.4
-click==7.1.2
-constantly==15.1.0
-coverage==5.3.1
-cryptography>=3.0.0
-cssselect==1.1.0
-flasgger==0.9.5
-Flask==1.1.2
-Flask-Cors==3.0.10
-Flask-DebugToolbar==0.11.0
-Flask-SQLAlchemy==2.4.4
+attrs==23.2.0
+Automat==22.10.0
+blinker==1.7.0
+certifi==2024.2.2
+cffi==1.16.0
+click==8.1.7
+constantly==23.10.4
+coverage==7.4.1
+cryptography==42.0.3
+cssselect==1.2.0
+flasgger==0.9.7.1
+Flask==3.0.2
+Flask-Cors==4.0.0
+Flask-DebugToolbar==0.14.1
+Flask-SQLAlchemy==3.1.1
 Flask-Testing==0.8.1
-GeoAlchemy2==0.8.4
-geojson==2.5.0
-gevent==20.12.1
-greenlet==0.4.17
-gunicorn==20.0.4
-hyperlink==20.0.1
-idna==3.1
-incremental==17.5.0
-itemadapter>=0.1.0
-itemloaders>=1.0.0
-itsdangerous==1.1.0
-Jinja2==2.11.3
-jmespath==0.10.0
-jsonschema==3.2.0
-lxml==4.9.1
-MarkupSafe==1.1.1
-mistune==2.0.3
-parsel==1.6.0
-Protego==0.1.16
-psycopg2-binary==2.8.6
-pyasn1==0.4.8
-pyasn1-modules==0.2.8
-pycparser==2.20
-PyDispatcher==2.0.5
-PyHamcrest==2.0.2
-pyOpenSSL==20.0.1
-pyproj==3.1.0
-pyrsistent==0.17.3
+GeoAlchemy2==0.14.4
+geojson==3.1.0
+gevent==24.2.1
+greenlet==3.0.3
+gunicorn==21.2.0
+hyperlink==21.0.0
+idna==3.6
+incremental==22.10.0
+itemadapter==0.8.0
+itemloaders==1.1.0
+itsdangerous==2.1.2
+Jinja2==3.1.3
+jmespath==1.0.1
+jsonschema==4.21.1
+lxml==5.1.0
+MarkupSafe==2.1.5
+mistune==3.0.2
+parsel==1.8.1
+Protego==0.3.0
+psycopg2-binary==2.9.9
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pycparser==2.21
+PyDispatcher==2.0.7
+PyHamcrest==2.1.0
+pyOpenSSL==24.0.0
+pyproj==3.6.1
+pyrsistent==0.20.0
 PyYAML
-queuelib==1.5.0
-Scrapy>=2.3.0
-service-identity==18.1.0
-Shapely==1.7.1
-six==1.15.0
-SQLAlchemy==1.3.22
-Twisted==21.2.0
-voluptuous==0.12.1
-w3lib==1.22.0
-Werkzeug==1.0.1
-zope.event==4.5.0
-zope.interface==5.2.0
+queuelib==1.6.2
+Scrapy==2.11.1
+service-identity==24.1.0
+Shapely==2.0.3
+six==1.16.0
+SQLAlchemy==2.0.27
+Twisted==23.10.0
+voluptuous==0.14.2
+w3lib==2.1.2
+Werkzeug==3.0.1
+zope.event==5.0
+zope.interface==6.2
 git+https://github.com/timmccauley/imposm-parser.git@python3