fix #167 fix #146 fix #147 - geojson: autotransform, autodetect, fix bug

datacity_ckan_dgp/ckan/__init__.py

@@ -145,3 +145,9 @@ def automation_group_set(instance_name, group_name, key, value):
         group_update(instance_name, group)
     else:
         group_create(instance_name, group_name, group_type='automation', json=json.dumps({key: value}))
+
+
+def datastore_info(instance_name, resource_id):
+    res = api_post(instance_name, 'datastore_info', json={'id': resource_id})
+    assert res['success'], res
+    return res['result']

datacity_ckan_dgp/package_processing_tasks/geojson.py

@@ -2,6 +2,7 @@
 import sys
 from decimal import Decimal
 
+import pyproj
 import geojson
 import dataflows as DF
 from geojson import Feature, Point, FeatureCollection
@@ -11,10 +12,45 @@
 from datacity_ckan_dgp.package_processing_tasks import common
 
 
+LAT_LON_FIELD_NAMES = (
+    ('lat', 'lon'),
+    ('lat', 'long'),
+    ('latitude', 'longtitude'),
+    ('y', 'x'),
+    ('y_itm', 'x_itm'),
+    ('n_ord', 'e_ord'),
+)
+
+
+CRS = '+ellps=GRS80 +k=1.00007 +lat_0=31.73439361111111 +lon_0=35.20451694444445 +no_defs +proj=tmerc +units=m +x_0=219529.584 +y_0=626907.39'
+projector = pyproj.Proj(CRS)
+
+
+def pop_case_insensitive(row, field, default=None):
+    value = row.get(field)
+    if value is not None:
+        return row.pop(field)
+    else:
+        for k, v in row.items():
+            if k.lower() == field.lower():
+                return row.pop(k)
+    return default
+
+
+def get_lat_lon_values(row, lon_field, lat_field):
+    lon = float(pop_case_insensitive(row, lon_field))
+    lat = float(pop_case_insensitive(row, lat_field))
+    if lon > 200 and lat > 200:
+        lon, lat = projector(lon, lat, inverse=True)
+    return lon, lat
+
+
 def process_resource(instance_name, package, resource, package_extras_processed_res):
+    lat_field = resource.get("geo_lat_field")
+    lon_field = resource.get("geo_lon_field")
     rows = DF.Flow(DF.load(resource['url'])).results()[0][0]
     fc = FeatureCollection([
-        Feature(geometry=Point((float(r.pop('lon')), float(r.pop('lat')))),
+        Feature(geometry=Point(get_lat_lon_values(r, lon_field, lat_field)),
                 properties=dict((k, float(v) if isinstance(v, Decimal) else v) for k, v in r.items()))
         for r in rows
     ])
@@ -32,7 +68,19 @@ def process_resource(instance_name, package, resource, package_extras_processed_
 
 
 def is_resource_valid_for_processing(instance_name, package, resource):
-    return resource.get("geo_lat_field") and resource.get("geo_lon_field") and resource.get('format') == 'CSV'
+    if resource.get('geo_lat_field') and resource.get('geo_lon_field'):
+        return True
+    elif resource.get('datastore_active'):
+        datastore_info = ckan.datastore_info(instance_name, resource['id'])
+        schema_fields = {f.lower().strip(): f for f in datastore_info.get('schema', {})}
+        for lat_field, lon_field in LAT_LON_FIELD_NAMES:
+            if lat_field in schema_fields and lon_field in schema_fields:
+                resource['geo_lat_field'] = schema_fields[lat_field]
+                resource['geo_lon_field'] = schema_fields[lon_field]
+                return True
+        return False
+    else:
+        return False
 
 
 def process_package(instance_name, package_id):

tests/requirements.txt

@@ -0,0 +1 @@
+pytest==6.2.1

tests/test_geo.py

@@ -0,0 +1,6 @@
+from datacity_ckan_dgp.package_processing_tasks import geojson
+
+
+def test_lat_lon():
+    row = {'X_ITM': '182418', 'Y_ITM': 652418}
+    assert geojson.get_lat_lon_values(row, 'X_ITM', 'Y_ITM') == (34.81195612163422, 31.963835962756367)