diff --git a/.travis.yml b/.travis.yml index 2e63f66b0..95a472200 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,10 +8,17 @@ python: services: - elasticsearch + - postgresql + +addons: + postgresql: 9.6 + before_install: - sudo apt-get -qq update - sudo apt-get install -y libsqlite3-mod-spatialite pandoc devscripts + - sudo apt-get install -y postgresql-9.6-postgis-2.4 + install: - pip install -r requirements.txt @@ -25,6 +32,9 @@ before_script: - sleep 20 - python tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson - pygeoapi generate-openapi-document -c pygeoapi-config.yml > pygeoapi-openapi.yml + - psql -U postgres -c 'create database test' + - psql -U postgres -d test -c 'create extension postgis' + - gunzip < tests/data/hotosm_bdi_waterways.sql.gz | psql -U postgres test script: - pytest --cov=pygeoapi diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..a53c46eb9 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,26 @@ +FROM frolvlad/alpine-python3 + +RUN apk update && apk add --no-cache \ + git \ + python3-dev \ + libffi \ + libffi-dev \ + musl-dev \ + gcc \ + openssl-dev \ + g++ + + +RUN git clone https://github.com/geopython/pygeoapi.git + +ENV PYGEOAPI_CONFIG=/pygeoapi/local.yml + +WORKDIR /pygeoapi +RUN pip3 install -r requirements.txt +RUN pip3 install -r requirements-dev.txt +RUN pip3 install -e . +RUN cp pygeoapi-config.yml local.yml +#export PYGEOAPI_CONFIG=`pwd`/local.yml + +ENTRYPOINT ["/usr/bin/python3", "/pygeoapi/pygeoapi/app.py"] + diff --git a/pygeoapi-config.yml b/pygeoapi-config.yml index 884e62d90..3fad52ac1 100644 --- a/pygeoapi-config.yml +++ b/pygeoapi-config.yml @@ -190,4 +190,39 @@ datasets: data: tests/data/poi_portugal.gpkg id_field: osm_id table: poi_portugal + + hotosm_bdi_waterways: + title: Waterways of Burundi + description: Waterways of Burundi, Africa. Dataset timestamp 1st Sep 2018 - Humanitarian OpenStreetMap Team (HOT) + keywords: + - Burundi + - Waterways + - Africa + - OSM + - HOT + crs: + - CRS84 + links: + - type: text/html + rel: canonical + title: information + href: https://data.humdata.org/dataset/hotosm_bdi_waterways + hreflang: en-US + extents: + spatial: + bbox: [28.9845376683957 -4.48174334765485,30.866396969019 -2.3096796] + temporal: + begin: None + end: now # or empty + provider: + name: PostgreSQL + data: + host: 127.0.0.1 + dbname: test + user: postgres + password: postgres + port: 5432 + schema: public + id_field: osm_id + table: hotosm_bdi_waterways \ No newline at end of file diff --git a/pygeoapi/provider/__init__.py b/pygeoapi/provider/__init__.py index fdd56863e..a487a21e3 100644 --- a/pygeoapi/provider/__init__.py +++ b/pygeoapi/provider/__init__.py @@ -37,6 +37,7 @@ 'Elasticsearch': 'pygeoapi.provider.elasticsearch_.ElasticsearchProvider', 'GeoJSON': 'pygeoapi.provider.geojson.GeoJSONProvider', 'GeoPackage': 'pygeoapi.provider.geopackage.GeoPackageProvider', + 'PostgreSQL': 'pygeoapi.provider.postgresql.PostgreSQLProvider', 'SQLite': 'pygeoapi.provider.sqlite.SQLiteProvider' } diff --git a/pygeoapi/provider/geopackage.py b/pygeoapi/provider/geopackage.py index 41368e651..9705f5158 100644 --- a/pygeoapi/provider/geopackage.py +++ b/pygeoapi/provider/geopackage.py @@ -97,8 +97,7 @@ def __response_feature_collection(self): def __response_feature_hits(self, hits): """Assembles GeoJSON/Feature number - e,g: http://localhost:5000/poi/items? - limit=1&resulttype=hits + e,g: http://localhost:5000/collections/poi/items?resulttype=hits :returns: GeoJSON FeaturesCollection """ diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py new file mode 100644 index 000000000..94af57d61 --- /dev/null +++ b/pygeoapi/provider/postgresql.py @@ -0,0 +1,295 @@ +# ================================================================= +# +# Authors: Jorge Samuel Mendes de Jesus +# +# Copyright (c) 2018 Jorge Samuel Mendes de Jesus +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +# Testing local docker: +# docker run --name "postgis" \ +# -v postgres_data:/var/lib/postgresql -p 5432:5432 \ +# -e ALLOW_IP_RANGE=0.0.0.0/0 \ +# -e POSTGRES_USER=postgres \ +# -e POSTGRES_PASS=postgres \ +# -e POSTGRES_DBNAME=test \ +# -d -t kartoza/postgis + +# Import dump: +# gunzip < tests/data/hotosm_bdi_waterways.sql.gz | +# psql -U postgres -h 127.0.0.1 -p 5432 test + +import logging +import json +import psycopg2 +from psycopg2.sql import SQL, Identifier +from pygeoapi.provider.base import BaseProvider, \ + ProviderConnectionError, ProviderQueryError + +from psycopg2.extras import RealDictCursor + +LOGGER = logging.getLogger(__name__) + + +class DatabaseConnection(object): + """Database connection class to be used as 'with' statement. + The class returns a connection object. + """ + + def __init__(self, conn_dic, table, context="query"): + """ + PostgreSQLProvider Class constructor returning + + :param conn: dictionary with connection parameters + to be used by psycopg2 + dbname – the database name (database is a deprecated alias) + user – user name used to authenticate + password – password used to authenticate + host – database host address + (defaults to UNIX socket if not provided) + port – connection port number + (defaults to 5432 if not provided) + schema – schema to use as search path, normally + data is in the public schema + + :param table: table name containing the data. This variable is used to + assemble column information + :param context: query or hits, if query then it will determine + table column otherwise will not do it + :returns: psycopg2.extensions.connection + """ + + self.conn_dic = conn_dic + self.table = table + self.context = context + self.columns = None + self.conn = None + self.schema = None + + def __enter__(self): + try: + self.schema = self.conn_dic.pop('schema', None) + if self.schema == 'public' or self.schema is None: + pass + else: + self.conn_dic["options"] = '-c search_path={}'.format( + self.schema) + LOGGER.debug('Using schema {} as search path'.format( + self.schema)) + self.conn = psycopg2.connect(**self.conn_dic) + + except psycopg2.OperationalError: + LOGGER.error('Couldnt connect to Postgis using:{}'.format( + str(self.conn_dic))) + raise ProviderConnectionError() + + self.cur = self.conn.cursor() + if self.context == 'query': + # Getting columns + query_cols = "SELECT column_name FROM information_schema.columns \ + WHERE table_name = '{}' and udt_name != 'geometry';".format( + self.table) + + self.cur.execute(query_cols) + result = self.cur.fetchall() + self.columns = SQL(', ').join( + [Identifier(item[0]) for item in result] + ) + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # some logic to commit/rollback + self.conn.close() + + +class PostgreSQLProvider(BaseProvider): + """Generic provider for Postgresql based on psycopg2 + using sync approach and server side + cursor (using support class DatabaseCursor) + """ + + def __init__(self, provider_def): + """ + PostgreSQLProvider Class constructor + + :param provider_def: provider definitions from yml pygeoapi-config. + data,id_field, name set in parent class + data contains the connection information + for class DatabaseCursor + + :returns: pygeoapi.providers.base.PostgreSQLProvider + """ + + BaseProvider.__init__(self, provider_def) + + self.table = provider_def['table'] + self.id_field = provider_def['id_field'] + self.conn_dic = provider_def['data'] + + LOGGER.debug('Setting Postgresql properties:') + LOGGER.debug('Connection String:{}'.format( + ",".join(("{}={}".format(*i) for i in self.conn_dic.items())))) + LOGGER.debug('Name:{}'.format(self.name)) + LOGGER.debug('ID_field:{}'.format(self.id_field)) + LOGGER.debug('Table:{}'.format(self.table)) + + def query(self, startindex=0, limit=10, resulttype='results', + bbox=[], time=None, properties=[]): + """ + Query Postgis for all the content. + e,g: http://localhost:5000/collections/hotosm_bdi_waterways/items? + limit=1&resulttype=results + + :param startindex: starting record to return (default 0) + :param limit: number of records to return (default 10) + :param resulttype: return results or hit limit (default results) + :param bbox: bounding box [minx,miny,maxx,maxy] + :param time: temporal (datestamp or extent) + :param properties: list of tuples (name, value) + + :returns: GeoJSON FeaturesCollection + """ + LOGGER.debug('Querying PostGIS') + + if resulttype == 'hits': + + with DatabaseConnection(self.conn_dic, + self.table, context="hits") as db: + cursor = db.conn.cursor(cursor_factory=RealDictCursor) + sql_query = SQL("select count(*) as hits from {}").\ + format(Identifier(self.table)) + try: + cursor.execute(sql_query) + except Exception as err: + LOGGER.error('Error executing sql_query: {}'.format( + sql_query.as_string(cursor))) + LOGGER.error('Using public schema: {}'.format(db.schema)) + raise ProviderQueryError() + + hits = cursor.fetchone()["hits"] + + return self.__response_feature_hits(hits) + + end_index = startindex + limit + + with DatabaseConnection(self.conn_dic, self.table) as db: + cursor = db.conn.cursor(cursor_factory=RealDictCursor) + sql_query = SQL("DECLARE \"geo_cursor\" CURSOR FOR \ + SELECT {0},ST_AsGeoJSON({1}) FROM {2}").\ + format(db.columns, + Identifier('geom'), + Identifier(self.table)) + + LOGGER.debug('SQL Query:{}'.format(sql_query)) + LOGGER.debug('Start Index:{}'.format(startindex)) + LOGGER.debug('End Index'.format(end_index)) + try: + cursor.execute(sql_query) + for index in [startindex, limit]: + cursor.execute("fetch forward {} from geo_cursor" + .format(index)) + except Exception as err: + LOGGER.error('Error executing sql_query: {}'.format( + sql_query.as_string(cursor))) + LOGGER.error('Using public schema: {}'.format(db.schema)) + raise ProviderQueryError() + + self.dataDB = cursor.fetchall() + feature_collection = self.__response_feature_collection() + return feature_collection + + def get(self, identifier): + """ + Query the provider for a specific + feature id e.g: /collections/hotosm_bdi_waterways/items/13990765 + + :param identifier: feature id + + :returns: GeoJSON FeaturesCollection + """ + + LOGGER.debug('Get item from Postgis') + with DatabaseConnection(self.conn_dic, self.table) as db: + cursor = db.conn.cursor(cursor_factory=RealDictCursor) + + sql_query = SQL("select {0},ST_AsGeoJSON({1}) \ + from {2} WHERE {3}=%s").format(db.columns, + Identifier('geom'), + Identifier(self.table), + Identifier(self.id_field)) + + LOGGER.debug('SQL Query:{}'.format(sql_query.as_string(db.conn))) + LOGGER.debug('Identifier:{}'.format(identifier)) + try: + cursor.execute(sql_query, (identifier, )) + except Exception as err: + LOGGER.error('Error executing sql_query: {}'.format( + sql_query.as_string(cursor))) + LOGGER.error('Using public schema: {}'.format(db.schema)) + raise ProviderQueryError() + + self.dataDB = cursor.fetchall() + feature_collection = self.__response_feature_collection() + return feature_collection + + def __response_feature_collection(self): + """Assembles GeoJSON output from DB query + + :returns: GeoJSON FeaturesCollection + """ + + feature_list = list() + for row_data in self.dataDB: + row_data = dict(row_data) + feature = { + 'type': 'Feature' + } + feature["geometry"] = json.loads( + row_data.pop('st_asgeojson') + ) + feature['properties'] = row_data + feature['id'] = feature['properties'].pop(self.id_field) + feature_list.append(feature) + + feature_collection = { + 'type': 'FeatureCollection', + 'features': feature_list + } + + return feature_collection + + def __response_feature_hits(self, hits): + """Assembles GeoJSON/Feature number + e.g: http://localhost:5000/collections/ + hotosm_bdi_waterways/items?resulttype=hits + + :returns: GeoJSON FeaturesCollection + """ + + feature_collection = {"features": [], + "type": "FeatureCollection"} + feature_collection['numberMatched'] = hits + + return feature_collection diff --git a/requirements-dev.txt b/requirements-dev.txt index f32a1a6a4..8f7bd9841 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,6 +5,7 @@ docutils==0.14 flake8==3.5.0 twine==1.11.0 wheel==0.31.0 +psycopg2==2.7.6 pypandoc==1.4 pytest==3.5.0 pytest-cov diff --git a/tests/data/README.md b/tests/data/README.md index fead5c88d..76879f161 100644 --- a/tests/data/README.md +++ b/tests/data/README.md @@ -36,4 +36,11 @@ This directory provides test data to demonstrate functionality. - source: Open Street Map - Natural GIS - URL: [http://www.naturalgis.pt/cgi-bin/opendata/mapserv?service=WFS&request=GetCapabilities](http://www.naturalgis.pt/cgi-bin/opendata/mapserv?service=WFS&request=GetCapabilities) - Data obtained from WFS instance of NaturalGIS company (http://www.naturalgis.pt/en/) and converted to geopackage -- Upstream data from Open Street Map extract for Portugal \ No newline at end of file +- Upstream data from Open Street Map extract for Portugal + +### `hotosm_bdi_waterways.sql.gz` +- source: Open Street Map - Humanitarian OpenStreetMap Team (HOT) +- URL: [hotosm_bdi_waterways](https://data.humdata.org/dataset/hotosm_bdi_waterways) +- Waterways of Burundi +- Date of dataset: Sep 01, 2018 +- Location: Burundi, Africa \ No newline at end of file diff --git a/tests/data/hotosm_bdi_waterways.sql.gz b/tests/data/hotosm_bdi_waterways.sql.gz new file mode 100644 index 000000000..fbfe579ad Binary files /dev/null and b/tests/data/hotosm_bdi_waterways.sql.gz differ diff --git a/tests/data/poi_portugal.gpkg b/tests/data/poi_portugal.gpkg index 737349a03..011af54a4 100644 Binary files a/tests/data/poi_portugal.gpkg and b/tests/data/poi_portugal.gpkg differ diff --git a/tests/test_postgresql_provider.py b/tests/test_postgresql_provider.py new file mode 100644 index 000000000..7dc4e4702 --- /dev/null +++ b/tests/test_postgresql_provider.py @@ -0,0 +1,42 @@ +# Needs to be run like: python3 -m pytest + +import pytest +from pygeoapi.provider.postgresql import PostgreSQLProvider + + +@pytest.fixture() +def config(): + return { + 'name': 'PostgreSQL', + 'data': {'host': '127.0.0.1', + 'dbname': 'test', + 'user': 'postgres', + 'password': 'postgres' + }, + 'id_field': "osm_id", + 'table': 'hotosm_bdi_waterways' + } + + +def test_query(config): + """Testing query for a valid JSON object with geometry""" + + p = PostgreSQLProvider(config) + feature_collection = p.query() + assert feature_collection.get('type', None) == "FeatureCollection" + features = feature_collection.get('features', None) + assert features is not None + feature = features[0] + properties = feature.get("properties", None) + assert properties is not None + geometry = feature.get("geometry", None) + assert geometry is not None + + +def test_get(config): + """Testing query for a specific object""" + p = PostgreSQLProvider(config) + results = p.get(29701937) + print(results) + assert len(results['features']) == 1 + assert "Kanyosha" in results['features'][0]['properties']['name']