diff --git a/ansible/roles/dataqs/tasks/main.yml b/ansible/roles/dataqs/tasks/main.yml index f8828ad..fb72e24 100644 --- a/ansible/roles/dataqs/tasks/main.yml +++ b/ansible/roles/dataqs/tasks/main.yml @@ -7,6 +7,7 @@ - libgdal-dev - cdo - netcdf-bin + - libffi-dev sudo: yes - name: install python dependencies @@ -15,6 +16,11 @@ - redis - flower +- name: upgrade urllib3 + pip: virtualenv={{virtualenv_dir}}/{{app_name}} name={{item}} state=forcereinstall + with_items: + - urllib3[secure] + - name: install dataqs python package pip: virtualenv={{virtualenv_dir}}/{{app_name}} name=git+https://github.com/OpenGeoscience/dataqs.git#egg=dataqs environment: @@ -67,30 +73,35 @@ - name: Create the usgs_quakes store command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/usgs_quakes/usgs_quakes.py" ignore_errors: yes + sudo: yes - include: geoserver_permissions.yml - name: Create the wqp store command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/wqp/wqp.py" ignore_errors: yes + sudo: yes - include: geoserver_permissions.yml - name: Create the gdacs store command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/gdacs/gdacs.py" ignore_errors: yes + sudo: yes - include: geoserver_permissions.yml - name: Create the gfms coverage store command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/gfms/gfms.py" ignore_errors: yes + sudo: yes - include: geoserver_permissions.yml - name: Create the gistemp coverage store command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/gistemp/gistemp.py" ignore_errors: yes + sudo: yes - name: create geoserver data directory file: path=/var/lib/tomcat7/webapps/geoserver/data/data/geonode/forecast_io_airtemp recurse=yes owner=tomcat7 group=tomcat7 state=directory mode=774 @@ -101,24 +112,28 @@ - name: Create the forecastio coverage store command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/forecastio/forecastio_air.py" ignore_errors: yes + sudo: yes - include: geoserver_permissions.yml - name: Create the hifld layers command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/hifld/hifld.py" ignore_errors: yes + sudo: yes - include: geoserver_permissions.yml - name: Create the cmap layer command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/cmap/cmap.py" ignore_errors: yes + sudo: yes - include: geoserver_permissions.yml - name: Create the landscan layer command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/landscan/landscan.py" ignore_errors: yes + sudo: yes - name: Django updatelayers django_manage: command=updatelayers diff --git a/dataqs/airnow/airnow.py b/dataqs/airnow/airnow.py old mode 100644 new mode 100755 index 82dbba5..8c1c26f --- a/dataqs/airnow/airnow.py +++ b/dataqs/airnow/airnow.py @@ -161,8 +161,10 @@ def run(self, days=1): with open(os.path.join( script_dir, 'resources/airnow.sld')) as sld: self.set_default_style(layer_name, layer_name, sld.read()) - self.update_geonode(layer_name, title=layer_title, - description=self.description, store=layer_name) + self.update_geonode( + layer_name, title=layer_title, + description=self.description, store=layer_name, + extra_keywords=['category:Climatology Meteorology Atmosphere']) self.truncate_gs_cache(layer_name) self.cleanup() diff --git a/dataqs/aqicn/aqicn.py b/dataqs/aqicn/aqicn.py old mode 100644 new mode 100755 index 93690dc..1e69e23 --- a/dataqs/aqicn/aqicn.py +++ b/dataqs/aqicn/aqicn.py @@ -326,10 +326,12 @@ def run(self): if not style_exists(layer_name): with open(os.path.join(script_dir, 'resources/aqicn.sld')) as sld: self.set_default_style(layer_name, layer_name, sld.read()) - self.update_geonode(layer_name, - title='Air Quality Index', - description=self.description, - store=datastore) + self.update_geonode( + layer_name, + title='Air Quality Index', + description=self.description, + store=datastore, + extra_keywords=['category:Climatology Meteorology Atmosphere']) self.truncate_gs_cache(layer_name) self.cleanup() diff --git a/dataqs/cmap/cmap.py b/dataqs/cmap/cmap.py index 7bcc2cd..121491e 100644 --- a/dataqs/cmap/cmap.py +++ b/dataqs/cmap/cmap.py @@ -143,11 +143,13 @@ def run(self): 'resources/cmap.sld')) as sld: self.set_default_style(self.layer_name, self.layer_name, sld.read().format(latest_band=bands)) - self.update_geonode(self.layer_name, title=self.get_title(bands), - description=self.abstract, - store=self.layer_name, - bounds=('-178.75', '178.75', '-88.75', '88.75', - 'EPSG:4326')) + self.update_geonode( + self.layer_name, title=self.get_title(bands), + description=self.abstract, + store=self.layer_name, + bounds=('-178.75', '178.75', '-88.75', '88.75', + 'EPSG:4326'), + extra_keywords=['category:Climatology Meteorology Atmosphere']) self.truncate_gs_cache(self.layer_name) self.cleanup() diff --git a/dataqs/forecastio/forecastio_air.py b/dataqs/forecastio/forecastio_air.py old mode 100644 new mode 100755 index 17c18b0..b775960 --- a/dataqs/forecastio/forecastio_air.py +++ b/dataqs/forecastio/forecastio_air.py @@ -119,11 +119,13 @@ def run(self, now=None): self.drop_old_hourly_images(now, self.layer_name) self.drop_old_daily_images(now, self.layer_name) - self.update_geonode(self.layer_name, title=self.parse_name(now), - description=self.description, - store=self.layer_name, - bounds=('-180.0', '180.0', - '-90.0', '90.0', 'EPSG:4326')) + self.update_geonode( + self.layer_name, title=self.parse_name(now), + description=self.description, + store=self.layer_name, + bounds=('-180.0', '180.0', + '-90.0', '90.0', 'EPSG:4326'), + extra_keywords=['category:Climatology Meteorology Atmosphere']) self.truncate_gs_cache(self.layer_name) self.cleanup() diff --git a/dataqs/gdacs/gdacs.py b/dataqs/gdacs/gdacs.py old mode 100644 new mode 100755 index c65c2a3..c302c86 --- a/dataqs/gdacs/gdacs.py +++ b/dataqs/gdacs/gdacs.py @@ -98,7 +98,8 @@ def run(self): script_dir, 'resources/gdacs.sld')) as sld: self.set_default_style(self.prefix, self.prefix, sld.read()) self.update_geonode(self.prefix, title=self.layer_title, - description=self.description, store=datastore) + description=self.description, store=datastore, + extra_keywords=['category:Disaster Alerts']) self.truncate_gs_cache(self.prefix) self.cleanup() diff --git a/dataqs/gfms/gfms.py b/dataqs/gfms/gfms.py old mode 100644 new mode 100755 index 1e72f81..23532e2 --- a/dataqs/gfms/gfms.py +++ b/dataqs/gfms/gfms.py @@ -166,7 +166,9 @@ def import_future(self): self.set_default_style(self.layer_future, self.layer_future, sld.read()) self.update_geonode(self.layer_future, title=new_title, - store=self.layer_future) + store=self.layer_future, + description=self.description, + extra_keywords=['category:Disaster Alerts']) self.truncate_gs_cache(self.layer_future) def import_current(self): @@ -185,7 +187,8 @@ def import_current(self): self.layer_current, sld.read()) self.update_geonode(self.layer_current, title=new_title, store=self.layer_current, - description=self.description) + description=self.description, + extra_keywords=['category:Disaster Alerts']) self.truncate_gs_cache(self.layer_current) def run(self): diff --git a/dataqs/gistemp/gistemp.py b/dataqs/gistemp/gistemp.py old mode 100644 new mode 100755 index 90fa2ef..2a29e1a --- a/dataqs/gistemp/gistemp.py +++ b/dataqs/gistemp/gistemp.py @@ -112,11 +112,13 @@ def run(self): 'resources/gistemp.sld')) as sld: self.set_default_style(self.layer_name, self.layer_name, sld.read().format(latest_band=bands)) - self.update_geonode(self.layer_name, title=self.get_title(bands), - description=self.abstract, - store=self.layer_name, - bounds=('-180.0', '180.0', '-90.0', '90.0', - 'EPSG:4326')) + self.update_geonode( + self.layer_name, title=self.get_title(bands), + description=self.abstract, + store=self.layer_name, + bounds=('-180.0', '180.0', '-90.0', '90.0', + 'EPSG:4326'), + extra_keywords=['category:Climatology Meteorology Atmosphere']) self.truncate_gs_cache(self.layer_name) self.cleanup() diff --git a/dataqs/hifld/hifld.py b/dataqs/hifld/hifld.py old mode 100644 new mode 100755 index c82a0ae..ef20600 --- a/dataqs/hifld/hifld.py +++ b/dataqs/hifld/hifld.py @@ -39,6 +39,19 @@ class HIFLDProcessor(GeoDataProcessor): prefix = 'hifld_' layers = [] base_url = "https://hifld-dhs-gii.opendata.arcgis.com/datasets/" + layer_category_mapping = { + 'us_state_boundaries': 'category:Boundaries', + 'us_county_boundaries': 'category:Boundaries', + 'us_urban_areas': 'category:Boundaries', + 'poultry_facilities': 'category:Agriculture', + 'state_fairgrounds': 'category:Agriculture', + 'epa_tsca_facilities': 'category:Chemicals', + 'epa_er_rmp_facilities': 'category:Chemicals', + 'epa_er_tri_facilities': 'category:Chemicals', + 'hospitals': 'category:Public Health', + 'pharmacies': 'category:Public Health', + 'hazmat_routes': 'category:Chemicals:' + } def __init__(self, layers=None): super(HIFLDProcessor, self).__init__() @@ -87,10 +100,13 @@ def run(self): sld_text = sld.read().format(table=layer['table'], title=layer['name']) self.set_default_style(table, table, sld_text) - self.update_geonode(table, - title=layer['name'], - description=layer['description'], - store=datastore) + keywords = self.layer_category_mapping[layer['table']] + self.update_geonode( + table, + title=layer['name'], + description=layer['description'], + store=datastore, + extra_keywords=[keywords]) self.truncate_gs_cache(table) except Exception: logger.error('Error with layer {}'.format(layer['name'])) diff --git a/dataqs/landscan/landscan.py b/dataqs/landscan/landscan.py old mode 100644 new mode 100755 index d3a0f47..8a58670 --- a/dataqs/landscan/landscan.py +++ b/dataqs/landscan/landscan.py @@ -94,7 +94,8 @@ def import_landscan(self, landscan_tiff): self.layer, sld.read()) self.update_geonode(self.layer, title=self.layer, store=self.layer, - description=self.description) + description=self.description, + extra_keywords=['category:Population']) self.truncate_gs_cache(self.layer) def run(self): diff --git a/dataqs/mmwr/mmwr.py b/dataqs/mmwr/mmwr.py old mode 100644 new mode 100755 index 5486e91..9f5202c --- a/dataqs/mmwr/mmwr.py +++ b/dataqs/mmwr/mmwr.py @@ -196,7 +196,8 @@ def update_layer(self, layer): self.update_geonode( table, title='{} {}'.format(self.base_title, layer), - description=self.description) + description=self.description, + extra_keywords=['category:Population']) self.truncate_gs_cache(table) def run(self): diff --git a/dataqs/nasa_gpm/nasa_gpm.py b/dataqs/nasa_gpm/nasa_gpm.py old mode 100644 new mode 100755 index b44b4a5..abfa241 --- a/dataqs/nasa_gpm/nasa_gpm.py +++ b/dataqs/nasa_gpm/nasa_gpm.py @@ -124,12 +124,14 @@ def run(self, days=1): with open(os.path.join(script_dir, 'resources/gpm.sld')) as sld: self.set_default_style(self.layer_name, self.layer_name, sld.read()) - self.update_geonode(self.layer_name, - title=layer_title, - description=self.description, - store=self.layer_name, - bounds=('-180.0', '180.0', '-90.0', '90.0', - 'EPSG:4326')) + self.update_geonode( + self.layer_name, + title=layer_title, + description=self.description, + store=self.layer_name, + bounds=('-180.0', '180.0', '-90.0', '90.0', + 'EPSG:4326'), + extra_keywords=['category:Climatology Meteorology Atmosphere']) self.truncate_gs_cache(self.layer_name) self.cleanup() diff --git a/dataqs/processor_base.py b/dataqs/processor_base.py old mode 100644 new mode 100755 index 1644366..115a4de --- a/dataqs/processor_base.py +++ b/dataqs/processor_base.py @@ -235,7 +235,8 @@ def update_gs_metadata(self, layer_name, json_data, vector=False, return res.content def update_geonode(self, layer_name, title="", description="", - category=None, bounds=None, store=None): + category=None, bounds=None, store=None, + extra_keywords=None): """ Update a layer and it's title in GeoNode :param layer_name: Name of the layer @@ -265,8 +266,19 @@ def update_geonode(self, layer_name, title="", description="", url = ogc_server_settings.rest gs_catalog = Catalog(url, _user, _password) gs_catalog.save(res) + if extra_keywords: + assert isinstance(extra_keywords, list) + # Append extra keywords to the default ones + res = lyr.gs_resource + keywords = res.keywords + extra_keywords + res.keywords = keywords + _user, _password = ogc_server_settings.credentials + url = ogc_server_settings.rest + gs_catalog = Catalog(url, _user, _password) + gs_catalog.save(res) - def set_default_style(self, layer_name, sld_name, sld_content, create=True): + def set_default_style(self, layer_name, sld_name, sld_content, + create=True): """ Create a style and assign it as default to a layer :param layer_name: the layer to assign the style to diff --git a/dataqs/spei/spei.py b/dataqs/spei/spei.py old mode 100644 new mode 100755 index f9392ca..e78a1f0 --- a/dataqs/spei/spei.py +++ b/dataqs/spei/spei.py @@ -85,7 +85,8 @@ def run(self): self.update_geonode(layer_name, title=self.spei_files[layer_name], description=self.description, - store=layer_name) + store=layer_name, + extra_keywords=['category:Agriculture']) self.truncate_gs_cache(layer_name) self.cleanup() diff --git a/dataqs/usgs_quakes/usgs_quakes.py b/dataqs/usgs_quakes/usgs_quakes.py old mode 100644 new mode 100755 index f60a750..47e10a5 --- a/dataqs/usgs_quakes/usgs_quakes.py +++ b/dataqs/usgs_quakes/usgs_quakes.py @@ -127,10 +127,12 @@ def run(self, rss_file=None): with open(os.path.join( script_dir, 'resources/usgs.sld')) as sld: self.set_default_style(table, table, sld.read()) - self.update_geonode(table, - title="Earthquakes - {}".format(title), - description=self.description, - store=datastore) + self.update_geonode( + table, + title="Earthquakes - {}".format(title), + description=self.description, + store=datastore, + extra_keywords=['category:Geoscientific Information']) self.truncate_gs_cache(table) self.purge_old_data() self.cleanup() diff --git a/dataqs/whisp/whisp.py b/dataqs/whisp/whisp.py old mode 100644 new mode 100755 index b4d8da2..3178678 --- a/dataqs/whisp/whisp.py +++ b/dataqs/whisp/whisp.py @@ -172,7 +172,8 @@ def run(self): self.set_default_style(self.prefix, self.prefix, sld.read()) self.update_geonode(self.prefix, title=self.title, - description=self.description) + description=self.description, + extra_keywords=['category:Wildlife']) self.truncate_gs_cache(self.prefix) self.cleanup() diff --git a/dataqs/worldclim/worldclim.py b/dataqs/worldclim/worldclim.py old mode 100644 new mode 100755 index c472772..1999a38 --- a/dataqs/worldclim/worldclim.py +++ b/dataqs/worldclim/worldclim.py @@ -144,8 +144,10 @@ def publish(self, tif, name, title, desc): self.set_default_style(name, style, sld.read(), create=not style_exists(style)) self.truncate_gs_cache(name) - self.update_geonode(name, title, - description=desc, category=category, store=name) + self.update_geonode( + name, title, + description=desc, category=category, store=name, + extra_keywords=['category:Climatology Meteorology Atmosphere']) def cleanup(self, outdir): """ diff --git a/dataqs/wqp/wqp.py b/dataqs/wqp/wqp.py old mode 100644 new mode 100755 index fc747f3..d2f2994 --- a/dataqs/wqp/wqp.py +++ b/dataqs/wqp/wqp.py @@ -263,7 +263,8 @@ def run(self): self.update_geonode(layer_name, title=layer_title, description=self.description, - store=datastore) + store=datastore, + extra_keywords=['category:Water Quality']) self.truncate_gs_cache(layer_name) self.cleanup()