Skip to content

Commit

Permalink
implement CQL to OGC filter transforms
Browse files Browse the repository at this point in the history
  • Loading branch information
@tomkralidis
tomkralidis committed Oct 18, 2016
1 parent c27c247 commit 522873e
Show file tree
Hide file tree
Showing 7 changed files with 213 additions and 11 deletions.
114 changes: 114 additions & 0 deletions pycsw/cql.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
# -*- coding: utf-8 -*-
# =================================================================
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2016 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================

import logging

from lxml import etree
from pycsw import util
from pycsw.fes import MODEL as fes1_model

LOGGER = logging.getLogger(__name__)


def cql2fes1(cql, namespaces):
"""transforms Common Query Language (CQL) query into OGC fes1 syntax"""

filters = []
tmp_list = []
logical_op = None

LOGGER.debug('CQL: %s', cql)

if ' or ' in cql:
logical_op = etree.Element(util.nspath_eval('ogc:Or', namespaces))
tmp_list = cql.split(' or ')
elif ' OR ' in cql:
logical_op = etree.Element(util.nspath_eval('ogc:Or', namespaces))
tmp_list = cql.split(' OR ')
elif ' and ' in cql:
logical_op = etree.Element(util.nspath_eval('ogc:And', namespaces))
tmp_list = cql.split(' and ')
elif ' AND ' in cql:
logical_op = etree.Element(util.nspath_eval('ogc:And', namespaces))
tmp_list = cql.split(' AND ')

if tmp_list:
LOGGER.debug('Logical operator found (AND/OR)')
else:
tmp_list.append(cql)

for t in tmp_list:
filters.append(_parse_condition(t))

root = etree.Element(util.nspath_eval('ogc:Filter', namespaces))

if logical_op is not None:
root.append(logical_op)

for flt in filters:
condition = etree.Element(util.nspath_eval(flt[0], namespaces))

etree.SubElement(
condition,
util.nspath_eval('ogc:PropertyName', namespaces)).text = flt[1]

etree.SubElement(
condition,
util.nspath_eval('ogc:Literal', namespaces)).text = flt[2]

if logical_op is not None:
logical_op.append(condition)
else:
root.append(condition)

LOGGER.debug('Resulting OGC Filter: %s',
etree.tostring(root, pretty_print=1))

return root


def _parse_condition(condition):
"""parses a single condition"""

LOGGER.debug('condition: %s', condition)

property_name, operator, literal = condition.split()

literal = literal.replace('"', '').replace('\'', '')

for k, v in fes1_model['ComparisonOperators'].items():
if v['opvalue'] == operator:
fes1_predicate = k

LOGGER.debug('parsed condition: %s %s %s', property_name, fes1_predicate,
literal)

return (fes1_predicate, property_name, literal)

41 changes: 30 additions & 11 deletions pycsw/server.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
from pycsw.plugins.profiles import profile as pprofile
import pycsw.plugins.outputschemas
from pycsw import config, fes, log, metadata, util, sru, oaipmh, opensearch
from pycsw.cql import cql2fes1
import logging

LOGGER = logging.getLogger(__name__)
Expand Down Expand Up @@ -1281,12 +1282,20 @@ def getrecords(self):
% self.kvp['constraintlanguage'])
if self.kvp['constraintlanguage'] == 'CQL_TEXT':
tmp = self.kvp['constraint']
self.kvp['constraint'] = {}
self.kvp['constraint']['type'] = 'cql'
self.kvp['constraint']['where'] = \
self._cql_update_queryables_mappings(tmp,
self.repository.queryables['_all'])
self.kvp['constraint']['values'] = {}
try:
LOGGER.debug('Transforming CQL into fes1')
LOGGER.debug('CQL: %s', tmp)
self.kvp['constraint'] = {}
self.kvp['constraint']['type'] = 'filter'
cql = cql2fes1(tmp, self.context.namespaces)
self.kvp['constraint']['where'], self.kvp['constraint']['values'] = fes.parse(cql,
self.repository.queryables['_all'], self.repository.dbtype,
self.context.namespaces, self.orm, self.language['text'], self.repository.fts)
except Exception as err:
LOGGER.error('Invalid CQL query %s', tmp)
LOGGER.error('Error message: %s', err, exc_info=True)
return self.exceptionreport('InvalidParameterValue',
'constraint', 'Invalid Filter syntax')
elif self.kvp['constraintlanguage'] == 'FILTER':
# validate filter XML
try:
Expand Down Expand Up @@ -1364,8 +1373,10 @@ def getrecords(self):
maxrecords=self.kvp['maxrecords'],
startposition=int(self.kvp['startposition'])-1)
except Exception as err:
LOGGER.debug('Invalid query syntax. Query: %s', self.kvp['constraint'])
LOGGER.debug('Invalid query syntax. Result: %s', err)
return self.exceptionreport('InvalidParameterValue', 'constraint',
'Invalid query: %s' % err)
'Invalid query syntax')

dsresults = []

Expand Down Expand Up @@ -2479,13 +2490,21 @@ def _parse_constraint(self, element):
self.context.namespaces, self.orm, self.language['text'], self.repository.fts)
except Exception as err:
return 'Invalid Filter request: %s' % err

tmp = element.find(util.nspath_eval('csw:CqlText', self.context.namespaces))
if tmp is not None:
LOGGER.debug('CQL specified: %s.' % tmp.text)
query['type'] = 'cql'
query['where'] = self._cql_update_queryables_mappings(tmp.text,
self.repository.queryables['_all'])
query['values'] = {}
try:
LOGGER.debug('Transforming CQL into OGC Filter')
query['type'] = 'filter'
cql = cql2fes1(tmp.text, self.context.namespaces)
query['where'], query['values'] = fes.parse(cql,
self.repository.queryables['_all'], self.repository.dbtype,
self.context.namespaces, self.orm, self.language['text'], self.repository.fts)
except Exception as err:
LOGGER.error('Invalid CQL request: %s', tmp.text)
LOGGER.error('Error message: %s', err, exc_info=True)
return 'Invalid CQL request'
return query

def _test_manager(self):
Expand Down
23 changes: 23 additions & 0 deletions tests/expected/suites_default_get_GetRecords-filter-cql-title-or-abstract.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- PYCSW_VERSION -->
<csw:GetRecordsResponse xmlns:soapenv="http://www.w3.org/2003/05/soap-envelope" xmlns:gml="http://www.opengis.net/gml" xmlns:dif="http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ows="http://www.opengis.net/ows" xmlns:fgdc="http://www.opengis.net/cat/csw/csdgm" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:sitemap="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:os="http://a9.com/-/spec/opensearch/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:ogc="http://www.opengis.net/ogc" version="2.0.2" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
<csw:SearchStatus timestamp="PYCSW_TIMESTAMP"/>
<csw:SearchResults nextRecord="0" numberOfRecordsMatched="2" numberOfRecordsReturned="2" recordSchema="http://www.opengis.net/cat/csw/2.0.2" elementSet="full">
<csw:Record>
<dc:identifier>urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f</dc:identifier>
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
<dc:format>image/svg+xml</dc:format>
<dc:title>Lorem ipsum</dc:title>
<dct:spatial>GR-22</dct:spatial>
<dc:subject>Tourism--Greece</dc:subject>
<dct:abstract>Quisque lacus diam, placerat mollis, pharetra in, commodo sed, augue. Duis iaculis arcu vel arcu.</dct:abstract>
</csw:Record>
<csw:Record>
<dc:identifier>urn:uuid:a06af396-3105-442d-8b40-22b57a90d2f2</dc:identifier>
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
<dc:title>Lorem ipsum dolor sit amet</dc:title>
<dc:format>image/jpeg</dc:format>
<dct:spatial>IT-FI</dct:spatial>
</csw:Record>
</csw:SearchResults>
</csw:GetRecordsResponse>
23 changes: 23 additions & 0 deletions tests/expected/suites_default_get_GetRecords-filter-cql-title.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- PYCSW_VERSION -->
<csw:GetRecordsResponse xmlns:soapenv="http://www.w3.org/2003/05/soap-envelope" xmlns:gml="http://www.opengis.net/gml" xmlns:dif="http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ows="http://www.opengis.net/ows" xmlns:fgdc="http://www.opengis.net/cat/csw/csdgm" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:sitemap="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:os="http://a9.com/-/spec/opensearch/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:ogc="http://www.opengis.net/ogc" version="2.0.2" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
<csw:SearchStatus timestamp="PYCSW_TIMESTAMP"/>
<csw:SearchResults nextRecord="0" numberOfRecordsMatched="2" numberOfRecordsReturned="2" recordSchema="http://www.opengis.net/cat/csw/2.0.2" elementSet="full">
<csw:Record>
<dc:identifier>urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f</dc:identifier>
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
<dc:format>image/svg+xml</dc:format>
<dc:title>Lorem ipsum</dc:title>
<dct:spatial>GR-22</dct:spatial>
<dc:subject>Tourism--Greece</dc:subject>
<dct:abstract>Quisque lacus diam, placerat mollis, pharetra in, commodo sed, augue. Duis iaculis arcu vel arcu.</dct:abstract>
</csw:Record>
<csw:Record>
<dc:identifier>urn:uuid:a06af396-3105-442d-8b40-22b57a90d2f2</dc:identifier>
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
<dc:title>Lorem ipsum dolor sit amet</dc:title>
<dc:format>image/jpeg</dc:format>
<dct:spatial>IT-FI</dct:spatial>
</csw:Record>
</csw:SearchResults>
</csw:GetRecordsResponse>
12 changes: 12 additions & 0 deletions tests/expected/suites_default_post_GetRecords-cql-title-and-abstract.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- PYCSW_VERSION -->
<csw:GetRecordsResponse xmlns:soapenv="http://www.w3.org/2003/05/soap-envelope" xmlns:gml="http://www.opengis.net/gml" xmlns:dif="http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ows="http://www.opengis.net/ows" xmlns:fgdc="http://www.opengis.net/cat/csw/csdgm" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:sitemap="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:os="http://a9.com/-/spec/opensearch/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:ogc="http://www.opengis.net/ogc" version="2.0.2" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
<csw:SearchStatus timestamp="PYCSW_TIMESTAMP"/>
<csw:SearchResults nextRecord="0" numberOfRecordsMatched="1" numberOfRecordsReturned="1" recordSchema="http://www.opengis.net/cat/csw/2.0.2" elementSet="brief">
<csw:BriefRecord>
<dc:identifier>urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f</dc:identifier>
<dc:title>Lorem ipsum</dc:title>
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
</csw:BriefRecord>
</csw:SearchResults>
</csw:GetRecordsResponse>
2 changes: 2 additions & 0 deletions tests/suites/default/get/requests.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,5 @@ GetRecords-filter,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=C
Exception-GetRepositoryItem-service-invalid1,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW%00&version=2.0.2&request=GetRepositoryItem&id=123
Exception-GetRepositoryItem-service-invalid2,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW%00'&version=2.0.2&request=GetRepositoryItem&id=123
Exception-GetRepositoryItem-version-invalid,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2'&request=GetRepositoryItem&id=123
GetRecords-filter-cql-title,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&constraintlanguage=CQL_TEXT&constraint=dc%3Atitle%20like%20%27%25lor%25%27
GetRecords-filter-cql-title-or-abstract,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&constraintlanguage=CQL_TEXT&constraint=dc%3Atitle%20like%20%27%25lor%25%27%20or%20dct%3Aabstract%20like%20%27%25pharetra%25%27
9 changes: 9 additions & 0 deletions tests/suites/default/post/GetRecords-cql-title-and-abstract.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="ISO-8859-1" standalone="no"?>
<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:ogc="http://www.opengis.net/ogc" service="CSW" version="2.0.2" resultType="results" startPosition="1" maxRecords="5" outputFormat="application/xml" outputSchema="http://www.opengis.net/cat/csw/2.0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
<csw:Query typeNames="csw:Record">
<csw:ElementSetName>brief</csw:ElementSetName>
<csw:Constraint version="1.1.0">
<csw:CqlText>dc:title like '%ips%' and dct:abstract like '%pharetra%'</csw:CqlText>
</csw:Constraint>
</csw:Query>
</csw:GetRecords>

0 comments on commit 522873e

Please sign in to comment.