Skip to content

Commit daaf09b

Browse files
tomkralidistomkralidis
committed
implement CQL to OGC filter transforms
1 parent 9ddcad1 commit daaf09b

7 files changed

+217
-11
lines changed

‎pycsw/cql.py‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
# -*- coding: utf-8 -*-
2+
# =================================================================
3+
#
4+
# Authors: Tom Kralidis <[email protected]>
5+
#
6+
# Copyright (c) 2016 Tom Kralidis
7+
#
8+
# Permission is hereby granted, free of charge, to any person
9+
# obtaining a copy of this software and associated documentation
10+
# files (the "Software"), to deal in the Software without
11+
# restriction, including without limitation the rights to use,
12+
# copy, modify, merge, publish, distribute, sublicense, and/or sell
13+
# copies of the Software, and to permit persons to whom the
14+
# Software is furnished to do so, subject to the following
15+
# conditions:
16+
#
17+
# The above copyright notice and this permission notice shall be
18+
# included in all copies or substantial portions of the Software.
19+
#
20+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
22+
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24+
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25+
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26+
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
27+
# OTHER DEALINGS IN THE SOFTWARE.
28+
#
29+
# =================================================================
30+
31+
import logging
32+
33+
from lxml import etree
34+
from pycsw import util
35+
from pycsw.fes import MODEL as fes1_model
36+
37+
LOGGER = logging.getLogger(__name__)
38+
39+
40+
def cql2fes1(cql, namespaces):
41+
"""transforms Common Query Language (CQL) query into OGC fes1 syntax"""
42+
43+
filters = []
44+
tmp_list = []
45+
logical_op = None
46+
47+
LOGGER.debug('CQL: %s', cql)
48+
49+
if ' or ' in cql:
50+
logical_op = etree.Element(util.nspath_eval('ogc:Or', namespaces))
51+
tmp_list = cql.split(' or ')
52+
elif ' OR ' in cql:
53+
logical_op = etree.Element(util.nspath_eval('ogc:Or', namespaces))
54+
tmp_list = cql.split(' OR ')
55+
elif ' and ' in cql:
56+
logical_op = etree.Element(util.nspath_eval('ogc:And', namespaces))
57+
tmp_list = cql.split(' and ')
58+
elif ' AND ' in cql:
59+
logical_op = etree.Element(util.nspath_eval('ogc:And', namespaces))
60+
tmp_list = cql.split(' AND ')
61+
62+
if tmp_list:
63+
LOGGER.debug('Logical operator found (AND/OR)')
64+
else:
65+
tmp_list.append(cql)
66+
67+
for t in tmp_list:
68+
filters.append(_parse_condition(t))
69+
70+
root = etree.Element(util.nspath_eval('ogc:Filter', namespaces))
71+
72+
if logical_op is not None:
73+
root.append(logical_op)
74+
75+
for flt in filters:
76+
condition = etree.Element(util.nspath_eval(flt[0], namespaces))
77+
78+
etree.SubElement(
79+
condition,
80+
util.nspath_eval('ogc:PropertyName', namespaces)).text = flt[1]
81+
82+
etree.SubElement(
83+
condition,
84+
util.nspath_eval('ogc:Literal', namespaces)).text = flt[2]
85+
86+
if logical_op is not None:
87+
logical_op.append(condition)
88+
else:
89+
root.append(condition)
90+
91+
LOGGER.debug('Resulting OGC Filter: %s',
92+
etree.tostring(root, pretty_print=1))
93+
94+
return root
95+
96+
97+
def _parse_condition(condition):
98+
"""parses a single condition"""
99+
100+
LOGGER.debug('condition: %s', condition)
101+
102+
property_name, operator, literal = condition.split()
103+
104+
literal = literal.replace('"', '').replace('\'', '')
105+
106+
for k, v in fes1_model['ComparisonOperators'].items():
107+
if v['opvalue'] == operator:
108+
fes1_predicate = k
109+
110+
LOGGER.debug('parsed condition: %s %s %s', property_name, fes1_predicate,
111+
literal)
112+
113+
return (fes1_predicate, property_name, literal)
114+

‎pycsw/server.py‎

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
from pycsw.plugins.profiles import profile as pprofile
4141
import pycsw.plugins.outputschemas
4242
from pycsw import config, fes, log, metadata, util, sru, opensearch
43+
from pycsw.cql import cql2fes1
4344
import logging
4445

4546
LOGGER = logging.getLogger(__name__)
@@ -1239,12 +1240,20 @@ def getrecords(self):
12391240
% self.kvp['constraintlanguage'])
12401241
if self.kvp['constraintlanguage'] == 'CQL_TEXT':
12411242
tmp = self.kvp['constraint']
1242-
self.kvp['constraint'] = {}
1243-
self.kvp['constraint']['type'] = 'cql'
1244-
self.kvp['constraint']['where'] = \
1245-
self._cql_update_queryables_mappings(tmp,
1246-
self.repository.queryables['_all'])
1247-
self.kvp['constraint']['values'] = {}
1243+
try:
1244+
LOGGER.debug('Transforming CQL into fes1')
1245+
LOGGER.debug('CQL: %s', tmp)
1246+
self.kvp['constraint'] = {}
1247+
self.kvp['constraint']['type'] = 'filter'
1248+
cql = cql2fes1(tmp, self.context.namespaces)
1249+
self.kvp['constraint']['where'], self.kvp['constraint']['values'] = fes.parse(cql,
1250+
self.repository.queryables['_all'], self.repository.dbtype,
1251+
self.context.namespaces, self.orm, self.language['text'], self.repository.fts)
1252+
except Exception as err:
1253+
LOGGER.error('Invalid CQL query %s', tmp)
1254+
LOGGER.error('Error message: %s', err, exc_info=True)
1255+
return self.exceptionreport('InvalidParameterValue',
1256+
'constraint', 'Invalid Filter syntax')
12481257
elif self.kvp['constraintlanguage'] == 'FILTER':
12491258
# validate filter XML
12501259
try:
@@ -1322,8 +1331,10 @@ def getrecords(self):
13221331
maxrecords=self.kvp['maxrecords'],
13231332
startposition=int(self.kvp['startposition'])-1)
13241333
except Exception, err:
1334+
LOGGER.debug('Invalid query syntax. Query: %s', self.kvp['constraint'])
1335+
LOGGER.debug('Invalid query syntax. Result: %s', err)
13251336
return self.exceptionreport('InvalidParameterValue', 'constraint',
1326-
'Invalid query: %s' % err)
1337+
'Invalid query syntax')
13271338

13281339
dsresults = []
13291340

@@ -2385,13 +2396,21 @@ def _parse_constraint(self, element):
23852396
self.context.namespaces, self.orm, self.language['text'], self.repository.fts)
23862397
except Exception, err:
23872398
return 'Invalid Filter request: %s' % err
2399+
23882400
tmp = element.find(util.nspath_eval('csw:CqlText', self.context.namespaces))
23892401
if tmp is not None:
23902402
LOGGER.debug('CQL specified: %s.' % tmp.text)
2391-
query['type'] = 'cql'
2392-
query['where'] = self._cql_update_queryables_mappings(tmp.text,
2393-
self.repository.queryables['_all'])
2394-
query['values'] = {}
2403+
try:
2404+
LOGGER.debug('Transforming CQL into OGC Filter')
2405+
query['type'] = 'filter'
2406+
cql = cql2fes1(tmp.text, self.context.namespaces)
2407+
query['where'], query['values'] = fes.parse(cql,
2408+
self.repository.queryables['_all'], self.repository.dbtype,
2409+
self.context.namespaces, self.orm, self.language['text'], self.repository.fts)
2410+
except Exception as err:
2411+
LOGGER.error('Invalid CQL request: %s', tmp.text)
2412+
LOGGER.error('Error message: %s', err, exc_info=True)
2413+
return 'Invalid CQL request'
23952414
return query
23962415

23972416
def _test_manager(self):

‎tests/expected/suites_default_get_GetRecords-filter-cql-title-or-abstract.xml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
2+
<!-- PYCSW_VERSION -->
3+
<csw:GetRecordsResponse xmlns:soapenv="http://www.w3.org/2003/05/soap-envelope" xmlns:gml="http://www.opengis.net/gml" xmlns:dif="http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ows="http://www.opengis.net/ows" xmlns:fgdc="http://www.opengis.net/cat/csw/csdgm" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:sitemap="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:os="http://a9.com/-/spec/opensearch/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:ogc="http://www.opengis.net/ogc" version="2.0.2" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
4+
<csw:SearchStatus timestamp="PYCSW_TIMESTAMP"/>
5+
<csw:SearchResults nextRecord="0" numberOfRecordsMatched="2" numberOfRecordsReturned="2" recordSchema="http://www.opengis.net/cat/csw/2.0.2" elementSet="full">
6+
<csw:Record>
7+
<dc:identifier>urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f</dc:identifier>
8+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
9+
<dc:format>image/svg+xml</dc:format>
10+
<dc:title>Lorem ipsum</dc:title>
11+
<dct:spatial>GR-22</dct:spatial>
12+
<dc:subject>Tourism--Greece</dc:subject>
13+
<dct:abstract>Quisque lacus diam, placerat mollis, pharetra in, commodo sed, augue. Duis iaculis arcu vel arcu.</dct:abstract>
14+
</csw:Record>
15+
<csw:Record>
16+
<dc:identifier>urn:uuid:a06af396-3105-442d-8b40-22b57a90d2f2</dc:identifier>
17+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
18+
<dc:title>Lorem ipsum dolor sit amet</dc:title>
19+
<dc:format>image/jpeg</dc:format>
20+
<dct:spatial>IT-FI</dct:spatial>
21+
</csw:Record>
22+
</csw:SearchResults>
23+
</csw:GetRecordsResponse>

‎tests/expected/suites_default_get_GetRecords-filter-cql-title.xml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
2+
<!-- PYCSW_VERSION -->
3+
<csw:GetRecordsResponse xmlns:soapenv="http://www.w3.org/2003/05/soap-envelope" xmlns:gml="http://www.opengis.net/gml" xmlns:dif="http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ows="http://www.opengis.net/ows" xmlns:fgdc="http://www.opengis.net/cat/csw/csdgm" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:sitemap="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:os="http://a9.com/-/spec/opensearch/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:ogc="http://www.opengis.net/ogc" version="2.0.2" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
4+
<csw:SearchStatus timestamp="PYCSW_TIMESTAMP"/>
5+
<csw:SearchResults nextRecord="0" numberOfRecordsMatched="2" numberOfRecordsReturned="2" recordSchema="http://www.opengis.net/cat/csw/2.0.2" elementSet="full">
6+
<csw:Record>
7+
<dc:identifier>urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f</dc:identifier>
8+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
9+
<dc:format>image/svg+xml</dc:format>
10+
<dc:title>Lorem ipsum</dc:title>
11+
<dct:spatial>GR-22</dct:spatial>
12+
<dc:subject>Tourism--Greece</dc:subject>
13+
<dct:abstract>Quisque lacus diam, placerat mollis, pharetra in, commodo sed, augue. Duis iaculis arcu vel arcu.</dct:abstract>
14+
</csw:Record>
15+
<csw:Record>
16+
<dc:identifier>urn:uuid:a06af396-3105-442d-8b40-22b57a90d2f2</dc:identifier>
17+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
18+
<dc:title>Lorem ipsum dolor sit amet</dc:title>
19+
<dc:format>image/jpeg</dc:format>
20+
<dct:spatial>IT-FI</dct:spatial>
21+
</csw:Record>
22+
</csw:SearchResults>
23+
</csw:GetRecordsResponse>

‎tests/expected/suites_default_post_GetRecords-cql-title-and-abstract.xml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
2+
<!-- PYCSW_VERSION -->
3+
<csw:GetRecordsResponse xmlns:soapenv="http://www.w3.org/2003/05/soap-envelope" xmlns:gml="http://www.opengis.net/gml" xmlns:dif="http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ows="http://www.opengis.net/ows" xmlns:fgdc="http://www.opengis.net/cat/csw/csdgm" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:sitemap="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:os="http://a9.com/-/spec/opensearch/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:ogc="http://www.opengis.net/ogc" version="2.0.2" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
4+
<csw:SearchStatus timestamp="PYCSW_TIMESTAMP"/>
5+
<csw:SearchResults nextRecord="0" numberOfRecordsMatched="1" numberOfRecordsReturned="1" recordSchema="http://www.opengis.net/cat/csw/2.0.2" elementSet="brief">
6+
<csw:BriefRecord>
7+
<dc:identifier>urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f</dc:identifier>
8+
<dc:title>Lorem ipsum</dc:title>
9+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
10+
</csw:BriefRecord>
11+
</csw:SearchResults>
12+
</csw:GetRecordsResponse>

‎tests/suites/default/get/requests.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,9 @@ GetRecords-sortby-asc,PYCSW_SERVER?config=tests/suites/default/default.cfg&servi
55
GetRecords-sortby-desc,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&sortby=dc:title:D
66
GetRecords-sortby-invalid-propertyname,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&sortby=dc:titlei:A
77
GetRecords-sortby-invalid-order,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&sortby=dc:title:FOO
8+
GetRecords-filter,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&constraintlanguage=FILTER&constraint=%3Cogc%3AFilter%20xmlns%3Aogc%3D%22http%3A%2F%2Fwww.opengis.net%2Fogc%22%3E%3Cogc%3APropertyIsEqualTo%3E%3Cogc%3APropertyName%3Edc%3Atitle%3C%2Fogc%3APropertyName%3E%3Cogc%3ALiteral%3ELorem%20ipsum%3C%2Fogc%3ALiteral%3E%3C%2Fogc%3APropertyIsEqualTo%3E%3C%2Fogc%3AFilter%3E
9+
Exception-GetRepositoryItem-service-invalid1,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW%00&version=2.0.2&request=GetRepositoryItem&id=123
10+
Exception-GetRepositoryItem-service-invalid2,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW%00'&version=2.0.2&request=GetRepositoryItem&id=123
11+
Exception-GetRepositoryItem-version-invalid,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2'&request=GetRepositoryItem&id=123
12+
GetRecords-filter-cql-title,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&constraintlanguage=CQL_TEXT&constraint=dc%3Atitle%20like%20%27%25lor%25%27
13+
GetRecords-filter-cql-title-or-abstract,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&constraintlanguage=CQL_TEXT&constraint=dc%3Atitle%20like%20%27%25lor%25%27%20or%20dct%3Aabstract%20like%20%27%25pharetra%25%27

‎tests/suites/default/post/GetRecords-cql-title-and-abstract.xml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
<?xml version="1.0" encoding="ISO-8859-1" standalone="no"?>
2+
<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:ogc="http://www.opengis.net/ogc" service="CSW" version="2.0.2" resultType="results" startPosition="1" maxRecords="5" outputFormat="application/xml" outputSchema="http://www.opengis.net/cat/csw/2.0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
3+
<csw:Query typeNames="csw:Record">
4+
<csw:ElementSetName>brief</csw:ElementSetName>
5+
<csw:Constraint version="1.1.0">
6+
<csw:CqlText>dc:title like '%ips%' and dct:abstract like '%pharetra%'</csw:CqlText>
7+
</csw:Constraint>
8+
</csw:Query>
9+
</csw:GetRecords>

0 commit comments

Comments
 (0)