/
SparqlQueryLauncher.py
361 lines (297 loc) · 10.2 KB
/
SparqlQueryLauncher.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
import time
import traceback
import sys
from SPARQLWrapper import JSON, SPARQLWrapper
from askomics.libaskomics.Params import Params
import requests
from urllib3.exceptions import HTTPError
class SparqlQueryLauncher(Params):
"""SparqlQueryLauncher
Attributes
----------
endpoint : SPARQLWrapper
The triplestore endpoint
query_time : time
Query execution time
triplestore : string
triplesotre (virtuoso, fuseki ...)
"""
def __init__(self, app, session, get_result_query=False, federated=False, endpoints=None):
"""init
Parameters
----------
app : Flask
Flask app
session :
AskOmics session
"""
Params.__init__(self, app, session)
self.query_time = None
# local endpoint (for federated query engine)
self.local_endpoint_f = self.settings.get('triplestore', 'endpoint')
try:
self.local_endpoint_f = self.settings.get('federation', 'local_endpoint')
except Exception:
pass
local = False
# Use the federated query engine
if federated:
self.federated = True
self.local_query = False
self.url_endpoint = self.settings.get('federation', 'endpoint')
self.url_updatepoint = self.settings.get('federation', 'endpoint')
self.triplestore = self.settings.get('federation', 'query_engine')
# use the external endpoint
elif endpoints is not None and endpoints != [self.local_endpoint_f]:
self.federated = False
self.local_query = False
self.triplestore = "unknown"
self.url_endpoint = endpoints[0]
self.url_updatepoint = endpoints[0]
# use the local endpoint
else:
self.federated = False
self.local_query = True
self.triplestore = self.settings.get('triplestore', 'triplestore')
self.url_endpoint = self.settings.get('triplestore', 'endpoint')
self.url_updatepoint = self.settings.get('triplestore', 'updatepoint')
local = True
self.endpoint = SPARQLWrapper(self.url_endpoint, self.url_updatepoint)
if local:
try:
self.endpoint.setCredentials(
self.settings.get('triplestore', 'username'),
self.settings.get('triplestore', 'password')
)
self.endpoint.setHTTPAuth(self.settings.get('triplestore', 'http_auth', fallback="basic"))
except Exception:
pass
def load_data(self, file_name, graph, host_url):
"""Load data in function of the triplestore
Parameters
----------
file_name : string
File name to load
graph : string
graph name
host_url : string
AskOmics url
"""
if self.triplestore == 'fuseki':
self.load_data_fuseki(file_name, graph)
else:
self.load_data_virtuoso(file_name, graph, host_url)
def load_data_fuseki(self, file_name, graph):
"""Load data using fuseki load request
Parameters
----------
file_name : string
File name to load
graph : string
graph name
Returns
-------
response
Response of request
"""
file_path = "{}/{}_{}/ttl/{}".format(
self.settings.get("askomics", "data_directory"),
self.session["user"]["id"],
self.session["user"]["username"],
file_name
)
data = {'graph': graph}
files = [('file', (file_name, open(file_path), 'text/turtle'))]
start_time = time.time()
response = requests.post(self.settings.get('triplestore', 'fuseki_upload_url'), data=data, files=files)
self.query_time = time.time() - start_time
return response
def load_data_virtuoso(self, file_name, graph, host_url):
"""Load data using virtuoso load query
Parameters
----------
file_name : string
File name to load
graph : string
graph name
host_url : string
AskOmics url
Returns
-------
TYPE
result of query
"""
try:
load_url = self.settings.get('triplestore', 'load_url')
except Exception:
load_url = host_url
if not load_url.endswith('/'):
load_url = load_url + "/"
file_url = '{}api/files/ttl/{}/{}/{}'.format(
load_url,
self.session['user']['id'],
self.session['user']['username'],
file_name
)
query = "LOAD <{}> INTO GRAPH <{}>".format(file_url, graph)
return self.execute_query(query)
def get_triples_from_graph(self, graph):
"""Get triples from a rdflib graph
Parameters
----------
graph : Graph
rdf graph
Returns
-------
string
ttl string
"""
ttl = ""
for s, p, o in graph.get_triple():
ttl += "<{}> <{}> <{}> .\n".format(s, p, o)
return ttl
def insert_ttl_string(self, ttl_string, graph):
"""Insert ttl into the triplestore
Parameters
----------
ttl_string : string
ttl triples to insert
graph : string
Insert in the named graph
Returns
-------
dict?
query result
"""
query = '''
INSERT {{
GRAPH <{}> {{
{}
}}
}}
'''.format(graph, ttl_string)
return self.execute_query(query)
def insert_data(self, ttl, graph, metadata=False):
"""Insert data into the triplesotre using INSERT
Parameters
----------
ttl : Graph
rdflib graph
graph : string
graph name
metadata : bool, optional
metadatas?
Returns
-------
TYPE
query result
"""
triples = self.get_triples_from_graph(ttl) if metadata else ttl.serialize(format='nt')
query = '''
INSERT {{
GRAPH <{}> {{
{}
}}
}}
'''.format(graph, triples)
return self.execute_query(query)
def drop_dataset(self, graph):
"""Drop the datasets of the triplestore and its metadata
Parameters
----------
graph : string
graph name to remove
"""
query = '''
DROP SILENT GRAPH <{}>
'''.format(graph)
self.execute_query(query, disable_log=True, isql_api=True)
def process_query(self, query, isql_api=False):
"""Execute a query and return parsed results
Parameters
----------
query : string
The query to execute
Returns
-------
list
Parsed results
"""
return self.parse_results(self.execute_query(query, isql_api=isql_api))
def execute_query(self, query, disable_log=False, isql_api=False):
"""Execute a sparql query
Parameters
----------
query : string
Query to perform
Returns
-------
TYPE
result
"""
try:
triplestore = self.settings.get("triplestore", "triplestore")
# Use ISQL or SPARQL
isql_api_url = None
try:
isql_api_url = self.settings.get("triplestore", "isqlapi")
except Exception:
pass
use_isql = True if triplestore == "virtuoso" and isql_api_url and self.local_query and isql_api else False
start_time = time.time()
self.endpoint.setQuery(query)
# Debug
if self.settings.getboolean('askomics', 'debug'):
print(query)
self.log.debug("Launch {} query on {} ({})".format("ISQL" if use_isql else "SPARQL", self.triplestore, self.url_endpoint))
self.log.debug(query)
if use_isql:
formatted_query = "SPARQL {}".format(query)
json = {"command": formatted_query, "disable_log": disable_log, "sparql_select": not self.endpoint.isSparqlUpdateRequest()}
response = requests.post(url=isql_api_url, json=json)
results = response.json()
if results["status"] == 500:
raise HTTPError("isqlapi: {}".format(results["message"]))
else:
# Update
if self.endpoint.isSparqlUpdateRequest():
self.endpoint.setMethod('POST')
# Virtuoso hack
if self.triplestore == 'virtuoso':
self.endpoint.queryType = "SELECT"
results = self.endpoint.query()
# Select
else:
self.endpoint.setReturnFormat(JSON)
results = self.endpoint.query().convert()
self.query_time = time.time() - start_time
return results
except Exception as e:
traceback.print_exc(file=sys.stdout)
raise type(e)("Triplestore error: {}".format(str(e))).with_traceback(sys.exc_info()[2])
def parse_results(self, json_results):
"""Parse result of sparql query
Parameters
----------
json_results : dict
Query result
Returns
-------
list, list
Header and data
"""
try:
# If isql, results are allready parsed
if "isql" in json_results:
return json_results["vars"], json_results["rows"]
header = json_results['head']['vars']
data = []
for row in json_results["results"]["bindings"]:
row_dict = {}
for key, value in row.items():
row_dict[key] = value['value']
data.append(row_dict)
except Exception:
traceback.print_exc(file=sys.stdout)
return [], []
return header, data