/
client.py
392 lines (340 loc) · 14.6 KB
/
client.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Client for accessing the `IRIS Library of Nominal Response for Seismic
Instruments <https://ds.iris.edu/NRL/>`_ (NRL). To cite use of the NRL, please
see [Templeton2017]_.
:copyright:
Lloyd Carothers IRIS/PASSCAL, 2016
The ObsPy Development Team (devs@obspy.org)
:license:
GNU Lesser General Public License, Version 3
(https://www.gnu.org/copyleft/lesser.html)
"""
import codecs
import io
import os
import warnings
from configparser import ConfigParser, DuplicateSectionError
from urllib.parse import urlparse
import requests
import obspy
from obspy.core.inventory.util import _textwrap
from obspy.core.util.decorator import deprecated
# Simple cache for remote NRL access. The total data amount will always be
# fairly small so I don't think it needs any cache eviction for now.
_remote_nrl_cache = {}
class NRL(object):
"""
NRL client base class for accessing the Nominal Response Library.
https://ds.iris.edu/NRL/
Created with a URL for remote access or filesystem accessing a local copy.
.. warning::
Remote access to online NRL is deprecated as it will stop working in
Spring 2023 due to server side changes.
"""
_index = 'index.txt'
def __new__(cls, root=None):
# root provided and it's no web URL
if root:
scheme = urlparse(root).scheme
if scheme in ('http', 'https'):
return super(NRL, cls).__new__(RemoteNRL)
# Check if it's really a folder on the file-system.
if not os.path.isdir(root):
msg = ("Provided path '{}' seems to be a local file path "
"but the directory does not exist.").format(root)
raise ValueError(msg)
return super(NRL, cls).__new__(LocalNRL)
# Otherwise delegate to the remote NRL client to deal with all kinds
# of remote resources (currently only HTTP).
return super(NRL, cls).__new__(RemoteNRL)
def __init__(self):
try:
sensor_index = self._join(self.root, 'sensors', self._index)
self.sensors = self._parse_ini(sensor_index)
datalogger_index = self._join(self.root, 'dataloggers',
self._index)
self.dataloggers = self._parse_ini(datalogger_index)
self._nrl_version = 1
except FileNotFoundError:
sensor_index = self._join(self.root, 'sensor', self._index)
self.sensors = self._parse_ini(sensor_index)
datalogger_index = self._join(self.root, 'datalogger', self._index)
self.dataloggers = self._parse_ini(datalogger_index)
# version 2 also has additional base nodes "integrated" and "soh"
self._nrl_version = 2
def __str__(self):
info = ['NRL library at ' + self.root]
if self.sensors is None:
info.append(' Sensors not parsed yet.')
else:
info.append(
' Sensors: {} manufacturers'.format(len(self.sensors)))
if len(self.sensors):
keys = [key for key in sorted(self.sensors)]
lines = _textwrap("'" + "', '".join(keys) + "'",
initial_indent=' ',
subsequent_indent=' ')
info.extend(lines)
if self.dataloggers is None:
info.append(' Dataloggers not parsed yet.')
else:
info.append(' Dataloggers: {} manufacturers'.format(
len(self.dataloggers)))
if len(self.dataloggers):
keys = [key for key in sorted(self.dataloggers)]
lines = _textwrap("'" + "', '".join(keys) + "'",
initial_indent=' ',
subsequent_indent=' ')
info.extend(lines)
return '\n'.join(_i.rstrip() for _i in info)
def _repr_pretty_(self, p, cycle): # pragma: no cover
p.text(str(self))
def _choose(self, choice, path):
# Should return either a path or a resp
cp = self._get_cp_from_ini(path)
options = cp.options(choice)
if 'path' in options:
newpath = cp.get(choice, 'path')
elif 'resp' in options:
newpath = cp.get(choice, 'resp')
elif 'xml' in options:
newpath = cp.get(choice, 'xml')
# Strip quotes of new path
newpath = self._clean_str(newpath)
path = os.path.dirname(path)
return self._join(path, newpath)
def _parse_ini(self, path):
nrl_dict = NRLDict(self)
cp = self._get_cp_from_ini(path)
for section in cp.sections():
options = sorted(cp.options(section))
if section.lower() == 'main':
if options not in (['question'],
['detail', 'question']): # pragma: no cover
msg = "Unexpected structure of NRL file '{}'".format(path)
raise NotImplementedError(msg)
nrl_dict._question = self._clean_str(cp.get(section,
'question'))
continue
else:
if options == ['path']:
nrl_dict[section] = NRLPath(self._choose(section, path))
continue
# sometimes the description field is named 'description', but
# sometimes also 'descr'
# NRL version 2 does not seem to have any of the 'descr = '
# oddities anymore, but it can be downloaded in RESP format or
# StationXML format and then the option name is different
elif options in (['description', 'resp'], ['descr', 'resp'],
['resp'], ['description', 'xml']):
if 'descr' in options:
descr = cp.get(section, 'descr')
elif 'description' in options:
descr = cp.get(section, 'description')
else:
descr = '<no description>'
descr = self._clean_str(descr)
resp_path = self._choose(section, path)
if 'resp' in options:
resp_type = 'RESP'
elif 'xml' in options:
resp_type = 'STATIONXML'
else:
raise NotImplementedError(msg)
nrl_dict[section] = (descr, resp_path, resp_type)
continue
else: # pragma: no cover
msg = "Unexpected structure of NRL file '{}'".format(path)
raise NotImplementedError(msg)
return nrl_dict
def _clean_str(self, string):
return string.strip('\'"')
def get_datalogger_response(self, datalogger_keys):
"""
Get the datalogger response.
:type datalogger_keys: list[str]
:rtype: :class:`~obspy.core.inventory.response.Response`
"""
datalogger = self.dataloggers
for key in datalogger_keys:
datalogger = datalogger[key]
# Parse to an inventory object and return a response object.
description, path, resp_type = datalogger
with io.BytesIO(self._read_resp(path).encode()) as buf:
buf.seek(0, 0)
return obspy.read_inventory(
buf, format=resp_type)[0][0][0].response
def get_sensor_response(self, sensor_keys):
"""
Get the sensor response.
:type sensor_keys: list[str]
:rtype: :class:`~obspy.core.inventory.response.Response`
"""
sensor = self.sensors
for key in sensor_keys:
sensor = sensor[key]
# Parse to an inventory object and return a response object.
description, path, resp_type = sensor
with io.BytesIO(self._read_resp(path).encode()) as buf:
buf.seek(0, 0)
return obspy.read_inventory(
buf, format=resp_type)[0][0][0].response
def get_response(self, datalogger_keys, sensor_keys):
"""
Get Response from NRL tree structure
:param datalogger_keys: List of data-loggers.
:type datalogger_keys: list[str]
:param sensor_keys: List of sensors.
:type sensor_keys: list[str]
:rtype: :class:`~obspy.core.inventory.response.Response`
>>> nrl = NRL()
>>> response = nrl.get_response(
... sensor_keys=['Nanometrics', 'Trillium Compact 120 (Vault, '
... 'Posthole, OBS)', '754 V/m/s'],
... datalogger_keys=['REF TEK', 'RT 130 & 130-SMA', '1', '200'])
>>> print(response) # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
Channel Response
From M/S (Velocity in Meters per Second) to COUNTS (Digital Counts)
Overall Sensitivity: 4.74576e+08 defined at 1.000 Hz
10 stages:
Stage 1: PolesZerosResponseStage from M/S to V, gain: 754.3
Stage 2: ResponseStage from V to V, gain: 1
Stage 3: Coefficients... from V to COUNTS, gain: 629129
Stage 4: Coefficients... from COUNTS to COUNTS, gain: 1
Stage 5: Coefficients... from COUNTS to COUNTS, gain: 1
Stage 6: Coefficients... from COUNTS to COUNTS, gain: 1
Stage 7: Coefficients... from COUNTS to COUNTS, gain: 1
Stage 8: Coefficients... from COUNTS to COUNTS, gain: 1
Stage 9: Coefficients... from COUNTS to COUNTS, gain: 1
Stage 10: Coefficients... from COUNTS to COUNTS, gain: 1
"""
dl_resp = self.get_datalogger_response(datalogger_keys)
sensor_resp = self.get_sensor_response(sensor_keys)
sensor_stage0 = sensor_resp.response_stages[0]
# information on changes between NRL v1 and v2:
# https://ds.iris.edu/files/nrl/NominalResponseLibraryVersions.pdf
if self._nrl_version == 1:
# Combine both by replace stage one in the data logger with stage
# one of the sensor.
dl_resp.response_stages.pop(0)
dl_resp.response_stages.insert(0, sensor_stage0)
elif self._nrl_version == 2:
for stage in dl_resp.response_stages:
stage.stage_sequence_number += len(sensor_resp.response_stages)
dl_resp.response_stages = (
sensor_resp.response_stages + dl_resp.response_stages)
else:
raise NotImplementedError()
dl_resp.instrument_sensitivity.input_units = sensor_stage0.input_units
dl_resp.instrument_sensitivity.input_units_description = \
sensor_stage0.input_units_description
try:
dl_resp.recalculate_overall_sensitivity()
except ValueError:
msg = "Failed to recalculate overall sensitivity."
warnings.warn(msg)
return dl_resp
class NRLDict(dict):
def __init__(self, nrl):
self._nrl = nrl
def __str__(self):
if len(self):
if self._question:
info = ['{} ({} items):'.format(self._question, len(self))]
else:
info = ['{} items:'.format(len(self))]
texts = ["'{}'".format(k) for k in sorted(self.keys())]
info.extend(_textwrap(", ".join(texts), initial_indent=' ',
subsequent_indent=' '))
return '\n'.join(_i.rstrip() for _i in info)
else:
return '0 items.'
def _repr_pretty_(self, p, cycle): # pragma: no cover
p.text(str(self))
def __getitem__(self, name):
value = super(NRLDict, self).__getitem__(name)
# if encountering a not yet parsed NRL Path, expand it now
if isinstance(value, NRLPath):
value = self._nrl._parse_ini(value)
self[name] = value
return value
class NRLPath(str):
pass
class LocalNRL(NRL):
"""
Subclass of NRL for accessing local copy NRL.
"""
def __init__(self, root):
self.root = root
self._join = os.path.join
super(self.__class__, self).__init__()
def _get_cp_from_ini(self, path):
"""
Returns a configparser from a path to an index.txt
"""
try:
cp = ConfigParser()
with codecs.open(path, mode='r', encoding='UTF-8') as f:
cp.read_file(f)
# it seems requesting a full RESP archive of NRL version 2 has all
# items duplicated in the index.txt files. expecting this to be fixed
# upstream so this is just for now
except DuplicateSectionError:
cp = ConfigParser(strict=False)
with codecs.open(path, mode='r', encoding='UTF-8') as f:
cp.read_file(f)
return cp
def _read_resp(self, path):
# Returns Unicode string of RESP
with open(path, 'r') as f:
return f.read()
class RemoteNRL(NRL):
"""
DEPRECATED
Subclass of NRL for accessing remote copy of NRL.
Direct access to online NRL is deprecated as it will stop working when the
original NRLv1 gets taken offline (Spring 2023), please consider working
locally with a downloaded full copy of the old NRLv1 or new NRLv2 following
instructions on the
`NRL landing page <https://ds.iris.edu/ds/nrl/>`_.
"""
@deprecated()
def __init__(self, root='https://ds.iris.edu/NRL'):
"""
DEPRECATED
Direct access to online NRL is deprecated as it will stop working when
the original NRLv1 gets taken offline (Spring 2023), please consider
working locally with a downloaded full copy of the old NRLv1 or new
NRLv2 following instructions on the
`NRL landing page <https://ds.iris.edu/ds/nrl/>`_.
"""
self.root = root
super(self.__class__, self).__init__()
def _download(self, url):
"""
Download service with basic cache.
"""
if url not in _remote_nrl_cache:
r = requests.get(url)
_remote_nrl_cache[url] = r.text
return _remote_nrl_cache[url]
def _join(self, *paths):
url = paths[0]
for path in paths[1:]:
url = requests.compat.urljoin(url + '/', path)
return url
def _get_cp_from_ini(self, path):
'''
Returns a configparser from a path to an index.txt
'''
cp = ConfigParser()
with io.StringIO(self._download(path)) as buf:
cp.read_file(buf)
return cp
def _read_resp(self, path):
return self._download(path)
if __name__ == "__main__": # pragma: no cover
import doctest
doctest.testmod(exclude_empty=True)