Skip to content

Commit

Permalink
Rewrap a bunch of code.
Browse files Browse the repository at this point in the history
  • Loading branch information
@153957
153957 committed Aug 9, 2018
1 parent d638306 commit b899cca
Show file tree
Hide file tree
Showing 4 changed files with 55 additions and 101 deletions.
4 changes: 2 additions & 2 deletions sapphire/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,8 @@ def _get_tsv(self, urlpath, names=None):
def _retrieve_url(urlpath, base=API_BASE):
"""Open a HiSPARC API URL and read the data
:param urlpath: the api urlpath (after http://data.hisparc.nl/api/)
to retrieve
:param urlpath: the api urlpath (after the base) to retrieve
:param base: base url for the API
:return: the data returned by the api as a string
"""
Expand Down
93 changes: 32 additions & 61 deletions sapphire/esd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,17 +86,15 @@ def _first_available_numbered_path():
def load_data(file, group, tsv_file, type='events'):
"""Load downloaded event summary data into PyTables file.
If you've previously downloaded event summary data from
http://data.hisparc.nl/ in TSV format, you can load them into a PyTables
If you've previously downloaded event summary data from the HiSPARC Public
Database in TSV format, you can load them into a PyTables
file using this method. The result is equal to directly downloading data
using :func:`download_data`.
:param file: the PyTables datafile handler.
:param group: the PyTables destination group, which need not exist.
:param tsv_file: path to the tsv file downloaded from the HiSPARC
Public Database.
:param type: the datatype to load, either 'events', 'weather',
'singles' or 'lightning'.
:param tsv_file: path to the tsv file downloaded from the HiSPARC Public Database.
:param type: the datatype to load, either 'events', 'weather', 'singles', or 'lightning'.
Example::
Expand Down Expand Up @@ -128,8 +126,7 @@ def load_data(file, group, tsv_file, type='events'):
writer.store_line(line)


def download_data(file, group, station_number, start=None, end=None,
type='events', progress=True):
def download_data(file, group, station_number, start=None, end=None, type='events', progress=True):
"""Download event summary data
:param file: the PyTables datafile handler.
Expand Down Expand Up @@ -167,8 +164,7 @@ def download_data(file, group, station_number, start=None, end=None,
# sensible defaults for start and end
if start is None:
if end is not None:
raise RuntimeError("Start is None, but end is not. "
"I can't go on like this.")
raise RuntimeError("Start is None, but end is not. I can't go on like this.")
else:
yesterday = datetime.date.today() - datetime.timedelta(days=1)
start = datetime.datetime.combine(yesterday, datetime.time(0, 0))
Expand Down Expand Up @@ -208,8 +204,7 @@ def download_data(file, group, station_number, start=None, end=None,
t_end = calendar.timegm(end.utctimetuple())
t_delta = t_end - t_start
if progress:
pbar = ProgressBar(max_value=1.,
widgets=[Percentage(), Bar(), ETA()]).start()
pbar = ProgressBar(max_value=1., widgets=[Percentage(), Bar(), ETA()]).start()

# loop over lines in tsv as they come streaming in
prev_update = time.time()
Expand All @@ -218,8 +213,7 @@ def download_data(file, group, station_number, start=None, end=None,
for line in reader:
timestamp = writer.store_line(line)
# update progressbar every 0.5 seconds
if (progress and time.time() - prev_update > 0.5 and
not timestamp == 0.):
if progress and time.time() - prev_update > 0.5 and not timestamp == 0.:
pbar.update((1. * timestamp - t_start) / t_delta)
prev_update = time.time()
if progress:
Expand All @@ -234,22 +228,17 @@ def download_data(file, group, station_number, start=None, end=None,
return
else:
# Last line is data, report failed download and date/time of last line
raise Exception('Failed to complete download, last received data '
'from: %s %s.' % tuple(line[:2]))
raise Exception('Failed to complete download, last received data from: %s %s.' % tuple(line[:2]))


def download_lightning(file, group, lightning_type=4, start=None, end=None,
progress=True):
def download_lightning(file, group, lightning_type=4, start=None, end=None, progress=True):
"""Download KNMI lightning data
:param file: the PyTables datafile handler.
:param group: the PyTables destination group, which need not exist.
:param lightning_type: type of lightning, see list below for the possible
type codes.
:param start: a datetime instance defining the start of the search
interval.
:param end: a datetime instance defining the end of the search
interval.
:param lightning_type: type of lightning, see list below for the possible type codes.
:param start: a datetime instance defining the start of the search interval.
:param end: a datetime instance defining the end of the search interval.
:param progress: if True show a progressbar while downloading.
Lightning types:
Expand All @@ -269,15 +258,14 @@ def download_lightning(file, group, lightning_type=4, start=None, end=None,
if lightning_type not in range(6):
raise ValueError("Invalid lightning type.")

download_data(file, group, lightning_type, start=start, end=end,
type='lightning', progress=progress)
download_data(file, group, lightning_type, start=start, end=end, type='lightning', progress=progress)


def load_coincidences(file, tsv_file, group=''):
"""Load downloaded event summary data into PyTables file.
If you've previously downloaded coincidence data from
http://data.hisparc.nl/ in TSV format, you can load them into a PyTables
If you've previously downloaded coincidence data from the HiSPARC Public
Database in TSV format, you can load them into a PyTables
file using this method. The result is equal to directly downloading data
using :func:`download_coincidences`.
Expand Down Expand Up @@ -310,16 +298,14 @@ def load_coincidences(file, tsv_file, group=''):
coincidence.append(line)
else:
# Full coincidence has been received, store it.
_read_lines_and_store_coincidence(file, c_group,
coincidence, station_groups)
_read_lines_and_store_coincidence(file, c_group, coincidence, station_groups)
coincidence = [line]
current_coincidence = int(line[0])
file.flush()

if len(coincidence):
# Store last coincidence
_read_lines_and_store_coincidence(file, c_group, coincidence,
station_groups)
_read_lines_and_store_coincidence(file, c_group, coincidence, station_groups)

if line[0][0] == '#':
if len(line[0]) == 1:
Expand All @@ -344,10 +330,8 @@ def download_coincidences(file, group='', cluster=None, stations=None,
:param group: path of destination group, which need not exist yet.
:param cluster: HiSPARC cluster name for which to get data.
:param stations: a list of HiSPARC station numbers for which to get data.
:param start: a datetime instance defining the start of the search
interval.
:param end: a datetime instance defining the end of the search
interval.
:param start: a datetime instance defining the start of the search interval.
:param end: a datetime instance defining the end of the search interval.
:param n: the minimum number of events in the coincidence.
The start and end parameters may both be None. In that case,
Expand Down Expand Up @@ -402,8 +386,7 @@ def download_coincidences(file, group='', cluster=None, stations=None,
t_end = calendar.timegm(end.utctimetuple())
t_delta = t_end - t_start
if progress:
pbar = ProgressBar(max_value=1.,
widgets=[Percentage(), Bar(), ETA()]).start()
pbar = ProgressBar(max_value=1., widgets=[Percentage(), Bar(), ETA()]).start()

# loop over lines in tsv as they come streaming in, keep temporary
# lists until a full coincidence is in.
Expand All @@ -418,12 +401,9 @@ def download_coincidences(file, group='', cluster=None, stations=None,
coincidence.append(line)
else:
# Full coincidence has been received, store it.
timestamp = _read_lines_and_store_coincidence(file, c_group,
coincidence,
station_groups)
timestamp = _read_lines_and_store_coincidence(file, c_group, coincidence, station_groups)
# update progressbar every 0.5 seconds
if (progress and time.time() - prev_update > 0.5 and
not timestamp == 0.):
if progress and time.time() - prev_update > 0.5 and not timestamp == 0.:
pbar.update((1. * timestamp - t_start) / t_delta)
prev_update = time.time()
coincidence = [line]
Expand All @@ -432,8 +412,7 @@ def download_coincidences(file, group='', cluster=None, stations=None,

if len(coincidence):
# Store last coincidence
_read_lines_and_store_coincidence(file, c_group, coincidence,
station_groups)
_read_lines_and_store_coincidence(file, c_group, coincidence, station_groups)
if progress:
pbar.finish()

Expand Down Expand Up @@ -471,8 +450,7 @@ def _read_or_get_station_groups(file, group):
for sid, station_group in enumerate(s_index):
station_group = station_group.decode()
station = int(re_number.search(station_group).group())
groups[station] = {'group': station_group,
's_index': sid}
groups[station] = {'group': station_group, 's_index': sid}
return groups


Expand All @@ -493,8 +471,7 @@ def _get_station_groups(group):
stations = network.station_numbers(cluster=cluster['number'])
for station in stations:
groups[station] = {'group': ('%s/hisparc/cluster_%s/station_%d' %
(group, cluster['name'].lower(),
station)),
(group, cluster['name'].lower(), station)),
's_index': s_index}
s_index += 1
return groups
Expand Down Expand Up @@ -525,8 +502,7 @@ def _create_coincidences_tables(file, group, station_groups):
s_columns = {'s%d' % station: tables.BoolCol(pos=p)
for p, station in enumerate(station_groups, 12)}
description.columns.update(s_columns)
coincidences = file.create_table(coin_group, 'coincidences', description,
createparents=True)
coincidences = file.create_table(coin_group, 'coincidences', description, createparents=True)

# Create c_index
file.create_vlarray(coin_group, 'c_index', tables.UInt32Col(shape=2))
Expand Down Expand Up @@ -679,12 +655,10 @@ def _create_lightning_table(file, group):
'longitude': tables.Float32Col(pos=5),
'current': tables.Float32Col(pos=6)}

return file.create_table(group, 'lightning', description,
createparents=True)
return file.create_table(group, 'lightning', description, createparents=True)


def _read_lines_and_store_coincidence(file, c_group, coincidence,
station_groups):
def _read_lines_and_store_coincidence(file, c_group, coincidence, station_groups):
"""Read TSV lines and store coincidence
Read lines from the TSV download and store the coincidence and events.
Expand All @@ -704,8 +678,7 @@ def _read_lines_and_store_coincidence(file, c_group, coincidence,
row['N'] = len(coincidence)
row['timestamp'] = int(coincidence[0][4])
row['nanoseconds'] = int(coincidence[0][5])
row['ext_timestamp'] = (int(coincidence[0][4]) * int(1e9) +
int(coincidence[0][5]))
row['ext_timestamp'] = int(coincidence[0][4]) * int(1e9) + int(coincidence[0][5])

for event in coincidence:
station_number = int(event[1])
Expand Down Expand Up @@ -751,8 +724,7 @@ def __enter__(self):
def store_line(self, line):
"""Store a single line
:param line: the line to store as a tuple of strings (one element
per column).
:param line: the line to store as a tuple of strings (one element per column).
:return: timestamp of the stored event, or 0 if the given line was a
comment line starting with a '#'.
Expand Down Expand Up @@ -895,8 +867,7 @@ def store_line(self, line):
return 0.

# break up TSV line
(date, time_str, timestamp, nanoseconds, latitude, longitude,
current) = line[:7]
(date, time_str, timestamp, nanoseconds, latitude, longitude, current) = line[:7]

row = self.table.row

Expand Down
33 changes: 11 additions & 22 deletions sapphire/publicdb.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,37 +146,30 @@ def datetimerange(start, stop):
Example::
>>> for x in datetimerange(datetime.datetime(2010, 1, 1, 11),
... datetime.datetime(2010, 1, 1, 13)):
>>> for x in datetimerange(datetime.datetime(2010, 1, 1, 11), datetime.datetime(2010, 1, 1, 13)):
... x
...
(datetime.datetime(2010, 1, 1, 11, 0),
datetime.datetime(2010, 1, 1, 13, 0))
(datetime.datetime(2010, 1, 1, 11, 0), datetime.datetime(2010, 1, 1, 13, 0))
>>> for x in datetimerange(datetime.datetime(2010, 1, 1, 11),
... datetime.datetime(2010, 1, 2)):
>>> for x in datetimerange(datetime.datetime(2010, 1, 1, 11), datetime.datetime(2010, 1, 2)):
... x
...
(datetime.datetime(2010, 1, 1, 11, 0), None)
>>> for x in datetimerange(datetime.datetime(2010, 1, 1, 11),
... datetime.datetime(2010, 1, 2, 13)):
>>> for x in datetimerange(datetime.datetime(2010, 1, 1, 11), datetime.datetime(2010, 1, 2, 13)):
... x
...
(datetime.datetime(2010, 1, 1, 11, 0), None)
(datetime.datetime(2010, 1, 2, 0, 0),
datetime.datetime(2010, 1, 2, 13, 0))
(datetime.datetime(2010, 1, 2, 0, 0), datetime.datetime(2010, 1, 2, 13, 0))
>>> for x in datetimerange(datetime.datetime(2010, 1, 1, 11),
... datetime.datetime(2010, 1, 5, 13)):
>>> for x in datetimerange(datetime.datetime(2010, 1, 1, 11), datetime.datetime(2010, 1, 5, 13)):
... x
...
(datetime.datetime(2010, 1, 1, 11, 0), None)
(datetime.datetime(2010, 1, 2, 0, 0), None)
(datetime.datetime(2010, 1, 3, 0, 0), None)
(datetime.datetime(2010, 1, 4, 0, 0), None)
(datetime.datetime(2010, 1, 5, 0, 0),
datetime.datetime(2010, 1, 5, 13, 0))
(datetime.datetime(2010, 1, 5, 0, 0), datetime.datetime(2010, 1, 5, 13, 0))
"""
if start > stop:
Expand All @@ -186,8 +179,7 @@ def datetimerange(start, stop):
return
else:
yield start, None
cur = (start.replace(hour=0, minute=0, second=0, microsecond=0) +
datetime.timedelta(days=1))
cur = start.replace(hour=0, minute=0, second=0, microsecond=0) + datetime.timedelta(days=1)
while cur.date() < stop.date():
yield cur, None
cur += datetime.timedelta(days=1)
Expand All @@ -204,8 +196,7 @@ def _get_or_create_group(file, group):
group = file.get_node(group)
except tables.NoSuchNodeError:
parent, newgroup = os.path.split(group)
group = file.create_group(parent, newgroup, 'Data group',
createparents=True)
group = file.create_group(parent, newgroup, 'Data group', createparents=True)
return group


Expand All @@ -216,11 +207,9 @@ def _get_or_create_node(file, group, src_node):
node = file.get_node(group, src_node.name)
except tables.NoSuchNodeError:
if isinstance(src_node, tables.Table):
node = file.create_table(group, src_node.name,
src_node.description, src_node.title)
node = file.create_table(group, src_node.name, src_node.description, src_node.title)
elif isinstance(src_node, tables.VLArray):
node = file.create_vlarray(group, src_node.name, src_node.atom,
src_node.title)
node = file.create_vlarray(group, src_node.name, src_node.atom, src_node.title)
else:
raise Exception("Unknown node class: %s" % type(src_node))

Expand Down
26 changes: 10 additions & 16 deletions sapphire/tests/esd_load_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,15 @@
from sapphire import esd

self_path = os.path.dirname(__file__)

test_data_path = os.path.join(self_path, 'test_data/esd_load_data.h5')
test_data_coincidences_path = os.path.join(self_path,
'test_data/esd_coincidence_data.h5')
test_data_coincidences_path = os.path.join(self_path, 'test_data/esd_coincidence_data.h5')

events_source = os.path.join(self_path, 'test_data/events-s501-20120101.tsv')
weather_source = os.path.join(self_path, 'test_data/weather-s501-20120101.tsv')
singles_source = os.path.join(self_path, 'test_data/singles-s501-20170101.tsv')
lightning_source = os.path.join(self_path,
'test_data/lightning-knmi-20150717.tsv')
coincidences_source = os.path.join(self_path,
'test_data/coincidences-20160310.tsv')
lightning_source = os.path.join(self_path, 'test_data/lightning-knmi-20150717.tsv')
coincidences_source = os.path.join(self_path, 'test_data/coincidences-20160310.tsv')


def create_tempfile_path():
Expand Down Expand Up @@ -52,14 +51,10 @@ def perform_esd_download_data(filename):
lightning_end = datetime.datetime(2015, 7, 17, 0, 10, 0)

with tables.open_file(filename, 'w', filters=filters) as datafile:
esd.download_data(datafile, '/', 501, start, end, type='events',
progress=False)
esd.download_data(datafile, '/', 501, start, end, type='weather',
progress=False)
esd.download_data(datafile, '/', 501, singles_start, singles_end,
type='singles', progress=False)
esd.download_lightning(datafile, '/', 4, lightning_start,
lightning_end, progress=False)
esd.download_data(datafile, '/', 501, start, end, type='events', progress=False)
esd.download_data(datafile, '/', 501, start, end, type='weather', progress=False)
esd.download_data(datafile, '/', 501, singles_start, singles_end, type='singles', progress=False)
esd.download_lightning(datafile, '/', 4, lightning_start, lightning_end, progress=False)


def perform_load_coincidences(filename):
Expand All @@ -78,8 +73,7 @@ def perform_download_coincidences(filename):
end = datetime.datetime(2016, 3, 10, 0, 1, 0)

with tables.open_file(filename, 'w', filters=filters) as datafile:
esd.download_coincidences(datafile, stations=[501, 510],
start=start, end=end, n=2, progress=False)
esd.download_coincidences(datafile, stations=[501, 510], start=start, end=end, n=2, progress=False)


def create_and_store_test_data():
Expand Down

0 comments on commit b899cca

Please sign in to comment.