From af6b5861ca9f59c5199dc69158ec798b6409e6bd Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sun, 24 Jul 2022 14:38:18 -0700 Subject: [PATCH 1/7] fix bug in start/end times --- .flake8 | 2 +- .pre-commit-config.yaml | 4 +- mth5/io/lemi424_new.py | 240 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 243 insertions(+), 3 deletions(-) create mode 100644 mth5/io/lemi424_new.py diff --git a/.flake8 b/.flake8 index d9ad0b40..c3c3f982 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -ignore = E203, E266, E501, W503, F403, F401 +ignore = E203, E266, E501, W503, F403, F401, W605 max-line-length = 79 max-complexity = 18 select = B,C,E,F,W,T4,B9 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c4ef355..68120fc3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,10 @@ repos: - repo: https://github.com/ambv/black - rev: stable + rev: 22.6.0 hooks: - id: black language_version: python3.6 - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.9 + rev: 3.9.2 hooks: - id: flake8 diff --git a/mth5/io/lemi424_new.py b/mth5/io/lemi424_new.py new file mode 100644 index 00000000..3c9ad665 --- /dev/null +++ b/mth5/io/lemi424_new.py @@ -0,0 +1,240 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue May 11 15:31:31 2021 + +:copyright: + Jared Peacock (jpeacock@usgs.gov) + +:license: MIT + +""" + +from pathlib import Path +import pandas as pd +import numpy as np +import logging + +from mth5.timeseries import ChannelTS, RunTS +from mt_metadata.timeseries import Station, Run + + +class LEMI424: + """ + Read in a LEMI424 file, this is a place holder until IRIS finalizes + their reader. + + """ + + def __init__(self, fn=None): + self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}") + self.fn = fn + self._has_data = False + self.sample_rate = 1.0 + self.chunk_size = 10000 + self.column_names = [ + "year", + "month", + "day", + "hour", + "minute", + "second", + "bx", + "by", + "bz", + "temperature_e", + "temperature_h", + "e1", + "e2", + "e3", + "e4", + "battery", + "elevation", + "latitude", + "lat_hemisphere", + "longitude", + "lon_hemisphere", + "n_satellites", + "gps_fix", + "tdiff", + ] + + if self.fn: + self.read() + + @property + def fn(self): + return self._fn + + @fn.setter + def fn(self, value): + if value is not None: + value = Path(value) + if not value.exists(): + raise IOError(f"Could not find {value}") + self._fn = value + + @property + def start(self): + if self._has_data: + return "T".join( + [ + "-".join( + [ + f"{self._df.iloc[0].year}", + f"{self._df.iloc[0].month:02d}", + f"{self._df.iloc[0].day:02d}", + ] + ), + ":".join( + [ + f"{self._df.iloc[0].hour:02d}", + f"{self._df.iloc[0].minute:02d}", + f"{self._df.iloc[0].second:02d}", + ] + ), + ] + ) + + @property + def end(self): + if self._has_data: + return "T".join( + [ + "-".join( + [ + f"{self._df.iloc[-1].year}", + f"{self._df.iloc[-1].month:02d}", + f"{self._df.iloc[-1].day:02d}", + ] + ), + ":".join( + [ + f"{self._df.iloc[-1].hour:02d}", + f"{self._df.iloc[-1].minute:02d}", + f"{self._df.iloc[-1].second:02d}", + ] + ), + ] + ) + + @property + def latitude(self): + if self._has_data: + return np.rad2deg(self._df.latitude.median() / 3600) + + @property + def longitude(self): + if self._has_data: + return np.rad2deg(self._df.longitude.median() / 3600) + + @property + def elevation(self): + if self._has_data: + return self._df.elevation.median() + + @property + def gps_lock(self): + if self._has_data: + return self._df.gps_fix.values + + @property + def station_metadata(self): + s = Station() + if self._has_data: + s.location.latitude = self.latitude + s.location.longitude = self.longitude + s.location.elevation = self.elevation + s.time_period.start = self.start + s.time_period.end = self.end + return s + + @property + def run_metadata(self): + r = Run() + r.sample_rate = self.sample_rate + r.data_logger.model = "LEMI424" + r.data_logger.manufacturer = "LEMI" + if self._has_data: + r.data_logger.power_source.voltage.start = self._df.battery.max() + r.data_logger.power_source.voltage.end = self._df.battery.min() + r.time_period.start = self.start + r.time_period.end = self.end + + def read(self, fn=None): + """ + Read a LEMI424 file using pandas + + :param fn: DESCRIPTION, defaults to None + :type fn: TYPE, optional + :return: DESCRIPTION + :rtype: TYPE + + """ + if fn is not None: + self.fn = fn + + if not self.fn.exists(): + msg = "Could not find file %s" + self.logger.error(msg, self.fn) + raise IOError(msg % self.fn) + + self._df = pd.read_csv(self.fn, delimiter="\s+", names=self.column_names) + + self._has_data = True + + def to_run_ts(self, fn=None, e_channels=["e1", "e2"]): + """ + Return a RunTS object from the data + + :param fn: DESCRIPTION, defaults to None + :type fn: TYPE, optional + :return: DESCRIPTION + :rtype: TYPE + + """ + ch_list = [] + for comp in ( + ["bx", "by", "bz"] + e_channels + ["temperature_e", "temperature_h"] + ): + if comp[0] in ["h", "b"]: + ch = ChannelTS("magnetic") + elif comp[0] in ["e"]: + ch = ChannelTS("electric") + else: + ch = ChannelTS("auxiliary") + + ch.sample_rate = self.sample_rate + ch.start = self.start + ch.ts = self._df[comp].values + ch.component = comp + ch_list.append(ch) + + return RunTS( + array_list=ch_list, + station_metadata=self.station_metadata, + run_metadata=self.run_metadata, + ) + + +# ============================================================================= +# define the reader +# ============================================================================= +def read_lemi424(fn, e_channels=["e1", "e2"], logger_file_handler=None): + """ + Read a LEMI 424 TXT file. + + :param fn: input file name + :type fn: string or Path + :param e_channels: A list of electric channels to read, + defaults to ["e1", "e2"] + :type e_channels: list of strings, optional + :return: A RunTS object with appropriate metadata + :rtype: :class:`mth5.timeseries.RunTS` + + """ + + txt_obj = LEMI424() + if logger_file_handler: + txt_obj.logger.addHandler(logger_file_handler) + txt_obj.read(fn) + return txt_obj.to_run_ts(e_channels=e_channels) From 0cf7802ead79af1d958f6fb950b250b745c3d2d5 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sun, 24 Jul 2022 16:21:15 -0700 Subject: [PATCH 2/7] add lemi reader example --- .../notebooks/lemi_reader_magdelena.ipynb | 3257 +++++++++++++++++ mth5/io/lemi424_new.py | 60 +- 2 files changed, 3305 insertions(+), 12 deletions(-) create mode 100644 docs/examples/notebooks/lemi_reader_magdelena.ipynb diff --git a/docs/examples/notebooks/lemi_reader_magdelena.ipynb b/docs/examples/notebooks/lemi_reader_magdelena.ipynb new file mode 100644 index 00000000..ea04bf4d --- /dev/null +++ b/docs/examples/notebooks/lemi_reader_magdelena.ipynb @@ -0,0 +1,3257 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "68281fe0-4875-47b0-8240-72945692705c", + "metadata": {}, + "source": [ + "## LEMI Example" + ] + }, + { + "cell_type": "markdown", + "id": "3a514086-1b8a-4e12-bc18-9a1de04746a1", + "metadata": {}, + "source": [ + "Recived from Andy Frassetto via email, 10 May, 2022.\n", + "_______________________\n", + "Karl,\n", + "\n", + "Here's one candidate. PASSCAL test set from fall 2020 in the Magdalena\n", + "mountains, so...should be fairly quiet.\n", + "\n", + "Cheers, A\n" + ] + }, + { + "cell_type": "markdown", + "id": "a004872a-e6e9-421a-84c5-a3a688cea857", + "metadata": {}, + "source": [ + "The data recieved were from a single station, and sit in a folder called \n", + "DATA0110. In general, it is recommended to group the LEMI files like this, in one folder per station." + ] + }, + { + "cell_type": "markdown", + "id": "c8c316cf-50ac-45e8-8a8b-97a280280348", + "metadata": {}, + "source": [ + "Within a station folder, there can be many files.\n", + "\n", + "Every file is associated with exactly one run.\n", + "However, some runs are associated with more than one file.\n", + "\n", + "Therefore it is desireable to group the files according to their runs.\n", + "\n", + "We could do this with subfolders, but in this example we use a dataframe\n", + "\n", + "\n", + "We can take advantage of the highly regular LEMI filename structure, \n", + "which is if the form YYYYMMDDhhmm.TXT\n", + "i.e. LEMI files start on the UTC minute.\n", + "\n", + "Thus we can easily sort these, and determine, based on filename _only_ whether the data are contiguous or not\n", + "202009302021.TXT\n", + "YYYYMMDDHHMM.TXT\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "327a5dae-6559-4f75-a6a0-c9b4673e6189", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-07-24 16:20:14,222 [line 135] mth5.setup_logger - INFO: Logging file can be found /home/kkappler/software/irismt/mth5/logs/mth5_debug.log\n" + ] + } + ], + "source": [ + "import os\n", + "import pandas as pd\n", + "from pathlib import Path\n", + "#from string import zfill\n", + "\n", + "from mth5 import read_file\n", + "from mth5 import mth5\n", + "from mth5.io.lemi424_new import LEMI424\n", + "\n", + "from mt_metadata import timeseries as metadata\n", + "from mt_metadata.utils.mttime import MTime\n" + ] + }, + { + "cell_type": "markdown", + "id": "2ee715c8-ff8a-47da-a866-0b35df118b83", + "metadata": {}, + "source": [ + "### Define path to the data\n", + "\n", + "The original data dump was in a folder called DATA0110.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "47a6458c-6932-4985-b3c6-3e0b1124c10f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Survey Directory Contents\n", + "DATA0110\n", + "from_lemi424.mth5\n", + "lemi_reader_test.py\n", + "stations\n", + "test_read_multiple_lemi.py\n" + ] + }, + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "survey_dir = Path(r\"/home/kkappler/software/irismt/aurora/tests/LEMI/\")\n", + "cmd = f\"ls {survey_dir}\"\n", + "print(\"Survey Directory Contents\")\n", + "os.system(cmd)" + ] + }, + { + "cell_type": "markdown", + "id": "ef319ad9-bb42-4c6e-92fb-21e15c5b6d71", + "metadata": {}, + "source": [ + "Let's make a _stations_ folder to better emulate how the data would be stored in a survey directory" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e1a1a96e-9417-4a23-b084-4ea06f63e973", + "metadata": {}, + "outputs": [], + "source": [ + "stations_dir = survey_dir.joinpath(\"stations\")\n", + "stations_dir.mkdir(exist_ok=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d2d86e19-961f-411a-9816-96155c309383", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DATA0110\n", + "from_lemi424.mth5\n", + "lemi_reader_test.py\n", + "stations\n", + "test_read_multiple_lemi.py\n" + ] + }, + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "os.system(cmd)" + ] + }, + { + "cell_type": "markdown", + "id": "1722d71a-914d-4d6b-9f46-ef12257da1a8", + "metadata": {}, + "source": [ + "Now in the stations folder, let's create a symlink to DATA0110.\n", + "Give the station a name, like 53" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "cef97bc3-fdd5-40ad-9ad1-302cb803e134", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ln -s /home/kkappler/software/irismt/aurora/tests/LEMI/DATA0110 /home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "ln: " + ] + }, + { + "data": { + "text/plain": [ + "256" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "failed to create symbolic link '/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/DATA0110': File exists\n" + ] + } + ], + "source": [ + "original_station_dir = survey_dir.joinpath(\"DATA0110\")\n", + "symlink_path = stations_dir.joinpath(\"station_53\")\n", + "cmd = f\"ln -s {original_station_dir} {symlink_path}\"\n", + "#cmd = f\"ln -s {symlink_path} {original_station_dir}\"\n", + "print(cmd)\n", + "os.system(cmd)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "30556705-5c14-425c-8a9e-ffc8cf1b0355", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "202009302021.TXT\n", + "202009302029.TXT\n", + "202009302054.TXT\n", + "202009302112.TXT\n", + "202009302114.TXT\n", + "202010010000.TXT\n", + "202010020000.TXT\n", + "202010030000.TXT\n", + "202010040000.TXT\n", + "202010050000.TXT\n", + "202010060000.TXT\n", + "202010070000.TXT\n", + "DATA0110\n", + "readme\n" + ] + }, + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "os.system(f\"ls {symlink_path}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "300e960d-c648-4a02-a281-285ab79f5239", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FILES:\n", + "\n", + "/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202009302021.TXT\n", + "/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202009302029.TXT\n", + "/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202009302054.TXT\n", + "/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202009302112.TXT\n", + "/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202009302114.TXT\n", + "/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010010000.TXT\n", + "/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010020000.TXT\n", + "/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010030000.TXT\n", + "/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010040000.TXT\n", + "/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010050000.TXT\n", + "/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010060000.TXT\n", + "/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010070000.TXT\n" + ] + } + ], + "source": [ + "p = symlink_path.glob(\"*.TXT\")\n", + "files_list = [x for x in p if x.is_file()]\n", + "files_list.sort() #Important: List is sorted so the files are sequential. We leverage this property\n", + "\n", + "print(\"FILES:\\n\")\n", + "for file in files_list:\n", + " print(file)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "d5130d40-38c7-4503-a97b-88e770e344f2", + "metadata": {}, + "source": [ + "### Make a list of LEMI424 objects, one per file" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "8d62ab7d-337a-4fb8-ac12-21e0ac8abd08", + "metadata": {}, + "outputs": [], + "source": [ + "l424_list = [LEMI424(fn=x) for x in files_list]" + ] + }, + { + "cell_type": "markdown", + "id": "7072b81a-1967-4e88-a7e9-5c237f9d7b48", + "metadata": {}, + "source": [ + "### Read in the data" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "deeb40f0-ecb5-437f-aa79-944464080839", + "metadata": {}, + "outputs": [], + "source": [ + "for l424 in l424_list:\n", + " l424.read()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "945f5474-ccfb-473b-a409-8c3e911c8bd3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "202009302021\n", + "2020-09-30T20:21:00\n", + "2020-09-30T20:28:15\n" + ] + } + ], + "source": [ + "L0 = l424_list[0]\n", + "print(L0.fn[0].stem)\n", + "print(L0.start)\n", + "print(L0.end)\n" + ] + }, + { + "cell_type": "markdown", + "id": "005c8ab1-79bd-47fb-b42c-2bdeacf53c98", + "metadata": {}, + "source": [ + "#### Now info can be accessed via data frame" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "4bbb966a-d056-4d11-869d-efdf76f3e1cc", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
yearmonthdayhourminutesecondbxbybztemperature_e...e4batteryelevationlatitudelat_hemispherelongitudelon_hemispheren_satellitesgps_fixtdiff
020209302021023813.621729.81641802.04239.76...9.71513.012204.53404.83911N10712.84475W1220
120209302021123813.586729.84241802.03039.76...9.54013.012204.53404.83911N10712.84473W1220
220209302021223813.553729.87541802.05839.75...9.52113.012204.63404.83910N10712.84470W1220
320209302021323813.477729.87841802.04239.81...9.35713.012204.73404.83910N10712.84468W1220
420209302021423813.449729.90841802.03439.77...9.42813.012204.73404.83909N10712.84467W1220
\n", + "

5 rows × 24 columns

\n", + "
" + ], + "text/plain": [ + " year month day hour minute second bx by bz \\\n", + "0 2020 9 30 20 21 0 23813.621 729.816 41802.042 \n", + "1 2020 9 30 20 21 1 23813.586 729.842 41802.030 \n", + "2 2020 9 30 20 21 2 23813.553 729.875 41802.058 \n", + "3 2020 9 30 20 21 3 23813.477 729.878 41802.042 \n", + "4 2020 9 30 20 21 4 23813.449 729.908 41802.034 \n", + "\n", + " temperature_e ... e4 battery elevation latitude lat_hemisphere \\\n", + "0 39.76 ... 9.715 13.01 2204.5 3404.83911 N \n", + "1 39.76 ... 9.540 13.01 2204.5 3404.83911 N \n", + "2 39.75 ... 9.521 13.01 2204.6 3404.83910 N \n", + "3 39.81 ... 9.357 13.01 2204.7 3404.83910 N \n", + "4 39.77 ... 9.428 13.01 2204.7 3404.83909 N \n", + "\n", + " longitude lon_hemisphere n_satellites gps_fix tdiff \n", + "0 10712.84475 W 12 2 0 \n", + "1 10712.84473 W 12 2 0 \n", + "2 10712.84470 W 12 2 0 \n", + "3 10712.84468 W 12 2 0 \n", + "4 10712.84467 W 12 2 0 \n", + "\n", + "[5 rows x 24 columns]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "l424_list[0]._df[0:5]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "4aa1d729-9c13-4108-8d86-242c4bc8bb8f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
yearmonthdayhourminutesecondbxbybztemperature_e...e4batteryelevationlatitudelat_hemispherelongitudelon_hemispheren_satellitesgps_fixtdiff
431202093020281123784.690215.65341830.50342.52...11.65013.012204.83404.83945N10712.84481W1220
432202093020281223784.723215.63741830.47242.54...12.19313.012204.83404.83946N10712.84481W1220
433202093020281323784.766215.61941830.44142.54...12.81613.012204.83404.83948N10712.84479W1220
434202093020281423784.856215.60141830.44142.58...13.34513.002204.83404.83949N10712.84479W1220
435202093020281523784.928215.57341830.44542.57...13.81713.002204.73404.83950N10712.84478W1220
\n", + "

5 rows × 24 columns

\n", + "
" + ], + "text/plain": [ + " year month day hour minute second bx by bz \\\n", + "431 2020 9 30 20 28 11 23784.690 215.653 41830.503 \n", + "432 2020 9 30 20 28 12 23784.723 215.637 41830.472 \n", + "433 2020 9 30 20 28 13 23784.766 215.619 41830.441 \n", + "434 2020 9 30 20 28 14 23784.856 215.601 41830.441 \n", + "435 2020 9 30 20 28 15 23784.928 215.573 41830.445 \n", + "\n", + " temperature_e ... e4 battery elevation latitude \\\n", + "431 42.52 ... 11.650 13.01 2204.8 3404.83945 \n", + "432 42.54 ... 12.193 13.01 2204.8 3404.83946 \n", + "433 42.54 ... 12.816 13.01 2204.8 3404.83948 \n", + "434 42.58 ... 13.345 13.00 2204.8 3404.83949 \n", + "435 42.57 ... 13.817 13.00 2204.7 3404.83950 \n", + "\n", + " lat_hemisphere longitude lon_hemisphere n_satellites gps_fix tdiff \n", + "431 N 10712.84481 W 12 2 0 \n", + "432 N 10712.84481 W 12 2 0 \n", + "433 N 10712.84479 W 12 2 0 \n", + "434 N 10712.84479 W 12 2 0 \n", + "435 N 10712.84478 W 12 2 0 \n", + "\n", + "[5 rows x 24 columns]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "l424_list[0]._df[-5:]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "3ee6c95f-cd75-45f7-88e7-25e6a4420261", + "metadata": {}, + "outputs": [], + "source": [ + "COLUMNS = [\"file_path\",\n", + " \"first_sample_time\", \n", + " \"last_sample_time\", \n", + " \"num_lines\", \n", + " \"run_id\", \n", + " \"sample_rate\",\n", + " \"new_run\",\n", + " \"file_base\"]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "cad8c364-b4ef-48b1-96e1-746138260416", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "12" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "n_files = len(files_list)\n", + "n_files\n", + "#start, end are first and last sampe time respectivel" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "c8db9eff-df74-4eef-879b-f44b46c0c440", + "metadata": {}, + "outputs": [], + "source": [ + "data_dict = {}\n", + "for col in COLUMNS:\n", + " data_dict[col] = n_files * [None]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "2eaefcde-8bcb-43df-af3b-7e7661c18e7a", + "metadata": {}, + "outputs": [], + "source": [ + "for i_file in range(n_files):\n", + " data_dict[\"file_path\"][i_file] = files_list[i_file]\n", + " data_dict[\"first_sample_time\"][i_file] = pd.Timestamp(l424_list[i_file].start)\n", + " data_dict[\"last_sample_time\"][i_file] = pd.Timestamp(l424_list[i_file].end)\n", + " data_dict[\"num_lines\"][i_file] = len(l424_list[i_file]._df)\n", + " data_dict[\"run_id\"][i_file] = \"\"\n", + " data_dict[\"sample_rate\"][i_file] = l424_list[i_file].sample_rate\n", + " data_dict[\"new_run\"][i_file] = True\n", + " data_dict[\"file_base\"][i_file] = files_list[i_file].name\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "92bed2d6-2e41-4527-be73-281a3cf68c29", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
first_sample_timelast_sample_timenum_linesrun_idsample_ratenew_runfile_base
02020-09-30 20:21:002020-09-30 20:28:154361.0True202009302021.TXT
12020-09-30 20:29:002020-09-30 20:42:167971.0True202009302029.TXT
22020-09-30 20:54:002020-09-30 21:11:0110221.0True202009302054.TXT
32020-09-30 21:12:002020-09-30 21:13:451061.0True202009302112.TXT
42020-09-30 21:14:002020-09-30 23:59:5999601.0True202009302114.TXT
52020-10-01 00:00:002020-10-01 23:59:59864001.0True202010010000.TXT
62020-10-02 00:00:002020-10-02 23:59:59864001.0True202010020000.TXT
72020-10-03 00:00:002020-10-03 23:59:59864001.0True202010030000.TXT
82020-10-04 00:00:002020-10-04 23:59:59864001.0True202010040000.TXT
92020-10-05 00:00:002020-10-05 23:59:59864001.0True202010050000.TXT
102020-10-06 00:00:002020-10-06 23:59:59864001.0True202010060000.TXT
112020-10-07 00:00:002020-10-07 14:19:46515871.0True202010070000.TXT
\n", + "
" + ], + "text/plain": [ + " first_sample_time last_sample_time num_lines run_id sample_rate \\\n", + "0 2020-09-30 20:21:00 2020-09-30 20:28:15 436 1.0 \n", + "1 2020-09-30 20:29:00 2020-09-30 20:42:16 797 1.0 \n", + "2 2020-09-30 20:54:00 2020-09-30 21:11:01 1022 1.0 \n", + "3 2020-09-30 21:12:00 2020-09-30 21:13:45 106 1.0 \n", + "4 2020-09-30 21:14:00 2020-09-30 23:59:59 9960 1.0 \n", + "5 2020-10-01 00:00:00 2020-10-01 23:59:59 86400 1.0 \n", + "6 2020-10-02 00:00:00 2020-10-02 23:59:59 86400 1.0 \n", + "7 2020-10-03 00:00:00 2020-10-03 23:59:59 86400 1.0 \n", + "8 2020-10-04 00:00:00 2020-10-04 23:59:59 86400 1.0 \n", + "9 2020-10-05 00:00:00 2020-10-05 23:59:59 86400 1.0 \n", + "10 2020-10-06 00:00:00 2020-10-06 23:59:59 86400 1.0 \n", + "11 2020-10-07 00:00:00 2020-10-07 14:19:46 51587 1.0 \n", + "\n", + " new_run file_base \n", + "0 True 202009302021.TXT \n", + "1 True 202009302029.TXT \n", + "2 True 202009302054.TXT \n", + "3 True 202009302112.TXT \n", + "4 True 202009302114.TXT \n", + "5 True 202010010000.TXT \n", + "6 True 202010020000.TXT \n", + "7 True 202010030000.TXT \n", + "8 True 202010040000.TXT \n", + "9 True 202010050000.TXT \n", + "10 True 202010060000.TXT \n", + "11 True 202010070000.TXT " + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "station_data_df = pd.DataFrame(data=data_dict)\n", + "station_data_df[COLUMNS[1:]]" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "447a8eb9-608f-46cf-9d88-c2f421b7e614", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SAME RUN\n", + "SAME RUN\n", + "SAME RUN\n", + "SAME RUN\n", + "SAME RUN\n", + "SAME RUN\n", + "SAME RUN\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
first_sample_timelast_sample_timenum_linesrun_idsample_ratenew_runfile_base
02020-09-30 20:21:002020-09-30 20:28:154361.0True202009302021.TXT
12020-09-30 20:29:002020-09-30 20:42:167971.0True202009302029.TXT
22020-09-30 20:54:002020-09-30 21:11:0110221.0True202009302054.TXT
32020-09-30 21:12:002020-09-30 21:13:451061.0True202009302112.TXT
42020-09-30 21:14:002020-09-30 23:59:5999601.0True202009302114.TXT
52020-10-01 00:00:002020-10-01 23:59:59864001.0False202010010000.TXT
62020-10-02 00:00:002020-10-02 23:59:59864001.0False202010020000.TXT
72020-10-03 00:00:002020-10-03 23:59:59864001.0False202010030000.TXT
82020-10-04 00:00:002020-10-04 23:59:59864001.0False202010040000.TXT
92020-10-05 00:00:002020-10-05 23:59:59864001.0False202010050000.TXT
102020-10-06 00:00:002020-10-06 23:59:59864001.0False202010060000.TXT
112020-10-07 00:00:002020-10-07 14:19:46515871.0False202010070000.TXT
\n", + "
" + ], + "text/plain": [ + " first_sample_time last_sample_time num_lines run_id sample_rate \\\n", + "0 2020-09-30 20:21:00 2020-09-30 20:28:15 436 1.0 \n", + "1 2020-09-30 20:29:00 2020-09-30 20:42:16 797 1.0 \n", + "2 2020-09-30 20:54:00 2020-09-30 21:11:01 1022 1.0 \n", + "3 2020-09-30 21:12:00 2020-09-30 21:13:45 106 1.0 \n", + "4 2020-09-30 21:14:00 2020-09-30 23:59:59 9960 1.0 \n", + "5 2020-10-01 00:00:00 2020-10-01 23:59:59 86400 1.0 \n", + "6 2020-10-02 00:00:00 2020-10-02 23:59:59 86400 1.0 \n", + "7 2020-10-03 00:00:00 2020-10-03 23:59:59 86400 1.0 \n", + "8 2020-10-04 00:00:00 2020-10-04 23:59:59 86400 1.0 \n", + "9 2020-10-05 00:00:00 2020-10-05 23:59:59 86400 1.0 \n", + "10 2020-10-06 00:00:00 2020-10-06 23:59:59 86400 1.0 \n", + "11 2020-10-07 00:00:00 2020-10-07 14:19:46 51587 1.0 \n", + "\n", + " new_run file_base \n", + "0 True 202009302021.TXT \n", + "1 True 202009302029.TXT \n", + "2 True 202009302054.TXT \n", + "3 True 202009302112.TXT \n", + "4 True 202009302114.TXT \n", + "5 False 202010010000.TXT \n", + "6 False 202010020000.TXT \n", + "7 False 202010030000.TXT \n", + "8 False 202010040000.TXT \n", + "9 False 202010050000.TXT \n", + "10 False 202010060000.TXT \n", + "11 False 202010070000.TXT " + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "new_run = n_files * [True]\n", + "for i_row, row in station_data_df.iterrows():\n", + " if i_row == 0:\n", + " pass\n", + "# row.run_id = run_id_str\n", + " else:\n", + " #Check of sample rate changed\n", + " previous = station_data_df.loc[i_row-1]\n", + " if previous.sample_rate != row.sample_rate:\n", + " print(\"CHANGED SAMPLE RATE\")\n", + " new_run[i_row] = True\n", + " #row.new_run = True\n", + " continue\n", + " \n", + " #check for continuity with previous\n", + " dt = pd.Timedelta(seconds=1./previous.sample_rate)\n", + " previous_next_sample = previous.last_sample_time + dt\n", + "\n", + " \n", + " if row.first_sample_time == previous_next_sample:\n", + " print(\"SAME RUN\")\n", + " new_run[i_row] = False\n", + " #station_data_df.iloc[i_row].replace(to_replace=True, value = False)\n", + " #station_data_df.at[i_row].new_run = False\n", + " continue\n", + " \n", + "station_data_df[\"new_run\"] = new_run \n", + "station_data_df[COLUMNS[1:]]\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2783b71a-06ad-4855-b92f-d9db81940a7b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'001'" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "run_id_int = 1\n", + "run_id_str = str(run_id_int).zfill(3)\n", + "run_id_str" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "d5cf56ca-1b46-421f-99cb-adf377b661f5", + "metadata": {}, + "outputs": [], + "source": [ + "run_ids = n_files * [\"\"]\n", + "run_ids[0] = run_id_str" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "f5e4b88f-1055-47ae-9b7d-ff4d97ffdbbf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['001', '002', '003', '004', '005', '005', '005', '005', '005', '005', '005', '005']\n" + ] + } + ], + "source": [ + "for i_row, row in station_data_df.iterrows():\n", + " if i_row==0:\n", + " run_ids[i_row] = run_id_str\n", + " continue\n", + " if row.new_run:\n", + " run_id_int += 1\n", + " run_id_str = str(run_id_int).zfill(3)\n", + " run_ids[i_row] = run_id_str\n", + "print(run_ids)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "b37ac4bd-87e3-44a8-aadc-c32c21fee65d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
first_sample_timelast_sample_timenum_linesrun_idsample_ratenew_runfile_base
02020-09-30 20:21:002020-09-30 20:28:154360011.0True202009302021.TXT
12020-09-30 20:29:002020-09-30 20:42:167970021.0True202009302029.TXT
22020-09-30 20:54:002020-09-30 21:11:0110220031.0True202009302054.TXT
32020-09-30 21:12:002020-09-30 21:13:451060041.0True202009302112.TXT
42020-09-30 21:14:002020-09-30 23:59:5999600051.0True202009302114.TXT
52020-10-01 00:00:002020-10-01 23:59:59864000051.0False202010010000.TXT
62020-10-02 00:00:002020-10-02 23:59:59864000051.0False202010020000.TXT
72020-10-03 00:00:002020-10-03 23:59:59864000051.0False202010030000.TXT
82020-10-04 00:00:002020-10-04 23:59:59864000051.0False202010040000.TXT
92020-10-05 00:00:002020-10-05 23:59:59864000051.0False202010050000.TXT
102020-10-06 00:00:002020-10-06 23:59:59864000051.0False202010060000.TXT
112020-10-07 00:00:002020-10-07 14:19:46515870051.0False202010070000.TXT
\n", + "
" + ], + "text/plain": [ + " first_sample_time last_sample_time num_lines run_id sample_rate \\\n", + "0 2020-09-30 20:21:00 2020-09-30 20:28:15 436 001 1.0 \n", + "1 2020-09-30 20:29:00 2020-09-30 20:42:16 797 002 1.0 \n", + "2 2020-09-30 20:54:00 2020-09-30 21:11:01 1022 003 1.0 \n", + "3 2020-09-30 21:12:00 2020-09-30 21:13:45 106 004 1.0 \n", + "4 2020-09-30 21:14:00 2020-09-30 23:59:59 9960 005 1.0 \n", + "5 2020-10-01 00:00:00 2020-10-01 23:59:59 86400 005 1.0 \n", + "6 2020-10-02 00:00:00 2020-10-02 23:59:59 86400 005 1.0 \n", + "7 2020-10-03 00:00:00 2020-10-03 23:59:59 86400 005 1.0 \n", + "8 2020-10-04 00:00:00 2020-10-04 23:59:59 86400 005 1.0 \n", + "9 2020-10-05 00:00:00 2020-10-05 23:59:59 86400 005 1.0 \n", + "10 2020-10-06 00:00:00 2020-10-06 23:59:59 86400 005 1.0 \n", + "11 2020-10-07 00:00:00 2020-10-07 14:19:46 51587 005 1.0 \n", + "\n", + " new_run file_base \n", + "0 True 202009302021.TXT \n", + "1 True 202009302029.TXT \n", + "2 True 202009302054.TXT \n", + "3 True 202009302112.TXT \n", + "4 True 202009302114.TXT \n", + "5 False 202010010000.TXT \n", + "6 False 202010020000.TXT \n", + "7 False 202010030000.TXT \n", + "8 False 202010040000.TXT \n", + "9 False 202010050000.TXT \n", + "10 False 202010060000.TXT \n", + "11 False 202010070000.TXT " + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "station_data_df[\"run_id\"] = run_ids\n", + "station_data_df[COLUMNS[1:]]" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "7d1e5e3a-fa85-407c-8c5f-e70bb0ed733c", + "metadata": {}, + "outputs": [], + "source": [ + "grouper = station_data_df.groupby(\"run_id\")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "53460041-dcfc-4da4-9d4e-0c8308d9fcf4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5\n", + "001\n", + "002\n", + "003\n", + "004\n", + "005\n" + ] + } + ], + "source": [ + "print(len(grouper))\n", + "fns = {}#len(grouper) * [None]\n", + "for run, grouped_df in grouper:\n", + " print(run)\n", + " #print(grouped_df[\"run_id\"])\n", + " fns[run] = grouped_df[\"file_path\"].to_list()\n", + " #print(grouped_df[\"file_path\"].to_list())" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "42adef43-c662-4495-9671-16e9304e9590", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'001': [PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202009302021.TXT')],\n", + " '002': [PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202009302029.TXT')],\n", + " '003': [PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202009302054.TXT')],\n", + " '004': [PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202009302112.TXT')],\n", + " '005': [PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202009302114.TXT'),\n", + " PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010010000.TXT'),\n", + " PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010020000.TXT'),\n", + " PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010030000.TXT'),\n", + " PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010040000.TXT'),\n", + " PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010050000.TXT'),\n", + " PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010060000.TXT'),\n", + " PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010070000.TXT')]}" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fns" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "c9d341b9-e3bc-4498-9175-300d40010046", + "metadata": {}, + "outputs": [], + "source": [ + "lemis = {}" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "4a7f9d48-6548-4f3c-81b1-f0cb97724965", + "metadata": {}, + "outputs": [], + "source": [ + "for run_id in fns.keys():\n", + " tmp = LEMI424(fn=fns[run_id]) \n", + " tmp = tmp.to_run_ts()\n", + " tmp.run_metadata.id = run_id\n", + " lemis[run_id] = tmp" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "aa6e0913-2fce-431b-bc85-3087aadfab37", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "RunTS Summary:\n", + "\tStation: None\n", + "\tRun: 001\n", + "\tStart: 2020-09-30T20:21:00+00:00\n", + "\tEnd: 2020-09-30T20:28:15+00:00\n", + "\tSample Rate: 1.0\n", + "\tComponents: ['bx', 'by', 'bz', 'e1', 'e2', 'temperature_e', 'temperature_h']" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lemis[\"001\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "f1b22561-7c05-436a-abad-a7b3c6cf39f9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{\n", + " \"run\": {\n", + " \"channels_recorded_auxiliary\": [\n", + " \"temperature_e\",\n", + " \"temperature_h\"\n", + " ],\n", + " \"channels_recorded_electric\": [\n", + " \"e1\",\n", + " \"e2\"\n", + " ],\n", + " \"channels_recorded_magnetic\": [\n", + " \"bx\",\n", + " \"by\",\n", + " \"bz\"\n", + " ],\n", + " \"data_logger.firmware.author\": null,\n", + " \"data_logger.firmware.name\": null,\n", + " \"data_logger.firmware.version\": null,\n", + " \"data_logger.id\": null,\n", + " \"data_logger.manufacturer\": null,\n", + " \"data_logger.timing_system.drift\": 0.0,\n", + " \"data_logger.timing_system.type\": \"GPS\",\n", + " \"data_logger.timing_system.uncertainty\": 0.0,\n", + " \"data_logger.type\": null,\n", + " \"data_type\": \"BBMT\",\n", + " \"id\": \"001\",\n", + " \"sample_rate\": 1.0,\n", + " \"time_period.end\": \"2020-09-30T20:28:15+00:00\",\n", + " \"time_period.start\": \"2020-09-30T20:21:00+00:00\"\n", + " }\n", + "}" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lemis[\"001\"].run_metadata" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "31afe739-e93e-4718-b0ab-881ab2de0e59", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "mth5.timeseries.run_ts.RunTS" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "type(lemis[\"001\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "4a6689ff-e1aa-437d-9a33-2bbd800bd0ef", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:        (time: 436)\n",
+       "Coordinates:\n",
+       "  * time           (time) datetime64[ns] 2020-09-30T20:21:00 ... 2020-09-30T2...\n",
+       "Data variables:\n",
+       "    bx             (time) float64 2.381e+04 2.381e+04 ... 2.378e+04 2.378e+04\n",
+       "    by             (time) float64 729.8 729.8 729.9 729.9 ... 215.6 215.6 215.6\n",
+       "    bz             (time) float64 4.18e+04 4.18e+04 ... 4.183e+04 4.183e+04\n",
+       "    e1             (time) float64 131.0 130.9 130.9 130.9 ... 133.8 134.2 134.6\n",
+       "    e2             (time) float64 -111.0 -111.2 -111.2 ... -109.3 -108.7 -108.3\n",
+       "    temperature_e  (time) float64 39.76 39.76 39.75 39.81 ... 42.54 42.58 42.57\n",
+       "    temperature_h  (time) float64 40.48 40.48 40.49 40.49 ... 41.31 41.32 41.32\n",
+       "Attributes:\n",
+       "    channels_recorded_auxiliary:            ['temperature_e', 'temperature_h']\n",
+       "    channels_recorded_electric:             ['e1', 'e2']\n",
+       "    channels_recorded_magnetic:             ['bx', 'by', 'bz']\n",
+       "    data_logger.firmware.author:            None\n",
+       "    data_logger.firmware.name:              None\n",
+       "    data_logger.firmware.version:           None\n",
+       "    data_logger.id:                         None\n",
+       "    data_logger.manufacturer:               None\n",
+       "    data_logger.timing_system.drift:        0.0\n",
+       "    data_logger.timing_system.type:         GPS\n",
+       "    data_logger.timing_system.uncertainty:  0.0\n",
+       "    data_logger.type:                       None\n",
+       "    data_type:                              BBMT\n",
+       "    id:                                     None\n",
+       "    sample_rate:                            1.0\n",
+       "    time_period.end:                        2020-09-30T20:28:15+00:00\n",
+       "    time_period.start:                      2020-09-30T20:21:00+00:00
" + ], + "text/plain": [ + "\n", + "Dimensions: (time: 436)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 2020-09-30T20:21:00 ... 2020-09-30T2...\n", + "Data variables:\n", + " bx (time) float64 2.381e+04 2.381e+04 ... 2.378e+04 2.378e+04\n", + " by (time) float64 729.8 729.8 729.9 729.9 ... 215.6 215.6 215.6\n", + " bz (time) float64 4.18e+04 4.18e+04 ... 4.183e+04 4.183e+04\n", + " e1 (time) float64 131.0 130.9 130.9 130.9 ... 133.8 134.2 134.6\n", + " e2 (time) float64 -111.0 -111.2 -111.2 ... -109.3 -108.7 -108.3\n", + " temperature_e (time) float64 39.76 39.76 39.75 39.81 ... 42.54 42.58 42.57\n", + " temperature_h (time) float64 40.48 40.48 40.49 40.49 ... 41.31 41.32 41.32\n", + "Attributes:\n", + " channels_recorded_auxiliary: ['temperature_e', 'temperature_h']\n", + " channels_recorded_electric: ['e1', 'e2']\n", + " channels_recorded_magnetic: ['bx', 'by', 'bz']\n", + " data_logger.firmware.author: None\n", + " data_logger.firmware.name: None\n", + " data_logger.firmware.version: None\n", + " data_logger.id: None\n", + " data_logger.manufacturer: None\n", + " data_logger.timing_system.drift: 0.0\n", + " data_logger.timing_system.type: GPS\n", + " data_logger.timing_system.uncertainty: 0.0\n", + " data_logger.type: None\n", + " data_type: BBMT\n", + " id: None\n", + " sample_rate: 1.0\n", + " time_period.end: 2020-09-30T20:28:15+00:00\n", + " time_period.start: 2020-09-30T20:21:00+00:00" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lemis[\"001\"].dataset\n" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "978b6a3f-e317-44b2-86df-1713674e1efe", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:        (time: 579947)\n",
+       "Coordinates:\n",
+       "  * time           (time) datetime64[ns] 2020-09-30T21:14:00 ... 2020-10-07T1...\n",
+       "Data variables:\n",
+       "    bx             (time) float64 2.378e+04 2.378e+04 ... 2.378e+04 2.378e+04\n",
+       "    by             (time) float64 213.9 213.9 213.9 213.9 ... 247.2 247.3 247.3\n",
+       "    bz             (time) float64 4.183e+04 4.183e+04 ... 4.184e+04 4.184e+04\n",
+       "    e1             (time) float64 134.8 134.9 135.0 135.1 ... 130.4 130.3 130.2\n",
+       "    e2             (time) float64 -109.8 -109.9 -109.9 ... -40.88 -40.95 -40.93\n",
+       "    temperature_e  (time) float64 46.5 46.46 46.47 46.47 ... 18.18 18.17 18.17\n",
+       "    temperature_h  (time) float64 36.3 36.3 36.3 36.3 ... 19.61 19.6 19.6 19.6\n",
+       "Attributes:\n",
+       "    channels_recorded_auxiliary:            ['temperature_e', 'temperature_h']\n",
+       "    channels_recorded_electric:             ['e1', 'e2']\n",
+       "    channels_recorded_magnetic:             ['bx', 'by', 'bz']\n",
+       "    data_logger.firmware.author:            None\n",
+       "    data_logger.firmware.name:              None\n",
+       "    data_logger.firmware.version:           None\n",
+       "    data_logger.id:                         None\n",
+       "    data_logger.manufacturer:               None\n",
+       "    data_logger.timing_system.drift:        0.0\n",
+       "    data_logger.timing_system.type:         GPS\n",
+       "    data_logger.timing_system.uncertainty:  0.0\n",
+       "    data_logger.type:                       None\n",
+       "    data_type:                              BBMT\n",
+       "    id:                                     None\n",
+       "    sample_rate:                            1.0\n",
+       "    time_period.end:                        2020-10-07T14:19:46+00:00\n",
+       "    time_period.start:                      2020-09-30T21:14:00+00:00
" + ], + "text/plain": [ + "\n", + "Dimensions: (time: 579947)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 2020-09-30T21:14:00 ... 2020-10-07T1...\n", + "Data variables:\n", + " bx (time) float64 2.378e+04 2.378e+04 ... 2.378e+04 2.378e+04\n", + " by (time) float64 213.9 213.9 213.9 213.9 ... 247.2 247.3 247.3\n", + " bz (time) float64 4.183e+04 4.183e+04 ... 4.184e+04 4.184e+04\n", + " e1 (time) float64 134.8 134.9 135.0 135.1 ... 130.4 130.3 130.2\n", + " e2 (time) float64 -109.8 -109.9 -109.9 ... -40.88 -40.95 -40.93\n", + " temperature_e (time) float64 46.5 46.46 46.47 46.47 ... 18.18 18.17 18.17\n", + " temperature_h (time) float64 36.3 36.3 36.3 36.3 ... 19.61 19.6 19.6 19.6\n", + "Attributes:\n", + " channels_recorded_auxiliary: ['temperature_e', 'temperature_h']\n", + " channels_recorded_electric: ['e1', 'e2']\n", + " channels_recorded_magnetic: ['bx', 'by', 'bz']\n", + " data_logger.firmware.author: None\n", + " data_logger.firmware.name: None\n", + " data_logger.firmware.version: None\n", + " data_logger.id: None\n", + " data_logger.manufacturer: None\n", + " data_logger.timing_system.drift: 0.0\n", + " data_logger.timing_system.type: GPS\n", + " data_logger.timing_system.uncertainty: 0.0\n", + " data_logger.type: None\n", + " data_type: BBMT\n", + " id: None\n", + " sample_rate: 1.0\n", + " time_period.end: 2020-10-07T14:19:46+00:00\n", + " time_period.start: 2020-09-30T21:14:00+00:00" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lemis[\"005\"].dataset\n" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "c93ebc85-6723-4e19-94b8-85f7dcd709c3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{\n", + " \"run\": {\n", + " \"channels_recorded_auxiliary\": [\n", + " \"temperature_e\",\n", + " \"temperature_h\"\n", + " ],\n", + " \"channels_recorded_electric\": [\n", + " \"e1\",\n", + " \"e2\"\n", + " ],\n", + " \"channels_recorded_magnetic\": [\n", + " \"bx\",\n", + " \"by\",\n", + " \"bz\"\n", + " ],\n", + " \"data_logger.firmware.author\": null,\n", + " \"data_logger.firmware.name\": null,\n", + " \"data_logger.firmware.version\": null,\n", + " \"data_logger.id\": null,\n", + " \"data_logger.manufacturer\": null,\n", + " \"data_logger.timing_system.drift\": 0.0,\n", + " \"data_logger.timing_system.type\": \"GPS\",\n", + " \"data_logger.timing_system.uncertainty\": 0.0,\n", + " \"data_logger.type\": null,\n", + " \"data_type\": \"BBMT\",\n", + " \"id\": \"005\",\n", + " \"sample_rate\": 1.0,\n", + " \"time_period.end\": \"2020-10-07T14:19:46+00:00\",\n", + " \"time_period.start\": \"2020-09-30T21:14:00+00:00\"\n", + " }\n", + "}" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lemis[\"005\"].run_metadata" + ] + }, + { + "cell_type": "markdown", + "id": "04f98f09-d535-4d2e-9b37-8b4d22ad14e0", + "metadata": {}, + "source": [ + "\n", + "### We have run time series, now let's pack it into an mth5" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "35dec618-dec2-4a72-a351-3c2b969511e5", + "metadata": {}, + "outputs": [], + "source": [ + "h5_fn = \"magdelena.h5\"\n", + "station_id = \"0110\"" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "840b5aa0-db95-4ebd-9c85-31113236ef5f", + "metadata": {}, + "outputs": [], + "source": [ + "# write some simple metadata for the survey\n", + "survey = metadata.Survey()\n", + "survey.acquired_by.author = \"MT Meister\"\n", + "survey.archive_id = \"LEMI_TEST_01\"\n", + "survey.archive_network = \"MT\"\n", + "survey.name = \"magdelena\"" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "2b4b714d-32f9-4e7b-b0c5-14079c99d31d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-07-24 16:20:51,784 [line 591] mth5.mth5.MTH5.open_mth5 - WARNING: magdelena.h5 will be overwritten in 'w' mode\n", + "2022-07-24 16:20:52,220 [line 656] mth5.mth5.MTH5._initialize_file - INFO: Initialized MTH5 0.1.0 file magdelena.h5 in mode w\n" + ] + } + ], + "source": [ + "m = mth5.MTH5(file_version=\"0.1.0\")\n", + "m.open_mth5(h5_fn, \"w\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "54c72d9e-e17e-45ff-9409-485788d8680a", + "metadata": {}, + "outputs": [], + "source": [ + "# add survey metadata\n", + "survey_group = m.survey_group\n", + "survey_group.metadata.update(survey)\n", + "survey_group.write_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "e8b96ee4-5139-448b-b670-9869499dee1d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Station was just initialized\n" + ] + } + ], + "source": [ + "# initialize a station\n", + "station_group = m.add_station(station_id)\n", + "print(\"Station was just initialized\")\n", + "station_group.validate_station_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "aaae5911-5d73-42cd-b489-7eee29beb6fd", + "metadata": {}, + "outputs": [], + "source": [ + "for run, run_ts in lemis.items():\n", + " run_group = station_group.add_run(run_ts.run_metadata.id, run_metadata=run_ts.run_metadata)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "b311fe48-59b2-4ff7-91e1-74cb4b0baeea", + "metadata": {}, + "outputs": [], + "source": [ + "station_group.validate_station_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "ddc2c102-c4fc-4bb5-8c9d-691625529b58", + "metadata": {}, + "outputs": [], + "source": [ + "survey_group.update_survey_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "5bd20e6a-3b17-4002-a6a5-5bf37e1473aa", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-07-24 16:20:56,500 [line 731] mth5.mth5.MTH5.close_mth5 - INFO: Flushing and closing magdelena.h5\n" + ] + } + ], + "source": [ + "m.close_mth5()" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "11f733da-43a4-48ed-a034-d065b875c56c", + "metadata": {}, + "outputs": [], + "source": [ + "mm = mth5.MTH5(file_version=\"0.1.0\")\n", + "mm.open_mth5(h5_fn, \"a\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "0818f400-7f65-4111-a1c9-593b68435a20", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "/:\n", + "====================\n", + " |- Group: Survey\n", + " ----------------\n", + " |- Group: Filters\n", + " -----------------\n", + " |- Group: coefficient\n", + " ---------------------\n", + " |- Group: fap\n", + " -------------\n", + " |- Group: fir\n", + " -------------\n", + " |- Group: time_delay\n", + " --------------------\n", + " |- Group: zpk\n", + " -------------\n", + " |- Group: Reports\n", + " -----------------\n", + " |- Group: Standards\n", + " -------------------\n", + " --> Dataset: summary\n", + " ......................\n", + " |- Group: Stations\n", + " ------------------\n", + " |- Group: 0110\n", + " --------------\n", + " |- Group: 001\n", + " -------------\n", + " |- Group: 002\n", + " -------------\n", + " |- Group: 003\n", + " -------------\n", + " |- Group: 004\n", + " -------------\n", + " |- Group: 005\n", + " -------------\n", + " |- Group: Transfer_Functions\n", + " ----------------------------\n", + " --> Dataset: channel_summary\n", + " ..............................\n", + " --> Dataset: tf_summary\n", + " ........................." + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mm" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "84ea94e5-0523-43fd-99ef-c4c6541ed401", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-07-24 16:20:58,636 [line 731] mth5.mth5.MTH5.close_mth5 - INFO: Flushing and closing magdelena.h5\n" + ] + } + ], + "source": [ + "mm.close_mth5()" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "7be3eb33-4448-4eb2-85e4-2a2e1ee834ef", + "metadata": {}, + "outputs": [], + "source": [ + "# def count_lines(file_name):\n", + "# \"\"\"\n", + "\n", + "# Parameters\n", + "# ----------\n", + "# fileName\n", + "\n", + "# Returns\n", + "# -------\n", + "# integer: Number of lines present in fileName or -1 if file does not exist\n", + "# acts like wc -l in unix, raise IOError: if file_name does not exist.\n", + "# \"\"\"\n", + "# i = -1\n", + "# with open(fileName) as f:\n", + "# for i, l in enumerate(f):\n", + "# pass\n", + "# return i+1" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py37", + "language": "python", + "name": "py37" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/mth5/io/lemi424_new.py b/mth5/io/lemi424_new.py index 3c9ad665..6eeb39ce 100644 --- a/mth5/io/lemi424_new.py +++ b/mth5/io/lemi424_new.py @@ -25,7 +25,7 @@ class LEMI424: """ - def __init__(self, fn=None): + def __init__(self, fn=[]): self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}") self.fn = fn self._has_data = False @@ -61,16 +61,44 @@ def __init__(self, fn=None): if self.fn: self.read() + @property + def num_source_files(self): + return len(self.fn) + @property def fn(self): return self._fn + @property + def validate_fn(self): + """ + Need to check that the filenames are sequential + """ + return True + @fn.setter - def fn(self, value): - if value is not None: - value = Path(value) - if not value.exists(): - raise IOError(f"Could not find {value}") + def fn(self, value, sort=True): + """ + + Parameters + ---------- + value:string or pathlib.Path, or list of these + + """ + if isinstance(value, list): + value = [Path(x) for x in value] + exists = [x.exists() for x in value] + for i_fn, cond in enumerate(exists): + if not cond: + raise IOError(f"Could not find {value[i_fn]}") + elif value is not None: + value = [ + Path(value), + ] + if not value[0].exists(): + raise IOError(f"Could not find {value[0]}") + if sort: + value.sort() self._fn = value @property @@ -160,7 +188,7 @@ def run_metadata(self): r.time_period.start = self.start r.time_period.end = self.end - def read(self, fn=None): + def read(self, fn=[]): """ Read a LEMI424 file using pandas @@ -170,16 +198,24 @@ def read(self, fn=None): :rtype: TYPE """ - if fn is not None: + if fn: self.fn = fn - if not self.fn.exists(): + exists = [x.exists() for x in self.fn] + if all(x for x in exists): + pass + else: msg = "Could not find file %s" - self.logger.error(msg, self.fn) - raise IOError(msg % self.fn) + for i_fn, cond in enumerate(exists): + if not cond: + self.logger.error(msg, self.fn[i_fn]) + raise IOError(msg % self.fn[i_fn]) - self._df = pd.read_csv(self.fn, delimiter="\s+", names=self.column_names) + dfs = self.num_source_files * [None] + for i, fn in enumerate(self.fn): + dfs[i] = pd.read_csv(fn, delimiter="\s+", names=self.column_names) + self._df = pd.concat(dfs) self._has_data = True def to_run_ts(self, fn=None, e_channels=["e1", "e2"]): From 1b5b78ae58cc61a662e5d2324f590aa536e3ac04 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 30 Jul 2022 09:23:40 -0700 Subject: [PATCH 3/7] change closed to inclusive to supress FutureWarning --- mth5/timeseries/channel_ts.py | 53 ++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/mth5/timeseries/channel_ts.py b/mth5/timeseries/channel_ts.py index 419ed0fc..ec0a5de4 100644 --- a/mth5/timeseries/channel_ts.py +++ b/mth5/timeseries/channel_ts.py @@ -2,16 +2,16 @@ """ .. module:: timeseries :synopsis: Deal with MT time series - -.. todo:: Check the conversion to netcdf. There are some weird serializations of + +.. todo:: Check the conversion to netcdf. There are some weird serializations of lists and arrays that goes on, seems easiest to convert all lists to strings and then convert them back if read in. :copyright: Jared Peacock (jpeacock@usgs.gov) - -:license: + +:license: MIT """ @@ -81,7 +81,7 @@ def make_dt_coordinates(start_time, sample_rate, n_samples, logger): start=start_time.iso_str.split("+", 1)[0], periods=n_samples, freq=dt_freq, - closed=None, + inclusive=None, ) return dt_index @@ -186,7 +186,7 @@ def __init__( ) ) elif isinstance(channel_metadata, dict): - if not channel_type in [cc.lower() for cc in channel_metadata.keys()]: + if channel_type not in [cc.lower() for cc in channel_metadata.keys()]: channel_metadata = {channel_type: channel_metadata} self.channel_metadata.from_dict(channel_metadata) self.logger.debug("Loading from metadata dict") @@ -204,7 +204,7 @@ def __init__( if isinstance(station_metadata, metadata.Station): self.station_metadata.update(station_metadata) elif isinstance(station_metadata, dict): - if not "station" in [cc.lower() for cc in station_metadata.keys()]: + if "station" not in [cc.lower() for cc in station_metadata.keys()]: station_metadata = {"Station": station_metadata} self.station_metadata.from_dict(station_metadata) self.logger.debug("Loading from metadata dict") @@ -220,7 +220,7 @@ def __init__( if isinstance(run_metadata, metadata.Run): self.run_metadata.update(run_metadata) elif isinstance(run_metadata, dict): - if not "run" in [cc.lower() for cc in run_metadata.keys()]: + if "run" not in [cc.lower() for cc in run_metadata.keys()]: run_metadata = {"Run": run_metadata} self.run_metadata.from_dict(run_metadata) self.logger.debug("Loading from metadata dict") @@ -312,7 +312,10 @@ def ts(self, ts_arr): dt = ts_arr.index else: dt = make_dt_coordinates( - self.start, self.sample_rate, ts_arr["data"].size, self.logger, + self.start, + self.sample_rate, + ts_arr["data"].size, + self.logger, ) try: self._ts = xr.DataArray( @@ -331,7 +334,10 @@ def ts(self, ts_arr): dt = ts_arr.index else: dt = make_dt_coordinates( - self.start, self.sample_rate, ts_arr["data"].size, self.logger, + self.start, + self.sample_rate, + ts_arr["data"].size, + self.logger, ) self._ts = xr.DataArray(ts_arr.values, coords=[("time", dt)], name="ts") self._update_xarray_metadata() @@ -488,7 +494,8 @@ def has_data(self): """ if len(self._ts) > 1: if isinstance( - self._ts.indexes["time"][0], pd._libs.tslibs.timestamps.Timestamp, + self._ts.indexes["time"][0], + pd._libs.tslibs.timestamps.Timestamp, ): return True return False @@ -663,9 +670,9 @@ def channel_response_filter(self, value): def remove_instrument_response(self, **kwargs): """ Remove instrument response from the given channel response filter - + The order of operations is important (if applied): - + 1) detrend 2) zero mean 3) zero pad @@ -674,25 +681,25 @@ def remove_instrument_response(self, **kwargs): 6) remove response 7) undo time window 8) bandpass - + **kwargs** - + :param plot: to plot the calibration process [ False | True ] - :type plot: boolean, default True + :type plot: boolean, default True :param detrend: Remove linar trend of the time series - :type detrend: boolean, default True + :type detrend: boolean, default True :param zero_mean: Remove the mean of the time series - :type zero_mean: boolean, default True + :type zero_mean: boolean, default True :param zero_pad: pad the time series to the next power of 2 for efficiency - :type zero_pad: boolean, default True + :type zero_pad: boolean, default True :param t_window: Time domain windown name see `scipy.signal.windows` for options - :type t_window: string, default None - :param t_window_params: Time domain window parameters, parameters can be - found in `scipy.signal.windows` + :type t_window: string, default None + :param t_window_params: Time domain window parameters, parameters can be + found in `scipy.signal.windows` :type t_window_params: dictionary :param f_window: Frequency domain windown name see `scipy.signal.windows` for options :type f_window: string, defualt None - :param f_window_params: Frequency window parameters, parameters can be + :param f_window_params: Frequency window parameters, parameters can be found in `scipy.signal.windows` :type f_window_params: dictionary :param bandpass: bandpass freequency and order {"low":, "high":, "order":,} From 0f75662f9f768160e258b9633a7ecf851cf87358 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 30 Jul 2022 09:34:32 -0700 Subject: [PATCH 4/7] whoops - changing closed to inclusive to supress FutureWarning broke something, changing it back --- mth5/timeseries/channel_ts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mth5/timeseries/channel_ts.py b/mth5/timeseries/channel_ts.py index ec0a5de4..d8c04115 100644 --- a/mth5/timeseries/channel_ts.py +++ b/mth5/timeseries/channel_ts.py @@ -81,7 +81,7 @@ def make_dt_coordinates(start_time, sample_rate, n_samples, logger): start=start_time.iso_str.split("+", 1)[0], periods=n_samples, freq=dt_freq, - inclusive=None, + closed=None, ) return dt_index From ed660ff3d4049f1d2d8758f4740d2fa7cec42b56 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 30 Jul 2022 09:36:34 -0700 Subject: [PATCH 5/7] re-test magdelena writer --- .../notebooks/lemi_reader_magdelena.ipynb | 359 +++++++++++++----- 1 file changed, 258 insertions(+), 101 deletions(-) diff --git a/docs/examples/notebooks/lemi_reader_magdelena.ipynb b/docs/examples/notebooks/lemi_reader_magdelena.ipynb index ea04bf4d..2b75e690 100644 --- a/docs/examples/notebooks/lemi_reader_magdelena.ipynb +++ b/docs/examples/notebooks/lemi_reader_magdelena.ipynb @@ -68,7 +68,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2022-07-24 16:20:14,222 [line 135] mth5.setup_logger - INFO: Logging file can be found /home/kkappler/software/irismt/mth5/logs/mth5_debug.log\n" + "2022-07-30 09:33:40,298 [line 135] mth5.setup_logger - INFO: Logging file can be found /home/kkappler/software/irismt/mth5/logs/mth5_debug.log\n" ] } ], @@ -209,7 +209,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "ln: " + "ln: failed to create symbolic link '/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/DATA0110': File exists\n" ] }, { @@ -221,13 +221,6 @@ "execution_count": 5, "metadata": {}, "output_type": "execute_result" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "failed to create symbolic link '/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/DATA0110': File exists\n" - ] } ], "source": [ @@ -389,6 +382,196 @@ { "cell_type": "code", "execution_count": 11, + "id": "8c9d85f4-d249-4ae5-8d3c-46b8d349e88a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
yearmonthdayhourminutesecondbxbybze1e2e3e4latitudelongitude
020209302021023813.621729.81641802.042131.013-111.026164.1669.7153404.8391110712.84475
120209302021123813.586729.84241802.030130.917-111.204164.0619.5403404.8391110712.84473
220209302021223813.553729.87541802.058130.918-111.227164.0719.5213404.8391010712.84470
320209302021323813.477729.87841802.042130.918-111.396164.0609.3573404.8391010712.84468
420209302021423813.449729.90841802.034131.018-111.326164.1709.4283404.8390910712.84467
\n", + "
" + ], + "text/plain": [ + " year month day hour minute second bx by bz \\\n", + "0 2020 9 30 20 21 0 23813.621 729.816 41802.042 \n", + "1 2020 9 30 20 21 1 23813.586 729.842 41802.030 \n", + "2 2020 9 30 20 21 2 23813.553 729.875 41802.058 \n", + "3 2020 9 30 20 21 3 23813.477 729.878 41802.042 \n", + "4 2020 9 30 20 21 4 23813.449 729.908 41802.034 \n", + "\n", + " e1 e2 e3 e4 latitude longitude \n", + "0 131.013 -111.026 164.166 9.715 3404.83911 10712.84475 \n", + "1 130.917 -111.204 164.061 9.540 3404.83911 10712.84473 \n", + "2 130.918 -111.227 164.071 9.521 3404.83910 10712.84470 \n", + "3 130.918 -111.396 164.060 9.357 3404.83910 10712.84468 \n", + "4 131.018 -111.326 164.170 9.428 3404.83909 10712.84467 " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "columns = [\"year\", \"month\", \"day\", \"hour\", \"minute\", \"second\", \"bx\", \"by\", \"bz\", 'e1', 'e2', 'e3', 'e4',\"latitude\", \"longitude\"]\n", + "l424_list[0]._df[columns][0:5]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "2775f0cf-b3ae-42e5-8cfa-b38952c6125b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Index(['year', 'month', 'day', 'hour', 'minute', 'second', 'bx', 'by', 'bz',\n", + " 'temperature_e', 'temperature_h', 'e1', 'e2', 'e3', 'e4', 'battery',\n", + " 'elevation', 'latitude', 'lat_hemisphere', 'longitude',\n", + " 'lon_hemisphere', 'n_satellites', 'gps_fix', 'tdiff'],\n", + " dtype='object')" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "l424_list[0]._df.columns" + ] + }, + { + "cell_type": "code", + "execution_count": 13, "id": "4bbb966a-d056-4d11-869d-efdf76f3e1cc", "metadata": {}, "outputs": [ @@ -587,7 +770,7 @@ "[5 rows x 24 columns]" ] }, - "execution_count": 11, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -598,7 +781,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 14, "id": "4aa1d729-9c13-4108-8d86-242c4bc8bb8f", "metadata": {}, "outputs": [ @@ -797,7 +980,7 @@ "[5 rows x 24 columns]" ] }, - "execution_count": 12, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -808,7 +991,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 15, "id": "3ee6c95f-cd75-45f7-88e7-25e6a4420261", "metadata": {}, "outputs": [], @@ -825,7 +1008,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 16, "id": "cad8c364-b4ef-48b1-96e1-746138260416", "metadata": {}, "outputs": [ @@ -835,7 +1018,7 @@ "12" ] }, - "execution_count": 14, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -848,7 +1031,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 17, "id": "c8db9eff-df74-4eef-879b-f44b46c0c440", "metadata": {}, "outputs": [], @@ -860,7 +1043,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 18, "id": "2eaefcde-8bcb-43df-af3b-7e7661c18e7a", "metadata": {}, "outputs": [], @@ -879,7 +1062,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "id": "92bed2d6-2e41-4527-be73-281a3cf68c29", "metadata": {}, "outputs": [ @@ -1068,7 +1251,7 @@ "11 True 202010070000.TXT " ] }, - "execution_count": 18, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -1080,7 +1263,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "id": "447a8eb9-608f-46cf-9d88-c2f421b7e614", "metadata": {}, "outputs": [ @@ -1282,7 +1465,7 @@ "11 False 202010070000.TXT " ] }, - "execution_count": 19, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -1321,7 +1504,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "id": "2783b71a-06ad-4855-b92f-d9db81940a7b", "metadata": {}, "outputs": [ @@ -1331,7 +1514,7 @@ "'001'" ] }, - "execution_count": 20, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -1344,7 +1527,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "id": "d5cf56ca-1b46-421f-99cb-adf377b661f5", "metadata": {}, "outputs": [], @@ -1355,7 +1538,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 23, "id": "f5e4b88f-1055-47ae-9b7d-ff4d97ffdbbf", "metadata": {}, "outputs": [ @@ -1381,7 +1564,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "id": "b37ac4bd-87e3-44a8-aadc-c32c21fee65d", "metadata": {}, "outputs": [ @@ -1570,7 +1753,7 @@ "11 False 202010070000.TXT " ] }, - "execution_count": 23, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -1582,7 +1765,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 25, "id": "7d1e5e3a-fa85-407c-8c5f-e70bb0ed733c", "metadata": {}, "outputs": [], @@ -1592,7 +1775,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 26, "id": "53460041-dcfc-4da4-9d4e-0c8308d9fcf4", "metadata": {}, "outputs": [ @@ -1621,7 +1804,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 27, "id": "42adef43-c662-4495-9671-16e9304e9590", "metadata": {}, "outputs": [ @@ -1642,7 +1825,7 @@ " PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010070000.TXT')]}" ] }, - "execution_count": 26, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -1653,7 +1836,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 28, "id": "c9d341b9-e3bc-4498-9175-300d40010046", "metadata": {}, "outputs": [], @@ -1663,7 +1846,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 29, "id": "4a7f9d48-6548-4f3c-81b1-f0cb97724965", "metadata": {}, "outputs": [], @@ -1677,7 +1860,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 30, "id": "aa6e0913-2fce-431b-bc85-3087aadfab37", "metadata": {}, "outputs": [ @@ -1693,7 +1876,7 @@ "\tComponents: ['bx', 'by', 'bz', 'e1', 'e2', 'temperature_e', 'temperature_h']" ] }, - "execution_count": 29, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } @@ -1704,7 +1887,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 31, "id": "f1b22561-7c05-436a-abad-a7b3c6cf39f9", "metadata": {}, "outputs": [ @@ -1744,7 +1927,7 @@ "}" ] }, - "execution_count": 30, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -1755,7 +1938,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 32, "id": "31afe739-e93e-4718-b0ab-881ab2de0e59", "metadata": {}, "outputs": [ @@ -1765,7 +1948,7 @@ "mth5.timeseries.run_ts.RunTS" ] }, - "execution_count": 31, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -1776,7 +1959,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 33, "id": "4a6689ff-e1aa-437d-9a33-2bbd800bd0ef", "metadata": {}, "outputs": [ @@ -2163,9 +2346,9 @@ " id: None\n", " sample_rate: 1.0\n", " time_period.end: 2020-09-30T20:28:15+00:00\n", - " time_period.start: 2020-09-30T20:21:00+00:00
  • channels_recorded_auxiliary :
    ['temperature_e', 'temperature_h']
    channels_recorded_electric :
    ['e1', 'e2']
    channels_recorded_magnetic :
    ['bx', 'by', 'bz']
    data_logger.firmware.author :
    None
    data_logger.firmware.name :
    None
    data_logger.firmware.version :
    None
    data_logger.id :
    None
    data_logger.manufacturer :
    None
    data_logger.timing_system.drift :
    0.0
    data_logger.timing_system.type :
    GPS
    data_logger.timing_system.uncertainty :
    0.0
    data_logger.type :
    None
    data_type :
    BBMT
    id :
    None
    sample_rate :
    1.0
    time_period.end :
    2020-09-30T20:28:15+00:00
    time_period.start :
    2020-09-30T20:21:00+00:00
    • " ], "text/plain": [ "\n", @@ -2480,7 +2663,7 @@ " time_period.start: 2020-09-30T20:21:00+00:00" ] }, - "execution_count": 32, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -2491,7 +2674,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 34, "id": "978b6a3f-e317-44b2-86df-1713674e1efe", "metadata": {}, "outputs": [ @@ -2878,11 +3061,11 @@ " id: None\n", " sample_rate: 1.0\n", " time_period.end: 2020-10-07T14:19:46+00:00\n", - " time_period.start: 2020-09-30T21:14:00+00:00
    • bx
      (time)
      float64
      2.378e+04 2.378e+04 ... 2.378e+04
      channel_number :
      0
      component :
      bx
      data_quality.rating.value :
      0
      filter.applied :
      [False]
      filter.name :
      []
      location.elevation :
      0.0
      location.latitude :
      0.0
      location.longitude :
      0.0
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      sample_rate :
      1.0
      sensor.id :
      None
      sensor.manufacturer :
      None
      sensor.type :
      None
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      magnetic
      units :
      None
      station.id :
      None
      run.id :
      None
      array([ 23778.555,  23778.572,  23778.602, ...,  23776.707,  23776.676,\n",
+       "        23776.633])
    • by
      (time)
      float64
      213.9 213.9 213.9 ... 247.3 247.3
      channel_number :
      0
      component :
      by
      data_quality.rating.value :
      0
      filter.applied :
      [False]
      filter.name :
      []
      location.elevation :
      0.0
      location.latitude :
      0.0
      location.longitude :
      0.0
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      sample_rate :
      1.0
      sensor.id :
      None
      sensor.manufacturer :
      None
      sensor.type :
      None
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      magnetic
      units :
      None
      station.id :
      None
      run.id :
      None
      array([ 213.914,  213.922,  213.92 , ...,  247.202,  247.268,  247.342])
    • bz
      (time)
      float64
      4.183e+04 4.183e+04 ... 4.184e+04
      channel_number :
      0
      component :
      bz
      data_quality.rating.value :
      0
      filter.applied :
      [False]
      filter.name :
      []
      location.elevation :
      0.0
      location.latitude :
      0.0
      location.longitude :
      0.0
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      sample_rate :
      1.0
      sensor.id :
      None
      sensor.manufacturer :
      None
      sensor.type :
      None
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      magnetic
      units :
      None
      station.id :
      None
      run.id :
      None
      array([ 41834.609,  41834.605,  41834.573, ...,  41843.23 ,  41843.23 ,\n",
+       "        41843.214])
    • e1
      (time)
      float64
      134.8 134.9 135.0 ... 130.3 130.2
      channel_number :
      0
      component :
      e1
      data_quality.rating.value :
      0
      dipole_length :
      None
      filter.applied :
      [False]
      filter.name :
      []
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      negative.elevation :
      0.0
      negative.id :
      None
      negative.latitude :
      0.0
      negative.longitude :
      0.0
      negative.manufacturer :
      None
      negative.type :
      None
      positive.elevation :
      0.0
      positive.id :
      None
      positive.latitude :
      0.0
      positive.longitude :
      0.0
      positive.manufacturer :
      None
      positive.type :
      None
      sample_rate :
      1.0
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      electric
      units :
      None
      station.id :
      None
      run.id :
      None
      array([ 134.816,  134.873,  134.98 , ...,  130.399,  130.299,  130.25 ])
    • e2
      (time)
      float64
      -109.8 -109.9 ... -40.95 -40.93
      channel_number :
      0
      component :
      e2
      data_quality.rating.value :
      0
      dipole_length :
      None
      filter.applied :
      [False]
      filter.name :
      []
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      negative.elevation :
      0.0
      negative.id :
      None
      negative.latitude :
      0.0
      negative.longitude :
      0.0
      negative.manufacturer :
      None
      negative.type :
      None
      positive.elevation :
      0.0
      positive.id :
      None
      positive.latitude :
      0.0
      positive.longitude :
      0.0
      positive.manufacturer :
      None
      positive.type :
      None
      sample_rate :
      1.0
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      electric
      units :
      None
      station.id :
      None
      run.id :
      None
      array([-109.831, -109.902, -109.877, ...,  -40.881,  -40.953,  -40.933])
    • temperature_e
      (time)
      float64
      46.5 46.46 46.47 ... 18.17 18.17
      channel_number :
      0
      component :
      temperature_e
      data_quality.rating.value :
      0
      filter.applied :
      [False]
      filter.name :
      []
      location.elevation :
      0.0
      location.latitude :
      0.0
      location.longitude :
      0.0
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      sample_rate :
      1.0
      sensor.id :
      None
      sensor.manufacturer :
      None
      sensor.type :
      None
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      auxiliary
      units :
      None
      station.id :
      None
      run.id :
      None
      array([ 46.5 ,  46.46,  46.47, ...,  18.18,  18.17,  18.17])
    • temperature_h
      (time)
      float64
      36.3 36.3 36.3 ... 19.6 19.6 19.6
      channel_number :
      0
      component :
      temperature_h
      data_quality.rating.value :
      0
      filter.applied :
      [False]
      filter.name :
      []
      location.elevation :
      0.0
      location.latitude :
      0.0
      location.longitude :
      0.0
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      sample_rate :
      1.0
      sensor.id :
      None
      sensor.manufacturer :
      None
      sensor.type :
      None
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      auxiliary
      units :
      None
      station.id :
      None
      run.id :
      None
      array([ 36.3,  36.3,  36.3, ...,  19.6,  19.6,  19.6])
  • channels_recorded_auxiliary :
    ['temperature_e', 'temperature_h']
    channels_recorded_electric :
    ['e1', 'e2']
    channels_recorded_magnetic :
    ['bx', 'by', 'bz']
    data_logger.firmware.author :
    None
    data_logger.firmware.name :
    None
    data_logger.firmware.version :
    None
    data_logger.id :
    None
    data_logger.manufacturer :
    None
    data_logger.timing_system.drift :
    0.0
    data_logger.timing_system.type :
    GPS
    data_logger.timing_system.uncertainty :
    0.0
    data_logger.type :
    None
    data_type :
    BBMT
    id :
    None
    sample_rate :
    1.0
    time_period.end :
    2020-10-07T14:19:46+00:00
    time_period.start :
    2020-09-30T21:14:00+00:00
    • " ], "text/plain": [ "\n", @@ -2917,7 +3100,7 @@ " time_period.start: 2020-09-30T21:14:00+00:00" ] }, - "execution_count": 33, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -2928,7 +3111,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 35, "id": "c93ebc85-6723-4e19-94b8-85f7dcd709c3", "metadata": {}, "outputs": [ @@ -2968,7 +3151,7 @@ "}" ] }, - "execution_count": 34, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -2988,7 +3171,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 36, "id": "35dec618-dec2-4a72-a351-3c2b969511e5", "metadata": {}, "outputs": [], @@ -2999,7 +3182,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 37, "id": "840b5aa0-db95-4ebd-9c85-31113236ef5f", "metadata": {}, "outputs": [], @@ -3014,7 +3197,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 38, "id": "2b4b714d-32f9-4e7b-b0c5-14079c99d31d", "metadata": {}, "outputs": [ @@ -3022,8 +3205,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "2022-07-24 16:20:51,784 [line 591] mth5.mth5.MTH5.open_mth5 - WARNING: magdelena.h5 will be overwritten in 'w' mode\n", - "2022-07-24 16:20:52,220 [line 656] mth5.mth5.MTH5._initialize_file - INFO: Initialized MTH5 0.1.0 file magdelena.h5 in mode w\n" + "2022-07-30 09:35:14,229 [line 591] mth5.mth5.MTH5.open_mth5 - WARNING: magdelena.h5 will be overwritten in 'w' mode\n", + "2022-07-30 09:35:14,652 [line 656] mth5.mth5.MTH5._initialize_file - INFO: Initialized MTH5 0.1.0 file magdelena.h5 in mode w\n" ] } ], @@ -3034,7 +3217,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 39, "id": "54c72d9e-e17e-45ff-9409-485788d8680a", "metadata": {}, "outputs": [], @@ -3047,7 +3230,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 40, "id": "e8b96ee4-5139-448b-b670-9869499dee1d", "metadata": {}, "outputs": [ @@ -3068,7 +3251,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 41, "id": "aaae5911-5d73-42cd-b489-7eee29beb6fd", "metadata": {}, "outputs": [], @@ -3079,7 +3262,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 42, "id": "b311fe48-59b2-4ff7-91e1-74cb4b0baeea", "metadata": {}, "outputs": [], @@ -3089,7 +3272,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 43, "id": "ddc2c102-c4fc-4bb5-8c9d-691625529b58", "metadata": {}, "outputs": [], @@ -3099,7 +3282,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 44, "id": "5bd20e6a-3b17-4002-a6a5-5bf37e1473aa", "metadata": {}, "outputs": [ @@ -3107,7 +3290,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2022-07-24 16:20:56,500 [line 731] mth5.mth5.MTH5.close_mth5 - INFO: Flushing and closing magdelena.h5\n" + "2022-07-30 09:35:21,938 [line 731] mth5.mth5.MTH5.close_mth5 - INFO: Flushing and closing magdelena.h5\n" ] } ], @@ -3117,7 +3300,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 45, "id": "11f733da-43a4-48ed-a034-d065b875c56c", "metadata": {}, "outputs": [], @@ -3128,7 +3311,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 46, "id": "0818f400-7f65-4111-a1c9-593b68435a20", "metadata": {}, "outputs": [ @@ -3179,7 +3362,7 @@ " ........................." ] }, - "execution_count": 45, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } @@ -3190,7 +3373,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 47, "id": "84ea94e5-0523-43fd-99ef-c4c6541ed401", "metadata": {}, "outputs": [ @@ -3198,39 +3381,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2022-07-24 16:20:58,636 [line 731] mth5.mth5.MTH5.close_mth5 - INFO: Flushing and closing magdelena.h5\n" + "2022-07-30 09:35:33,832 [line 731] mth5.mth5.MTH5.close_mth5 - INFO: Flushing and closing magdelena.h5\n" ] } ], "source": [ "mm.close_mth5()" ] - }, - { - "cell_type": "code", - "execution_count": 47, - "id": "7be3eb33-4448-4eb2-85e4-2a2e1ee834ef", - "metadata": {}, - "outputs": [], - "source": [ - "# def count_lines(file_name):\n", - "# \"\"\"\n", - "\n", - "# Parameters\n", - "# ----------\n", - "# fileName\n", - "\n", - "# Returns\n", - "# -------\n", - "# integer: Number of lines present in fileName or -1 if file does not exist\n", - "# acts like wc -l in unix, raise IOError: if file_name does not exist.\n", - "# \"\"\"\n", - "# i = -1\n", - "# with open(fileName) as f:\n", - "# for i, l in enumerate(f):\n", - "# pass\n", - "# return i+1" - ] } ], "metadata": { From daedf4d81c1e81f37478aaca6567e450398cd7d9 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 30 Jul 2022 15:30:39 -0700 Subject: [PATCH 6/7] add support for survey kwarg in read_back_data --- mth5/utils/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mth5/utils/helpers.py b/mth5/utils/helpers.py index 725692fc..79de5cfd 100644 --- a/mth5/utils/helpers.py +++ b/mth5/utils/helpers.py @@ -42,7 +42,7 @@ def initialize_mth5(h5_path, mode="a", file_version="0.1.0"): return mth5_obj -def read_back_data(mth5_path, station_id, run_id): +def read_back_data(mth5_path, station_id, run_id, survey=None): """ Parameters @@ -63,7 +63,7 @@ def read_back_data(mth5_path, station_id, run_id): processing_config["local_station_id"] = station_id config = processing_config m = initialize_mth5(config["mth5_path"], mode="r") - local_run_obj = m.get_run(config["local_station_id"], run_id) + local_run_obj = m.get_run(config["local_station_id"], run_id, survey=survey) local_run_ts = local_run_obj.to_runts() data_array = local_run_ts.dataset.to_array() print(f"data shape = {data_array.shape}") From e156bad8f31513c644d418819fb54a323343c38a Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Fri, 5 Aug 2022 17:02:20 -0700 Subject: [PATCH 7/7] overwrite lemi424.py with lemi424_new.py --- .../notebooks/lemi_reader_magdelena.ipynb | 229 +++++++++------ mth5/io/lemi424.py | 86 ++++-- mth5/io/lemi424_new.py | 276 ------------------ 3 files changed, 206 insertions(+), 385 deletions(-) delete mode 100644 mth5/io/lemi424_new.py diff --git a/docs/examples/notebooks/lemi_reader_magdelena.ipynb b/docs/examples/notebooks/lemi_reader_magdelena.ipynb index 2b75e690..8f3da6b6 100644 --- a/docs/examples/notebooks/lemi_reader_magdelena.ipynb +++ b/docs/examples/notebooks/lemi_reader_magdelena.ipynb @@ -68,7 +68,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2022-07-30 09:33:40,298 [line 135] mth5.setup_logger - INFO: Logging file can be found /home/kkappler/software/irismt/mth5/logs/mth5_debug.log\n" + "2022-08-05 16:56:20,375 [line 135] mth5.setup_logger - INFO: Logging file can be found /home/kkappler/software/irismt/mth5/logs/mth5_debug.log\n" ] } ], @@ -80,7 +80,7 @@ "\n", "from mth5 import read_file\n", "from mth5 import mth5\n", - "from mth5.io.lemi424_new import LEMI424\n", + "from mth5.io.lemi424 import LEMI424\n", "\n", "from mt_metadata import timeseries as metadata\n", "from mt_metadata.utils.mttime import MTime\n" @@ -110,8 +110,13 @@ "DATA0110\n", "from_lemi424.mth5\n", "lemi_reader_test.py\n", + "magdelena.h5\n", + "out.png\n", + "process_lemi.py\n", + "sample.json\n", "stations\n", - "test_read_multiple_lemi.py\n" + "test_read_multiple_lemi.py\n", + "tmp\n" ] }, { @@ -164,8 +169,13 @@ "DATA0110\n", "from_lemi424.mth5\n", "lemi_reader_test.py\n", + "magdelena.h5\n", + "out.png\n", + "process_lemi.py\n", + "sample.json\n", "stations\n", - "test_read_multiple_lemi.py\n" + "test_read_multiple_lemi.py\n", + "tmp\n" ] }, { @@ -242,10 +252,15 @@ "name": "stdout", "output_type": "stream", "text": [ + "202009302020.INF\n", "202009302021.TXT\n", + "202009302028.INF\n", "202009302029.TXT\n", + "202009302053.INF\n", "202009302054.TXT\n", + "202009302111.INF\n", "202009302112.TXT\n", + "202009302113.INF\n", "202009302114.TXT\n", "202010010000.TXT\n", "202010020000.TXT\n", @@ -311,6 +326,14 @@ "\n" ] }, + { + "cell_type": "markdown", + "id": "c1c36a95-f3a5-4de7-933b-71c070f62d42", + "metadata": {}, + "source": [ + "## Replace l424_list below with sniffer that takes first and last lines of TXT" + ] + }, { "cell_type": "markdown", "id": "d5130d40-38c7-4503-a97b-88e770e344f2", @@ -1261,9 +1284,17 @@ "station_data_df[COLUMNS[1:]]" ] }, + { + "cell_type": "markdown", + "id": "000d81a2-6942-4842-9903-b688052bc873", + "metadata": {}, + "source": [ + "Here is some logic we will wrap as a function to figure out which files are part of a contiguous run" + ] + }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 34, "id": "447a8eb9-608f-46cf-9d88-c2f421b7e614", "metadata": {}, "outputs": [ @@ -1316,7 +1347,7 @@ " 2020-09-30 20:21:00\n", " 2020-09-30 20:28:15\n", " 436\n", - " \n", + " 001\n", " 1.0\n", " True\n", " 202009302021.TXT\n", @@ -1326,7 +1357,7 @@ " 2020-09-30 20:29:00\n", " 2020-09-30 20:42:16\n", " 797\n", - " \n", + " 002\n", " 1.0\n", " True\n", " 202009302029.TXT\n", @@ -1336,7 +1367,7 @@ " 2020-09-30 20:54:00\n", " 2020-09-30 21:11:01\n", " 1022\n", - " \n", + " 003\n", " 1.0\n", " True\n", " 202009302054.TXT\n", @@ -1346,7 +1377,7 @@ " 2020-09-30 21:12:00\n", " 2020-09-30 21:13:45\n", " 106\n", - " \n", + " 004\n", " 1.0\n", " True\n", " 202009302112.TXT\n", @@ -1356,7 +1387,7 @@ " 2020-09-30 21:14:00\n", " 2020-09-30 23:59:59\n", " 9960\n", - " \n", + " 005\n", " 1.0\n", " True\n", " 202009302114.TXT\n", @@ -1366,7 +1397,7 @@ " 2020-10-01 00:00:00\n", " 2020-10-01 23:59:59\n", " 86400\n", - " \n", + " 005\n", " 1.0\n", " False\n", " 202010010000.TXT\n", @@ -1376,7 +1407,7 @@ " 2020-10-02 00:00:00\n", " 2020-10-02 23:59:59\n", " 86400\n", - " \n", + " 005\n", " 1.0\n", " False\n", " 202010020000.TXT\n", @@ -1386,7 +1417,7 @@ " 2020-10-03 00:00:00\n", " 2020-10-03 23:59:59\n", " 86400\n", - " \n", + " 005\n", " 1.0\n", " False\n", " 202010030000.TXT\n", @@ -1396,7 +1427,7 @@ " 2020-10-04 00:00:00\n", " 2020-10-04 23:59:59\n", " 86400\n", - " \n", + " 005\n", " 1.0\n", " False\n", " 202010040000.TXT\n", @@ -1406,7 +1437,7 @@ " 2020-10-05 00:00:00\n", " 2020-10-05 23:59:59\n", " 86400\n", - " \n", + " 005\n", " 1.0\n", " False\n", " 202010050000.TXT\n", @@ -1416,7 +1447,7 @@ " 2020-10-06 00:00:00\n", " 2020-10-06 23:59:59\n", " 86400\n", - " \n", + " 005\n", " 1.0\n", " False\n", " 202010060000.TXT\n", @@ -1426,7 +1457,7 @@ " 2020-10-07 00:00:00\n", " 2020-10-07 14:19:46\n", " 51587\n", - " \n", + " 005\n", " 1.0\n", " False\n", " 202010070000.TXT\n", @@ -1437,18 +1468,18 @@ ], "text/plain": [ " first_sample_time last_sample_time num_lines run_id sample_rate \\\n", - "0 2020-09-30 20:21:00 2020-09-30 20:28:15 436 1.0 \n", - "1 2020-09-30 20:29:00 2020-09-30 20:42:16 797 1.0 \n", - "2 2020-09-30 20:54:00 2020-09-30 21:11:01 1022 1.0 \n", - "3 2020-09-30 21:12:00 2020-09-30 21:13:45 106 1.0 \n", - "4 2020-09-30 21:14:00 2020-09-30 23:59:59 9960 1.0 \n", - "5 2020-10-01 00:00:00 2020-10-01 23:59:59 86400 1.0 \n", - "6 2020-10-02 00:00:00 2020-10-02 23:59:59 86400 1.0 \n", - "7 2020-10-03 00:00:00 2020-10-03 23:59:59 86400 1.0 \n", - "8 2020-10-04 00:00:00 2020-10-04 23:59:59 86400 1.0 \n", - "9 2020-10-05 00:00:00 2020-10-05 23:59:59 86400 1.0 \n", - "10 2020-10-06 00:00:00 2020-10-06 23:59:59 86400 1.0 \n", - "11 2020-10-07 00:00:00 2020-10-07 14:19:46 51587 1.0 \n", + "0 2020-09-30 20:21:00 2020-09-30 20:28:15 436 001 1.0 \n", + "1 2020-09-30 20:29:00 2020-09-30 20:42:16 797 002 1.0 \n", + "2 2020-09-30 20:54:00 2020-09-30 21:11:01 1022 003 1.0 \n", + "3 2020-09-30 21:12:00 2020-09-30 21:13:45 106 004 1.0 \n", + "4 2020-09-30 21:14:00 2020-09-30 23:59:59 9960 005 1.0 \n", + "5 2020-10-01 00:00:00 2020-10-01 23:59:59 86400 005 1.0 \n", + "6 2020-10-02 00:00:00 2020-10-02 23:59:59 86400 005 1.0 \n", + "7 2020-10-03 00:00:00 2020-10-03 23:59:59 86400 005 1.0 \n", + "8 2020-10-04 00:00:00 2020-10-04 23:59:59 86400 005 1.0 \n", + "9 2020-10-05 00:00:00 2020-10-05 23:59:59 86400 005 1.0 \n", + "10 2020-10-06 00:00:00 2020-10-06 23:59:59 86400 005 1.0 \n", + "11 2020-10-07 00:00:00 2020-10-07 14:19:46 51587 005 1.0 \n", "\n", " new_run file_base \n", "0 True 202009302021.TXT \n", @@ -1465,12 +1496,16 @@ "11 False 202010070000.TXT " ] }, - "execution_count": 20, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "\"\"\"\n", + "Builds the boolean valued column \"new run\".\n", + "True when there is a gap between the data of the row under consideration and False if contiguous\n", + "\"\"\"\n", "new_run = n_files * [True]\n", "for i_row, row in station_data_df.iterrows():\n", " if i_row == 0:\n", @@ -1502,9 +1537,27 @@ " " ] }, + { + "cell_type": "markdown", + "id": "4e1897f3-c164-45f6-9545-12c48939255a", + "metadata": {}, + "source": [ + "Now all the rows that have True in column \"new_run\" are separate runs.\n", + "\n", + "All the columns that are false should recursively look upward until they find a True value to see the file with the start of the run they belong to.\n", + "\n", + "Next step is to issue some run labels. Here we will use a scheme like:\n", + "001\n", + "002\n", + "...\n", + "\n", + "\n", + "So call the first run 001" + ] + }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 35, "id": "2783b71a-06ad-4855-b92f-d9db81940a7b", "metadata": {}, "outputs": [ @@ -1514,7 +1567,7 @@ "'001'" ] }, - "execution_count": 21, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -1527,7 +1580,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 36, "id": "d5cf56ca-1b46-421f-99cb-adf377b661f5", "metadata": {}, "outputs": [], @@ -1538,7 +1591,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 37, "id": "f5e4b88f-1055-47ae-9b7d-ff4d97ffdbbf", "metadata": {}, "outputs": [ @@ -1564,7 +1617,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 38, "id": "b37ac4bd-87e3-44a8-aadc-c32c21fee65d", "metadata": {}, "outputs": [ @@ -1753,7 +1806,7 @@ "11 False 202010070000.TXT " ] }, - "execution_count": 24, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -1765,7 +1818,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 39, "id": "7d1e5e3a-fa85-407c-8c5f-e70bb0ed733c", "metadata": {}, "outputs": [], @@ -1775,7 +1828,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 40, "id": "53460041-dcfc-4da4-9d4e-0c8308d9fcf4", "metadata": {}, "outputs": [ @@ -1804,7 +1857,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 41, "id": "42adef43-c662-4495-9671-16e9304e9590", "metadata": {}, "outputs": [ @@ -1825,7 +1878,7 @@ " PosixPath('/home/kkappler/software/irismt/aurora/tests/LEMI/stations/station_53/202010070000.TXT')]}" ] }, - "execution_count": 27, + "execution_count": 41, "metadata": {}, "output_type": "execute_result" } @@ -1836,7 +1889,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 42, "id": "c9d341b9-e3bc-4498-9175-300d40010046", "metadata": {}, "outputs": [], @@ -1846,7 +1899,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 43, "id": "4a7f9d48-6548-4f3c-81b1-f0cb97724965", "metadata": {}, "outputs": [], @@ -1860,7 +1913,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 44, "id": "aa6e0913-2fce-431b-bc85-3087aadfab37", "metadata": {}, "outputs": [ @@ -1876,7 +1929,7 @@ "\tComponents: ['bx', 'by', 'bz', 'e1', 'e2', 'temperature_e', 'temperature_h']" ] }, - "execution_count": 30, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } @@ -1887,7 +1940,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 45, "id": "f1b22561-7c05-436a-abad-a7b3c6cf39f9", "metadata": {}, "outputs": [ @@ -1927,7 +1980,7 @@ "}" ] }, - "execution_count": 31, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } @@ -1938,7 +1991,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 46, "id": "31afe739-e93e-4718-b0ab-881ab2de0e59", "metadata": {}, "outputs": [ @@ -1948,7 +2001,7 @@ "mth5.timeseries.run_ts.RunTS" ] }, - "execution_count": 32, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } @@ -1959,7 +2012,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 47, "id": "4a6689ff-e1aa-437d-9a33-2bbd800bd0ef", "metadata": {}, "outputs": [ @@ -2346,9 +2399,9 @@ " id: None\n", " sample_rate: 1.0\n", " time_period.end: 2020-09-30T20:28:15+00:00\n", - " time_period.start: 2020-09-30T20:21:00+00:00
  • channels_recorded_auxiliary :
    ['temperature_e', 'temperature_h']
    channels_recorded_electric :
    ['e1', 'e2']
    channels_recorded_magnetic :
    ['bx', 'by', 'bz']
    data_logger.firmware.author :
    None
    data_logger.firmware.name :
    None
    data_logger.firmware.version :
    None
    data_logger.id :
    None
    data_logger.manufacturer :
    None
    data_logger.timing_system.drift :
    0.0
    data_logger.timing_system.type :
    GPS
    data_logger.timing_system.uncertainty :
    0.0
    data_logger.type :
    None
    data_type :
    BBMT
    id :
    None
    sample_rate :
    1.0
    time_period.end :
    2020-09-30T20:28:15+00:00
    time_period.start :
    2020-09-30T20:21:00+00:00
    • " ], "text/plain": [ "\n", @@ -2663,7 +2716,7 @@ " time_period.start: 2020-09-30T20:21:00+00:00" ] }, - "execution_count": 33, + "execution_count": 47, "metadata": {}, "output_type": "execute_result" } @@ -2674,7 +2727,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 48, "id": "978b6a3f-e317-44b2-86df-1713674e1efe", "metadata": {}, "outputs": [ @@ -3061,11 +3114,11 @@ " id: None\n", " sample_rate: 1.0\n", " time_period.end: 2020-10-07T14:19:46+00:00\n", - " time_period.start: 2020-09-30T21:14:00+00:00
    • bx
      (time)
      float64
      2.378e+04 2.378e+04 ... 2.378e+04
      channel_number :
      0
      component :
      bx
      data_quality.rating.value :
      0
      filter.applied :
      [False]
      filter.name :
      []
      location.elevation :
      0.0
      location.latitude :
      0.0
      location.longitude :
      0.0
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      sample_rate :
      1.0
      sensor.id :
      None
      sensor.manufacturer :
      None
      sensor.type :
      None
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      magnetic
      units :
      None
      station.id :
      None
      run.id :
      None
      array([ 23778.555,  23778.572,  23778.602, ...,  23776.707,  23776.676,\n",
+       "        23776.633])
    • by
      (time)
      float64
      213.9 213.9 213.9 ... 247.3 247.3
      channel_number :
      0
      component :
      by
      data_quality.rating.value :
      0
      filter.applied :
      [False]
      filter.name :
      []
      location.elevation :
      0.0
      location.latitude :
      0.0
      location.longitude :
      0.0
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      sample_rate :
      1.0
      sensor.id :
      None
      sensor.manufacturer :
      None
      sensor.type :
      None
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      magnetic
      units :
      None
      station.id :
      None
      run.id :
      None
      array([ 213.914,  213.922,  213.92 , ...,  247.202,  247.268,  247.342])
    • bz
      (time)
      float64
      4.183e+04 4.183e+04 ... 4.184e+04
      channel_number :
      0
      component :
      bz
      data_quality.rating.value :
      0
      filter.applied :
      [False]
      filter.name :
      []
      location.elevation :
      0.0
      location.latitude :
      0.0
      location.longitude :
      0.0
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      sample_rate :
      1.0
      sensor.id :
      None
      sensor.manufacturer :
      None
      sensor.type :
      None
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      magnetic
      units :
      None
      station.id :
      None
      run.id :
      None
      array([ 41834.609,  41834.605,  41834.573, ...,  41843.23 ,  41843.23 ,\n",
+       "        41843.214])
    • e1
      (time)
      float64
      134.8 134.9 135.0 ... 130.3 130.2
      channel_number :
      0
      component :
      e1
      data_quality.rating.value :
      0
      dipole_length :
      None
      filter.applied :
      [False]
      filter.name :
      []
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      negative.elevation :
      0.0
      negative.id :
      None
      negative.latitude :
      0.0
      negative.longitude :
      0.0
      negative.manufacturer :
      None
      negative.type :
      None
      positive.elevation :
      0.0
      positive.id :
      None
      positive.latitude :
      0.0
      positive.longitude :
      0.0
      positive.manufacturer :
      None
      positive.type :
      None
      sample_rate :
      1.0
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      electric
      units :
      None
      station.id :
      None
      run.id :
      None
      array([ 134.816,  134.873,  134.98 , ...,  130.399,  130.299,  130.25 ])
    • e2
      (time)
      float64
      -109.8 -109.9 ... -40.95 -40.93
      channel_number :
      0
      component :
      e2
      data_quality.rating.value :
      0
      dipole_length :
      None
      filter.applied :
      [False]
      filter.name :
      []
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      negative.elevation :
      0.0
      negative.id :
      None
      negative.latitude :
      0.0
      negative.longitude :
      0.0
      negative.manufacturer :
      None
      negative.type :
      None
      positive.elevation :
      0.0
      positive.id :
      None
      positive.latitude :
      0.0
      positive.longitude :
      0.0
      positive.manufacturer :
      None
      positive.type :
      None
      sample_rate :
      1.0
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      electric
      units :
      None
      station.id :
      None
      run.id :
      None
      array([-109.831, -109.902, -109.877, ...,  -40.881,  -40.953,  -40.933])
    • temperature_e
      (time)
      float64
      46.5 46.46 46.47 ... 18.17 18.17
      channel_number :
      0
      component :
      temperature_e
      data_quality.rating.value :
      0
      filter.applied :
      [False]
      filter.name :
      []
      location.elevation :
      0.0
      location.latitude :
      0.0
      location.longitude :
      0.0
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      sample_rate :
      1.0
      sensor.id :
      None
      sensor.manufacturer :
      None
      sensor.type :
      None
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      auxiliary
      units :
      None
      station.id :
      None
      run.id :
      None
      array([ 46.5 ,  46.46,  46.47, ...,  18.18,  18.17,  18.17])
    • temperature_h
      (time)
      float64
      36.3 36.3 36.3 ... 19.6 19.6 19.6
      channel_number :
      0
      component :
      temperature_h
      data_quality.rating.value :
      0
      filter.applied :
      [False]
      filter.name :
      []
      location.elevation :
      0.0
      location.latitude :
      0.0
      location.longitude :
      0.0
      measurement_azimuth :
      0.0
      measurement_tilt :
      0.0
      sample_rate :
      1.0
      sensor.id :
      None
      sensor.manufacturer :
      None
      sensor.type :
      None
      time_period.end :
      2020-10-07T14:19:46+00:00
      time_period.start :
      2020-09-30T21:14:00+00:00
      type :
      auxiliary
      units :
      None
      station.id :
      None
      run.id :
      None
      array([ 36.3,  36.3,  36.3, ...,  19.6,  19.6,  19.6])
  • channels_recorded_auxiliary :
    ['temperature_e', 'temperature_h']
    channels_recorded_electric :
    ['e1', 'e2']
    channels_recorded_magnetic :
    ['bx', 'by', 'bz']
    data_logger.firmware.author :
    None
    data_logger.firmware.name :
    None
    data_logger.firmware.version :
    None
    data_logger.id :
    None
    data_logger.manufacturer :
    None
    data_logger.timing_system.drift :
    0.0
    data_logger.timing_system.type :
    GPS
    data_logger.timing_system.uncertainty :
    0.0
    data_logger.type :
    None
    data_type :
    BBMT
    id :
    None
    sample_rate :
    1.0
    time_period.end :
    2020-10-07T14:19:46+00:00
    time_period.start :
    2020-09-30T21:14:00+00:00
    • " ], "text/plain": [ "\n", @@ -3100,7 +3153,7 @@ " time_period.start: 2020-09-30T21:14:00+00:00" ] }, - "execution_count": 34, + "execution_count": 48, "metadata": {}, "output_type": "execute_result" } @@ -3111,7 +3164,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 49, "id": "c93ebc85-6723-4e19-94b8-85f7dcd709c3", "metadata": {}, "outputs": [ @@ -3151,7 +3204,7 @@ "}" ] }, - "execution_count": 35, + "execution_count": 49, "metadata": {}, "output_type": "execute_result" } @@ -3171,7 +3224,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 50, "id": "35dec618-dec2-4a72-a351-3c2b969511e5", "metadata": {}, "outputs": [], @@ -3182,7 +3235,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 53, "id": "840b5aa0-db95-4ebd-9c85-31113236ef5f", "metadata": {}, "outputs": [], @@ -3197,7 +3250,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 54, "id": "2b4b714d-32f9-4e7b-b0c5-14079c99d31d", "metadata": {}, "outputs": [ @@ -3205,8 +3258,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "2022-07-30 09:35:14,229 [line 591] mth5.mth5.MTH5.open_mth5 - WARNING: magdelena.h5 will be overwritten in 'w' mode\n", - "2022-07-30 09:35:14,652 [line 656] mth5.mth5.MTH5._initialize_file - INFO: Initialized MTH5 0.1.0 file magdelena.h5 in mode w\n" + "2022-08-05 17:01:07,351 [line 591] mth5.mth5.MTH5.open_mth5 - WARNING: magdelena.h5 will be overwritten in 'w' mode\n", + "2022-08-05 17:01:08,014 [line 656] mth5.mth5.MTH5._initialize_file - INFO: Initialized MTH5 0.1.0 file magdelena.h5 in mode w\n" ] } ], @@ -3217,7 +3270,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 55, "id": "54c72d9e-e17e-45ff-9409-485788d8680a", "metadata": {}, "outputs": [], @@ -3230,7 +3283,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 56, "id": "e8b96ee4-5139-448b-b670-9869499dee1d", "metadata": {}, "outputs": [ @@ -3251,7 +3304,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 57, "id": "aaae5911-5d73-42cd-b489-7eee29beb6fd", "metadata": {}, "outputs": [], @@ -3262,7 +3315,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 58, "id": "b311fe48-59b2-4ff7-91e1-74cb4b0baeea", "metadata": {}, "outputs": [], @@ -3272,7 +3325,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 59, "id": "ddc2c102-c4fc-4bb5-8c9d-691625529b58", "metadata": {}, "outputs": [], @@ -3282,7 +3335,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 60, "id": "5bd20e6a-3b17-4002-a6a5-5bf37e1473aa", "metadata": {}, "outputs": [ @@ -3290,7 +3343,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2022-07-30 09:35:21,938 [line 731] mth5.mth5.MTH5.close_mth5 - INFO: Flushing and closing magdelena.h5\n" + "2022-08-05 17:01:13,615 [line 731] mth5.mth5.MTH5.close_mth5 - INFO: Flushing and closing magdelena.h5\n" ] } ], @@ -3300,7 +3353,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 61, "id": "11f733da-43a4-48ed-a034-d065b875c56c", "metadata": {}, "outputs": [], @@ -3311,7 +3364,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 62, "id": "0818f400-7f65-4111-a1c9-593b68435a20", "metadata": {}, "outputs": [ @@ -3362,7 +3415,7 @@ " ........................." ] }, - "execution_count": 46, + "execution_count": 62, "metadata": {}, "output_type": "execute_result" } @@ -3373,7 +3426,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 63, "id": "84ea94e5-0523-43fd-99ef-c4c6541ed401", "metadata": {}, "outputs": [ @@ -3381,13 +3434,21 @@ "name": "stderr", "output_type": "stream", "text": [ - "2022-07-30 09:35:33,832 [line 731] mth5.mth5.MTH5.close_mth5 - INFO: Flushing and closing magdelena.h5\n" + "2022-08-05 17:01:18,659 [line 731] mth5.mth5.MTH5.close_mth5 - INFO: Flushing and closing magdelena.h5\n" ] } ], "source": [ "mm.close_mth5()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "383af849-4458-4978-a243-5f7fe7c1b5d4", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/mth5/io/lemi424.py b/mth5/io/lemi424.py index d050c2f9..6eeb39ce 100644 --- a/mth5/io/lemi424.py +++ b/mth5/io/lemi424.py @@ -2,7 +2,7 @@ """ Created on Tue May 11 15:31:31 2021 -:copyright: +:copyright: Jared Peacock (jpeacock@usgs.gov) :license: MIT @@ -25,7 +25,7 @@ class LEMI424: """ - def __init__(self, fn=None): + def __init__(self, fn=[]): self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}") self.fn = fn self._has_data = False @@ -61,16 +61,44 @@ def __init__(self, fn=None): if self.fn: self.read() + @property + def num_source_files(self): + return len(self.fn) + @property def fn(self): return self._fn + @property + def validate_fn(self): + """ + Need to check that the filenames are sequential + """ + return True + @fn.setter - def fn(self, value): - if value is not None: - value = Path(value) - if not value.exists(): - raise IOError(f"Could not find {value}") + def fn(self, value, sort=True): + """ + + Parameters + ---------- + value:string or pathlib.Path, or list of these + + """ + if isinstance(value, list): + value = [Path(x) for x in value] + exists = [x.exists() for x in value] + for i_fn, cond in enumerate(exists): + if not cond: + raise IOError(f"Could not find {value[i_fn]}") + elif value is not None: + value = [ + Path(value), + ] + if not value[0].exists(): + raise IOError(f"Could not find {value[0]}") + if sort: + value.sort() self._fn = value @property @@ -80,16 +108,16 @@ def start(self): [ "-".join( [ - f"{self._df.year.min()}", - f"{self._df.month.min():02d}", - f"{self._df.day.min():02d}", + f"{self._df.iloc[0].year}", + f"{self._df.iloc[0].month:02d}", + f"{self._df.iloc[0].day:02d}", ] ), ":".join( [ - f"{self._df.hour.min():02d}", - f"{self._df.minute.min():02d}", - f"{self._df.second.min():02d}", + f"{self._df.iloc[0].hour:02d}", + f"{self._df.iloc[0].minute:02d}", + f"{self._df.iloc[0].second:02d}", ] ), ] @@ -102,16 +130,16 @@ def end(self): [ "-".join( [ - f"{self._df.year.max()}", - f"{self._df.month.max():02d}", - f"{self._df.day.max():02d}", + f"{self._df.iloc[-1].year}", + f"{self._df.iloc[-1].month:02d}", + f"{self._df.iloc[-1].day:02d}", ] ), ":".join( [ - f"{self._df.hour.max():02d}", - f"{self._df.minute.max():02d}", - f"{self._df.second.max():02d}", + f"{self._df.iloc[-1].hour:02d}", + f"{self._df.iloc[-1].minute:02d}", + f"{self._df.iloc[-1].second:02d}", ] ), ] @@ -160,7 +188,7 @@ def run_metadata(self): r.time_period.start = self.start r.time_period.end = self.end - def read(self, fn=None): + def read(self, fn=[]): """ Read a LEMI424 file using pandas @@ -170,16 +198,24 @@ def read(self, fn=None): :rtype: TYPE """ - if fn is not None: + if fn: self.fn = fn - if not self.fn.exists(): + exists = [x.exists() for x in self.fn] + if all(x for x in exists): + pass + else: msg = "Could not find file %s" - self.logger.error(msg, self.fn) - raise IOError(msg % self.fn) + for i_fn, cond in enumerate(exists): + if not cond: + self.logger.error(msg, self.fn[i_fn]) + raise IOError(msg % self.fn[i_fn]) - self._df = pd.read_csv(self.fn, delimiter="\s+", names=self.column_names) + dfs = self.num_source_files * [None] + for i, fn in enumerate(self.fn): + dfs[i] = pd.read_csv(fn, delimiter="\s+", names=self.column_names) + self._df = pd.concat(dfs) self._has_data = True def to_run_ts(self, fn=None, e_channels=["e1", "e2"]): diff --git a/mth5/io/lemi424_new.py b/mth5/io/lemi424_new.py deleted file mode 100644 index 6eeb39ce..00000000 --- a/mth5/io/lemi424_new.py +++ /dev/null @@ -1,276 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Tue May 11 15:31:31 2021 - -:copyright: - Jared Peacock (jpeacock@usgs.gov) - -:license: MIT - -""" - -from pathlib import Path -import pandas as pd -import numpy as np -import logging - -from mth5.timeseries import ChannelTS, RunTS -from mt_metadata.timeseries import Station, Run - - -class LEMI424: - """ - Read in a LEMI424 file, this is a place holder until IRIS finalizes - their reader. - - """ - - def __init__(self, fn=[]): - self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}") - self.fn = fn - self._has_data = False - self.sample_rate = 1.0 - self.chunk_size = 10000 - self.column_names = [ - "year", - "month", - "day", - "hour", - "minute", - "second", - "bx", - "by", - "bz", - "temperature_e", - "temperature_h", - "e1", - "e2", - "e3", - "e4", - "battery", - "elevation", - "latitude", - "lat_hemisphere", - "longitude", - "lon_hemisphere", - "n_satellites", - "gps_fix", - "tdiff", - ] - - if self.fn: - self.read() - - @property - def num_source_files(self): - return len(self.fn) - - @property - def fn(self): - return self._fn - - @property - def validate_fn(self): - """ - Need to check that the filenames are sequential - """ - return True - - @fn.setter - def fn(self, value, sort=True): - """ - - Parameters - ---------- - value:string or pathlib.Path, or list of these - - """ - if isinstance(value, list): - value = [Path(x) for x in value] - exists = [x.exists() for x in value] - for i_fn, cond in enumerate(exists): - if not cond: - raise IOError(f"Could not find {value[i_fn]}") - elif value is not None: - value = [ - Path(value), - ] - if not value[0].exists(): - raise IOError(f"Could not find {value[0]}") - if sort: - value.sort() - self._fn = value - - @property - def start(self): - if self._has_data: - return "T".join( - [ - "-".join( - [ - f"{self._df.iloc[0].year}", - f"{self._df.iloc[0].month:02d}", - f"{self._df.iloc[0].day:02d}", - ] - ), - ":".join( - [ - f"{self._df.iloc[0].hour:02d}", - f"{self._df.iloc[0].minute:02d}", - f"{self._df.iloc[0].second:02d}", - ] - ), - ] - ) - - @property - def end(self): - if self._has_data: - return "T".join( - [ - "-".join( - [ - f"{self._df.iloc[-1].year}", - f"{self._df.iloc[-1].month:02d}", - f"{self._df.iloc[-1].day:02d}", - ] - ), - ":".join( - [ - f"{self._df.iloc[-1].hour:02d}", - f"{self._df.iloc[-1].minute:02d}", - f"{self._df.iloc[-1].second:02d}", - ] - ), - ] - ) - - @property - def latitude(self): - if self._has_data: - return np.rad2deg(self._df.latitude.median() / 3600) - - @property - def longitude(self): - if self._has_data: - return np.rad2deg(self._df.longitude.median() / 3600) - - @property - def elevation(self): - if self._has_data: - return self._df.elevation.median() - - @property - def gps_lock(self): - if self._has_data: - return self._df.gps_fix.values - - @property - def station_metadata(self): - s = Station() - if self._has_data: - s.location.latitude = self.latitude - s.location.longitude = self.longitude - s.location.elevation = self.elevation - s.time_period.start = self.start - s.time_period.end = self.end - return s - - @property - def run_metadata(self): - r = Run() - r.sample_rate = self.sample_rate - r.data_logger.model = "LEMI424" - r.data_logger.manufacturer = "LEMI" - if self._has_data: - r.data_logger.power_source.voltage.start = self._df.battery.max() - r.data_logger.power_source.voltage.end = self._df.battery.min() - r.time_period.start = self.start - r.time_period.end = self.end - - def read(self, fn=[]): - """ - Read a LEMI424 file using pandas - - :param fn: DESCRIPTION, defaults to None - :type fn: TYPE, optional - :return: DESCRIPTION - :rtype: TYPE - - """ - if fn: - self.fn = fn - - exists = [x.exists() for x in self.fn] - if all(x for x in exists): - pass - else: - msg = "Could not find file %s" - for i_fn, cond in enumerate(exists): - if not cond: - self.logger.error(msg, self.fn[i_fn]) - raise IOError(msg % self.fn[i_fn]) - - dfs = self.num_source_files * [None] - for i, fn in enumerate(self.fn): - dfs[i] = pd.read_csv(fn, delimiter="\s+", names=self.column_names) - - self._df = pd.concat(dfs) - self._has_data = True - - def to_run_ts(self, fn=None, e_channels=["e1", "e2"]): - """ - Return a RunTS object from the data - - :param fn: DESCRIPTION, defaults to None - :type fn: TYPE, optional - :return: DESCRIPTION - :rtype: TYPE - - """ - ch_list = [] - for comp in ( - ["bx", "by", "bz"] + e_channels + ["temperature_e", "temperature_h"] - ): - if comp[0] in ["h", "b"]: - ch = ChannelTS("magnetic") - elif comp[0] in ["e"]: - ch = ChannelTS("electric") - else: - ch = ChannelTS("auxiliary") - - ch.sample_rate = self.sample_rate - ch.start = self.start - ch.ts = self._df[comp].values - ch.component = comp - ch_list.append(ch) - - return RunTS( - array_list=ch_list, - station_metadata=self.station_metadata, - run_metadata=self.run_metadata, - ) - - -# ============================================================================= -# define the reader -# ============================================================================= -def read_lemi424(fn, e_channels=["e1", "e2"], logger_file_handler=None): - """ - Read a LEMI 424 TXT file. - - :param fn: input file name - :type fn: string or Path - :param e_channels: A list of electric channels to read, - defaults to ["e1", "e2"] - :type e_channels: list of strings, optional - :return: A RunTS object with appropriate metadata - :rtype: :class:`mth5.timeseries.RunTS` - - """ - - txt_obj = LEMI424() - if logger_file_handler: - txt_obj.logger.addHandler(logger_file_handler) - txt_obj.read(fn) - return txt_obj.to_run_ts(e_channels=e_channels)