docs/notebooks/operate_aurora.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Outines the process of making an MTH5 file, generating a processing config, and runs the aurora processor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-05 15:25:44,710 [line 135] mth5.setup_logger - INFO: Logging file can be found /Users/tronan/Desktop/Projects/mth5/logs/mth5_debug.log\n"
     ]
    }
   ],
   "source": [
    "# Required imports for theh program. \n",
    "from pathlib import Path\n",
    "import sys\n",
    "import pandas as pd\n",
    "from mth5.clients.make_mth5 import MakeMTH5\n",
    "from mth5 import mth5, timeseries\n",
    "from mt_metadata.utils.mttime import get_now_utc, MTime\n",
    "from aurora.config.config_creator import ConfigCreator\n",
    "from aurora.pipelines.process_mth5 import process_mth5_run"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Builds MTH5 File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set path so MTH5 file builds to current working directory. \n",
    "default_path = Path().cwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the Make MTH5 code. \n",
    "m = MakeMTH5(mth5_version='0.1.0')\n",
    "m.client = \"IRIS\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate data frame of FDSN Network, Station, Location, Channel, Startime, Endtime codes of interest\n",
    "\n",
    "ZUCAS04LQ1 = ['ZU', 'CAS04', '', 'LQE', '2020-06-02T19:00:00', '2020-07-13T19:00:00']\n",
    "ZUCAS04LQ2 = ['ZU', 'CAS04', '', 'LQN', '2020-06-02T19:00:00', '2020-07-13T19:00:00']\n",
    "ZUCAS04BF1 = ['ZU', 'CAS04', '', 'LFE', '2020-06-02T19:00:00', '2020-07-13T19:00:00']\n",
    "ZUCAS04BF2 = ['ZU', 'CAS04', '', 'LFN', '2020-06-02T19:00:00', '2020-07-13T19:00:00']\n",
    "ZUCAS04BF3 = ['ZU', 'CAS04', '', 'LFZ', '2020-06-02T19:00:00', '2020-07-13T19:00:00']\n",
    "\n",
    "request_list = [ZUCAS04LQ1, ZUCAS04LQ2, ZUCAS04BF1, ZUCAS04BF2, ZUCAS04BF3]\n",
    "\n",
    "# Turn list into dataframe\n",
    "request_df =  pd.DataFrame(request_list, columns=m.column_names)\\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  network station location channel                start                  end\n",
      "0      ZU   CAS04              LQE  2020-06-02T19:00:00  2020-07-13T19:00:00\n",
      "1      ZU   CAS04              LQN  2020-06-02T19:00:00  2020-07-13T19:00:00\n",
      "2      ZU   CAS04              LFE  2020-06-02T19:00:00  2020-07-13T19:00:00\n",
      "3      ZU   CAS04              LFN  2020-06-02T19:00:00  2020-07-13T19:00:00\n",
      "4      ZU   CAS04              LFZ  2020-06-02T19:00:00  2020-07-13T19:00:00\n"
     ]
    }
   ],
   "source": [
    "# Inspect the dataframe\n",
    "print(request_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Request the inventory information from IRIS\n",
    "inventory = m.get_inventory_from_df(request_df, data=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Inventory created at 2021-11-05T22:26:14.221364Z\n",
       "\tCreated by: ObsPy 1.2.2\n",
       "\t\t    https://www.obspy.org\n",
       "\tSending institution: MTH5\n",
       "\tContains:\n",
       "\t\tNetworks (1):\n",
       "\t\t\tZU\n",
       "\t\tStations (1):\n",
       "\t\t\tZU.CAS04 (Corral Hollow, CA, USA)\n",
       "\t\tChannels (5):\n",
       "\t\t\tZU.CAS04..LFZ, ZU.CAS04..LFN, ZU.CAS04..LFE, ZU.CAS04..LQN, \n",
       "\t\t\tZU.CAS04..LQE,\n",
       " 0 Trace(s) in Stream:\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspect the inventory\n",
    "inventory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate te MTH5 from the Input DataFrame and leave it open. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-05 15:26:20,273 [line 594] mth5.mth5.MTH5.open_mth5 - WARNING: ZU_CAS04.h5 will be overwritten in 'w' mode\n",
      "2021-11-05 15:26:21,459 [line 653] mth5.mth5.MTH5._initialize_file - INFO: Initialized MTH5 file /Users/tronan/Desktop/Projects/aurora/tests/ZU_CAS04.h5 in mode w\n",
      "2021-11-05 15:26:38,467 [line 127] mt_metadata.base.metadata.station.add_run - WARNING: Run a is being overwritten with current information\n",
      "2021-11-05 15:26:41,799 [line 709] mth5.groups.base.Station.add_run - INFO: run a already exists, returning existing group.\n",
      "2021-11-05 15:26:42,338 [line 224] mth5.timeseries.run_ts.RunTS.validate_metadata - WARNING: start time of dataset 2020-06-02T19:00:00+00:00 does not match metadata start 2020-06-02T18:41:43+00:00 updating metatdata value to 2020-06-02T19:00:00+00:00\n",
      "2021-11-05 15:27:01,653 [line 709] mth5.groups.base.Station.add_run - INFO: run b already exists, returning existing group.\n",
      "2021-11-05 15:27:25,841 [line 709] mth5.groups.base.Station.add_run - INFO: run c already exists, returning existing group.\n",
      "2021-11-05 15:27:51,950 [line 709] mth5.groups.base.Station.add_run - INFO: run d already exists, returning existing group.\n",
      "2021-11-05 15:27:54,225 [line 235] mth5.timeseries.run_ts.RunTS.validate_metadata - WARNING: end time of dataset 2020-07-13T19:00:00+00:00 does not match metadata end 2020-07-13T21:46:12+00:00 updating metatdata value to 2020-07-13T19:00:00+00:00\n"
     ]
    }
   ],
   "source": [
    "mth5_object = m.make_mth5_from_fdsnclient(request_df, interact=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mth5_object.open_mth5(h5_path, 'w')\n",
    "# h5_path = str(default_path)+'/ZU_CAS04.h5'\n",
    "#mth5_object.close_mth5()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract information from the open MTH5 Object "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "/:\n",
       "====================\n",
       "    |- Group: Survey\n",
       "    ----------------\n",
       "        |- Group: Filters\n",
       "        -----------------\n",
       "            |- Group: coefficient\n",
       "            ---------------------\n",
       "                |- Group: v to counts (electric)\n",
       "                --------------------------------\n",
       "                |- Group: v to counts (magnetic)\n",
       "                --------------------------------\n",
       "            |- Group: fap\n",
       "            -------------\n",
       "            |- Group: fir\n",
       "            -------------\n",
       "            |- Group: time_delay\n",
       "            --------------------\n",
       "                |- Group: electric time offset\n",
       "                ------------------------------\n",
       "                |- Group: hx time offset\n",
       "                ------------------------\n",
       "                |- Group: hy time offset\n",
       "                ------------------------\n",
       "                |- Group: hz time offset\n",
       "                ------------------------\n",
       "            |- Group: zpk\n",
       "            -------------\n",
       "                |- Group: electric field 1 pole butterworth high-pass\n",
       "                -----------------------------------------------------\n",
       "                    --> Dataset: poles\n",
       "                    ....................\n",
       "                    --> Dataset: zeros\n",
       "                    ....................\n",
       "                |- Group: electric field 5 pole butterworth low-pass\n",
       "                ----------------------------------------------------\n",
       "                    --> Dataset: poles\n",
       "                    ....................\n",
       "                    --> Dataset: zeros\n",
       "                    ....................\n",
       "                |- Group: magnetic field 3 pole butterworth low-pass\n",
       "                ----------------------------------------------------\n",
       "                    --> Dataset: poles\n",
       "                    ....................\n",
       "                    --> Dataset: zeros\n",
       "                    ....................\n",
       "                |- Group: mv per km to v per m\n",
       "                ------------------------------\n",
       "                    --> Dataset: poles\n",
       "                    ....................\n",
       "                    --> Dataset: zeros\n",
       "                    ....................\n",
       "                |- Group: v per m to v\n",
       "                ----------------------\n",
       "                    --> Dataset: poles\n",
       "                    ....................\n",
       "                    --> Dataset: zeros\n",
       "                    ....................\n",
       "        |- Group: Reports\n",
       "        -----------------\n",
       "        |- Group: Standards\n",
       "        -------------------\n",
       "            --> Dataset: summary\n",
       "            ......................\n",
       "        |- Group: Stations\n",
       "        ------------------\n",
       "            |- Group: CAS04\n",
       "            ---------------\n",
       "                |- Group: a\n",
       "                -----------\n",
       "                    --> Dataset: ex\n",
       "                    .................\n",
       "                    --> Dataset: ey\n",
       "                    .................\n",
       "                    --> Dataset: hx\n",
       "                    .................\n",
       "                    --> Dataset: hy\n",
       "                    .................\n",
       "                    --> Dataset: hz\n",
       "                    .................\n",
       "                |- Group: b\n",
       "                -----------\n",
       "                    --> Dataset: ex\n",
       "                    .................\n",
       "                    --> Dataset: ey\n",
       "                    .................\n",
       "                    --> Dataset: hx\n",
       "                    .................\n",
       "                    --> Dataset: hy\n",
       "                    .................\n",
       "                    --> Dataset: hz\n",
       "                    .................\n",
       "                |- Group: c\n",
       "                -----------\n",
       "                    --> Dataset: ex\n",
       "                    .................\n",
       "                    --> Dataset: ey\n",
       "                    .................\n",
       "                    --> Dataset: hx\n",
       "                    .................\n",
       "                    --> Dataset: hy\n",
       "                    .................\n",
       "                    --> Dataset: hz\n",
       "                    .................\n",
       "                |- Group: d\n",
       "                -----------\n",
       "                    --> Dataset: ex\n",
       "                    .................\n",
       "                    --> Dataset: ey\n",
       "                    .................\n",
       "                    --> Dataset: hx\n",
       "                    .................\n",
       "                    --> Dataset: hy\n",
       "                    .................\n",
       "                    --> Dataset: hz\n",
       "                    ................."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mth5_object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Collect information from the MTh5 Object and use it in the config files. \n",
    "mth5_filename = mth5_object.filename\n",
    "version = mth5_object.file_version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Edit and update the MTH5 metadata \n",
    "s = mth5_object.get_station(\"CAS04\")\n",
    "s.metadata.location.declination.model = 'IGRF'\n",
    "s.write_metadata()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the available stations and runs from the MTH5 object\n",
    "ch_summary = mth5_object.channel_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>station</th>\n",
       "      <th>run</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>elevation</th>\n",
       "      <th>component</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>n_samples</th>\n",
       "      <th>sample_rate</th>\n",
       "      <th>measurement_type</th>\n",
       "      <th>azimuth</th>\n",
       "      <th>tilt</th>\n",
       "      <th>units</th>\n",
       "      <th>hdf5_reference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>a</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>ex</td>\n",
       "      <td>2020-06-02 19:00:00</td>\n",
       "      <td>2020-06-02 22:07:46</td>\n",
       "      <td>11267</td>\n",
       "      <td>1.0</td>\n",
       "      <td>electric</td>\n",
       "      <td>13.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>millivolts per kilometer</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>a</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>ey</td>\n",
       "      <td>2020-06-02 19:00:00</td>\n",
       "      <td>2020-06-02 22:07:46</td>\n",
       "      <td>11267</td>\n",
       "      <td>1.0</td>\n",
       "      <td>electric</td>\n",
       "      <td>103.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>millivolts per kilometer</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>a</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>hx</td>\n",
       "      <td>2020-06-02 19:00:00</td>\n",
       "      <td>2020-06-02 22:07:46</td>\n",
       "      <td>11267</td>\n",
       "      <td>1.0</td>\n",
       "      <td>magnetic</td>\n",
       "      <td>13.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nanotesla</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>a</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>hy</td>\n",
       "      <td>2020-06-02 19:00:00</td>\n",
       "      <td>2020-06-02 22:07:46</td>\n",
       "      <td>11267</td>\n",
       "      <td>1.0</td>\n",
       "      <td>magnetic</td>\n",
       "      <td>103.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nanotesla</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>a</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>hz</td>\n",
       "      <td>2020-06-02 19:00:00</td>\n",
       "      <td>2020-06-02 22:07:46</td>\n",
       "      <td>11267</td>\n",
       "      <td>1.0</td>\n",
       "      <td>magnetic</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nanotesla</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>b</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>ex</td>\n",
       "      <td>2020-06-02 22:24:55</td>\n",
       "      <td>2020-06-12 17:52:23</td>\n",
       "      <td>847649</td>\n",
       "      <td>1.0</td>\n",
       "      <td>electric</td>\n",
       "      <td>13.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>millivolts per kilometer</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>b</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>ey</td>\n",
       "      <td>2020-06-02 22:24:55</td>\n",
       "      <td>2020-06-12 17:52:23</td>\n",
       "      <td>847649</td>\n",
       "      <td>1.0</td>\n",
       "      <td>electric</td>\n",
       "      <td>103.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>millivolts per kilometer</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>b</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>hx</td>\n",
       "      <td>2020-06-02 22:24:55</td>\n",
       "      <td>2020-06-12 17:52:23</td>\n",
       "      <td>847649</td>\n",
       "      <td>1.0</td>\n",
       "      <td>magnetic</td>\n",
       "      <td>13.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nanotesla</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>b</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>hy</td>\n",
       "      <td>2020-06-02 22:24:55</td>\n",
       "      <td>2020-06-12 17:52:23</td>\n",
       "      <td>847649</td>\n",
       "      <td>1.0</td>\n",
       "      <td>magnetic</td>\n",
       "      <td>103.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nanotesla</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>b</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>hz</td>\n",
       "      <td>2020-06-02 22:24:55</td>\n",
       "      <td>2020-06-12 17:52:23</td>\n",
       "      <td>847649</td>\n",
       "      <td>1.0</td>\n",
       "      <td>magnetic</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nanotesla</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>c</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>ex</td>\n",
       "      <td>2020-06-12 18:32:17</td>\n",
       "      <td>2020-07-01 17:32:59</td>\n",
       "      <td>1638043</td>\n",
       "      <td>1.0</td>\n",
       "      <td>electric</td>\n",
       "      <td>13.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>millivolts per kilometer</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>c</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>ey</td>\n",
       "      <td>2020-06-12 18:32:17</td>\n",
       "      <td>2020-07-01 17:32:59</td>\n",
       "      <td>1638043</td>\n",
       "      <td>1.0</td>\n",
       "      <td>electric</td>\n",
       "      <td>103.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>millivolts per kilometer</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>c</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>hx</td>\n",
       "      <td>2020-06-12 18:32:17</td>\n",
       "      <td>2020-07-01 17:32:59</td>\n",
       "      <td>1638043</td>\n",
       "      <td>1.0</td>\n",
       "      <td>magnetic</td>\n",
       "      <td>13.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nanotesla</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>c</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>hy</td>\n",
       "      <td>2020-06-12 18:32:17</td>\n",
       "      <td>2020-07-01 17:32:59</td>\n",
       "      <td>1638043</td>\n",
       "      <td>1.0</td>\n",
       "      <td>magnetic</td>\n",
       "      <td>103.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nanotesla</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>c</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>hz</td>\n",
       "      <td>2020-06-12 18:32:17</td>\n",
       "      <td>2020-07-01 17:32:59</td>\n",
       "      <td>1638043</td>\n",
       "      <td>1.0</td>\n",
       "      <td>magnetic</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nanotesla</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>d</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>ex</td>\n",
       "      <td>2020-07-01 19:36:55</td>\n",
       "      <td>2020-07-13 19:00:00</td>\n",
       "      <td>1034586</td>\n",
       "      <td>1.0</td>\n",
       "      <td>electric</td>\n",
       "      <td>13.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>millivolts per kilometer</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>d</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>ey</td>\n",
       "      <td>2020-07-01 19:36:55</td>\n",
       "      <td>2020-07-13 19:00:00</td>\n",
       "      <td>1034586</td>\n",
       "      <td>1.0</td>\n",
       "      <td>electric</td>\n",
       "      <td>103.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>millivolts per kilometer</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>d</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>hx</td>\n",
       "      <td>2020-07-01 19:36:55</td>\n",
       "      <td>2020-07-13 19:00:00</td>\n",
       "      <td>1034586</td>\n",
       "      <td>1.0</td>\n",
       "      <td>magnetic</td>\n",
       "      <td>13.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nanotesla</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>d</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>hy</td>\n",
       "      <td>2020-07-01 19:36:55</td>\n",
       "      <td>2020-07-13 19:00:00</td>\n",
       "      <td>1034586</td>\n",
       "      <td>1.0</td>\n",
       "      <td>magnetic</td>\n",
       "      <td>103.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nanotesla</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>CAS04</td>\n",
       "      <td>d</td>\n",
       "      <td>37.633351</td>\n",
       "      <td>-121.468382</td>\n",
       "      <td>329.3875</td>\n",
       "      <td>hz</td>\n",
       "      <td>2020-07-01 19:36:55</td>\n",
       "      <td>2020-07-13 19:00:00</td>\n",
       "      <td>1034586</td>\n",
       "      <td>1.0</td>\n",
       "      <td>magnetic</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nanotesla</td>\n",
       "      <td>&lt;HDF5 object reference&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   station run   latitude   longitude  elevation component  \\\n",
       "0    CAS04   a  37.633351 -121.468382   329.3875        ex   \n",
       "1    CAS04   a  37.633351 -121.468382   329.3875        ey   \n",
       "2    CAS04   a  37.633351 -121.468382   329.3875        hx   \n",
       "3    CAS04   a  37.633351 -121.468382   329.3875        hy   \n",
       "4    CAS04   a  37.633351 -121.468382   329.3875        hz   \n",
       "5    CAS04   b  37.633351 -121.468382   329.3875        ex   \n",
       "6    CAS04   b  37.633351 -121.468382   329.3875        ey   \n",
       "7    CAS04   b  37.633351 -121.468382   329.3875        hx   \n",
       "8    CAS04   b  37.633351 -121.468382   329.3875        hy   \n",
       "9    CAS04   b  37.633351 -121.468382   329.3875        hz   \n",
       "10   CAS04   c  37.633351 -121.468382   329.3875        ex   \n",
       "11   CAS04   c  37.633351 -121.468382   329.3875        ey   \n",
       "12   CAS04   c  37.633351 -121.468382   329.3875        hx   \n",
       "13   CAS04   c  37.633351 -121.468382   329.3875        hy   \n",
       "14   CAS04   c  37.633351 -121.468382   329.3875        hz   \n",
       "15   CAS04   d  37.633351 -121.468382   329.3875        ex   \n",
       "16   CAS04   d  37.633351 -121.468382   329.3875        ey   \n",
       "17   CAS04   d  37.633351 -121.468382   329.3875        hx   \n",
       "18   CAS04   d  37.633351 -121.468382   329.3875        hy   \n",
       "19   CAS04   d  37.633351 -121.468382   329.3875        hz   \n",
       "\n",
       "                 start                 end  n_samples  sample_rate  \\\n",
       "0  2020-06-02 19:00:00 2020-06-02 22:07:46      11267          1.0   \n",
       "1  2020-06-02 19:00:00 2020-06-02 22:07:46      11267          1.0   \n",
       "2  2020-06-02 19:00:00 2020-06-02 22:07:46      11267          1.0   \n",
       "3  2020-06-02 19:00:00 2020-06-02 22:07:46      11267          1.0   \n",
       "4  2020-06-02 19:00:00 2020-06-02 22:07:46      11267          1.0   \n",
       "5  2020-06-02 22:24:55 2020-06-12 17:52:23     847649          1.0   \n",
       "6  2020-06-02 22:24:55 2020-06-12 17:52:23     847649          1.0   \n",
       "7  2020-06-02 22:24:55 2020-06-12 17:52:23     847649          1.0   \n",
       "8  2020-06-02 22:24:55 2020-06-12 17:52:23     847649          1.0   \n",
       "9  2020-06-02 22:24:55 2020-06-12 17:52:23     847649          1.0   \n",
       "10 2020-06-12 18:32:17 2020-07-01 17:32:59    1638043          1.0   \n",
       "11 2020-06-12 18:32:17 2020-07-01 17:32:59    1638043          1.0   \n",
       "12 2020-06-12 18:32:17 2020-07-01 17:32:59    1638043          1.0   \n",
       "13 2020-06-12 18:32:17 2020-07-01 17:32:59    1638043          1.0   \n",
       "14 2020-06-12 18:32:17 2020-07-01 17:32:59    1638043          1.0   \n",
       "15 2020-07-01 19:36:55 2020-07-13 19:00:00    1034586          1.0   \n",
       "16 2020-07-01 19:36:55 2020-07-13 19:00:00    1034586          1.0   \n",
       "17 2020-07-01 19:36:55 2020-07-13 19:00:00    1034586          1.0   \n",
       "18 2020-07-01 19:36:55 2020-07-13 19:00:00    1034586          1.0   \n",
       "19 2020-07-01 19:36:55 2020-07-13 19:00:00    1034586          1.0   \n",
       "\n",
       "   measurement_type  azimuth  tilt                     units  \\\n",
       "0          electric     13.2   0.0  millivolts per kilometer   \n",
       "1          electric    103.2   0.0  millivolts per kilometer   \n",
       "2          magnetic     13.2   0.0                 nanotesla   \n",
       "3          magnetic    103.2   0.0                 nanotesla   \n",
       "4          magnetic      0.0   0.0                 nanotesla   \n",
       "5          electric     13.2   0.0  millivolts per kilometer   \n",
       "6          electric    103.2   0.0  millivolts per kilometer   \n",
       "7          magnetic     13.2   0.0                 nanotesla   \n",
       "8          magnetic    103.2   0.0                 nanotesla   \n",
       "9          magnetic      0.0   0.0                 nanotesla   \n",
       "10         electric     13.2   0.0  millivolts per kilometer   \n",
       "11         electric    103.2   0.0  millivolts per kilometer   \n",
       "12         magnetic     13.2   0.0                 nanotesla   \n",
       "13         magnetic    103.2   0.0                 nanotesla   \n",
       "14         magnetic      0.0   0.0                 nanotesla   \n",
       "15         electric     13.2   0.0  millivolts per kilometer   \n",
       "16         electric    103.2   0.0  millivolts per kilometer   \n",
       "17         magnetic     13.2   0.0                 nanotesla   \n",
       "18         magnetic    103.2   0.0                 nanotesla   \n",
       "19         magnetic      0.0   0.0                 nanotesla   \n",
       "\n",
       "             hdf5_reference  \n",
       "0   <HDF5 object reference>  \n",
       "1   <HDF5 object reference>  \n",
       "2   <HDF5 object reference>  \n",
       "3   <HDF5 object reference>  \n",
       "4   <HDF5 object reference>  \n",
       "5   <HDF5 object reference>  \n",
       "6   <HDF5 object reference>  \n",
       "7   <HDF5 object reference>  \n",
       "8   <HDF5 object reference>  \n",
       "9   <HDF5 object reference>  \n",
       "10  <HDF5 object reference>  \n",
       "11  <HDF5 object reference>  \n",
       "12  <HDF5 object reference>  \n",
       "13  <HDF5 object reference>  \n",
       "14  <HDF5 object reference>  \n",
       "15  <HDF5 object reference>  \n",
       "16  <HDF5 object reference>  \n",
       "17  <HDF5 object reference>  \n",
       "18  <HDF5 object reference>  \n",
       "19  <HDF5 object reference>  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ch_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "available_runs = ch_summary.run.unique()\n",
    "sr = ch_summary.sample_rate.unique()\n",
    "if len(sr) != 1:\n",
    "    print('Only one sample rate per run is available')\n",
    "available_stations = ch_summary.station.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sr[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CAS04'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "available_stations[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "/:\n",
       "====================\n",
       "    |- Group: Survey\n",
       "    ----------------\n",
       "        |- Group: Filters\n",
       "        -----------------\n",
       "            |- Group: coefficient\n",
       "            ---------------------\n",
       "                |- Group: v to counts (electric)\n",
       "                --------------------------------\n",
       "                |- Group: v to counts (magnetic)\n",
       "                --------------------------------\n",
       "            |- Group: fap\n",
       "            -------------\n",
       "            |- Group: fir\n",
       "            -------------\n",
       "            |- Group: time_delay\n",
       "            --------------------\n",
       "                |- Group: electric time offset\n",
       "                ------------------------------\n",
       "                |- Group: hx time offset\n",
       "                ------------------------\n",
       "                |- Group: hy time offset\n",
       "                ------------------------\n",
       "                |- Group: hz time offset\n",
       "                ------------------------\n",
       "            |- Group: zpk\n",
       "            -------------\n",
       "                |- Group: electric field 1 pole butterworth high-pass\n",
       "                -----------------------------------------------------\n",
       "                    --> Dataset: poles\n",
       "                    ....................\n",
       "                    --> Dataset: zeros\n",
       "                    ....................\n",
       "                |- Group: electric field 5 pole butterworth low-pass\n",
       "                ----------------------------------------------------\n",
       "                    --> Dataset: poles\n",
       "                    ....................\n",
       "                    --> Dataset: zeros\n",
       "                    ....................\n",
       "                |- Group: magnetic field 3 pole butterworth low-pass\n",
       "                ----------------------------------------------------\n",
       "                    --> Dataset: poles\n",
       "                    ....................\n",
       "                    --> Dataset: zeros\n",
       "                    ....................\n",
       "                |- Group: mv per km to v per m\n",
       "                ------------------------------\n",
       "                    --> Dataset: poles\n",
       "                    ....................\n",
       "                    --> Dataset: zeros\n",
       "                    ....................\n",
       "                |- Group: v per m to v\n",
       "                ----------------------\n",
       "                    --> Dataset: poles\n",
       "                    ....................\n",
       "                    --> Dataset: zeros\n",
       "                    ....................\n",
       "        |- Group: Reports\n",
       "        -----------------\n",
       "        |- Group: Standards\n",
       "        -------------------\n",
       "            --> Dataset: summary\n",
       "            ......................\n",
       "        |- Group: Stations\n",
       "        ------------------\n",
       "            |- Group: CAS04\n",
       "            ---------------\n",
       "                |- Group: a\n",
       "                -----------\n",
       "                    --> Dataset: ex\n",
       "                    .................\n",
       "                    --> Dataset: ey\n",
       "                    .................\n",
       "                    --> Dataset: hx\n",
       "                    .................\n",
       "                    --> Dataset: hy\n",
       "                    .................\n",
       "                    --> Dataset: hz\n",
       "                    .................\n",
       "                |- Group: b\n",
       "                -----------\n",
       "                    --> Dataset: ex\n",
       "                    .................\n",
       "                    --> Dataset: ey\n",
       "                    .................\n",
       "                    --> Dataset: hx\n",
       "                    .................\n",
       "                    --> Dataset: hy\n",
       "                    .................\n",
       "                    --> Dataset: hz\n",
       "                    .................\n",
       "                |- Group: c\n",
       "                -----------\n",
       "                    --> Dataset: ex\n",
       "                    .................\n",
       "                    --> Dataset: ey\n",
       "                    .................\n",
       "                    --> Dataset: hx\n",
       "                    .................\n",
       "                    --> Dataset: hy\n",
       "                    .................\n",
       "                    --> Dataset: hz\n",
       "                    .................\n",
       "                |- Group: d\n",
       "                -----------\n",
       "                    --> Dataset: ex\n",
       "                    .................\n",
       "                    --> Dataset: ey\n",
       "                    .................\n",
       "                    --> Dataset: hx\n",
       "                    .................\n",
       "                    --> Dataset: hy\n",
       "                    .................\n",
       "                    --> Dataset: hz\n",
       "                    ................."
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mth5_object"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make a configuration file using the MTH5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 1, 2, 3]\n"
     ]
    }
   ],
   "source": [
    "station_id = available_stations[0]\n",
    "run_id = available_runs[0]\n",
    "sample_rate = sr[0]\n",
    "config_maker = ConfigCreator()\n",
    "config_path = config_maker.create_run_config(station_id, run_id, mth5_filename, sample_rate)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('config/CAS04-a_run_config.json')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run the Aurora Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_plot='True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_cls = process_mth5_run(\n",
    "        config_path,\n",
    "        run_id,\n",
    "        mth5_path=mth5_filename,\n",
    "        units=\"MT\",\n",
    "        show_plot=False,\n",
    "        z_file_path=None,\n",
    "        return_collection=False,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TF( station='CAS04', latitude=37.63, longitude=-121.47, elevation=329.39 )"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write the transfer functions to emtf emft and other output file types. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "    \"e_m_t_f_x_m_l\": {\n",
       "        \"description\": \"USMTArray South Magnetotelluric Time Series (USMTArray CONUS South-USGS)\",\n",
       "        \"product_id\": \"CAS04\",\n",
       "        \"sub_type\": \"MT_TF\",\n",
       "        \"tags\": \"impedance, tipper\"\n",
       "    }\n",
       "}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " tf_cls.write_tf_file(fn=\"emtfxml_test.xml\", file_type=\"emtfxml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_cls.write_tf_file(fn=\"emtfxml_test.xml\", file_type=\"edi\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " tf_cls.write_tf_file(fn=\"emtfxml_test.xml\", file_type=\"zmm\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}