From 728c75e1f12cece68041832bdb504a8b174aaec2 Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Tue, 15 Sep 2020 18:12:05 +0100 Subject: [PATCH] AUX_OBS Support (#51) * Add basic support for AUX_OBS * Add available_observatories query * Add support for AUX_OBSH * Fix available_collections with AUX_OBS * Add forgotten get_observatories template * Tweak docstring * Add AUX_OBS acknowledgement mechanism * Add AUX_OBS info to docs --- docs/available_parameters.rst | 22 +++ viresclient/_client_swarm.py | 184 ++++++++++++++++-- .../templates/vires_get_observatories.xml | 31 +++ 3 files changed, 216 insertions(+), 21 deletions(-) create mode 100644 viresclient/_wps/templates/vires_get_observatories.xml diff --git a/docs/available_parameters.rst b/docs/available_parameters.rst index 7b59198..df29106 100644 --- a/docs/available_parameters.rst +++ b/docs/available_parameters.rst @@ -63,6 +63,18 @@ SW_OPER_AEJxPBS_2F:GroundMagneticDisturbance AEJ_PBS:GroundMagneticDisturbance - SW_OPER_AOBxFAC_2F AOB_FAC Auroral oval boundaries derived from FACs ============================================ ================================= ============================================================== +The AUX_OBS collections contain ground magnetic observatory data from `INTERMAGNET `_ and `WDC `_. Please note that these data are provided under different usage terms than the ESA data, and must be acknowledged accordingly. + +======================== ================ ============================================================== +Collection full name Collection type Description +======================== ================ ============================================================== +SW_OPER_AUX_OBSH2\_ AUX_OBSH Hourly values derived from both WDC and INTERMAGNET data +SW_OPER_AUX_OBSM2\_ AUX_OBSM Minute values from INTERMAGNET +SW_OPER_AUX_OBSS2\_ AUX_OBSS Second values from INTERMAGNET +======================== ================ ============================================================== + +The AUX_OBS collections contain data from all observatories together (distinguishable by the ``IAGA_code`` variable). Data from a single observatory can be accessed with special collection names like ``SW_OPER_AUX_OBSM2_:ABK`` where ``ABK`` can be replaced with the IAGA code of the observatory. Use :py:meth:`viresclient.SwarmRequest.available_observatories` to find these IAGA codes. + The ``measurements``, ``models``, and ``auxiliaries`` chosen will match the cadence of the ``collection`` chosen. ---- @@ -100,6 +112,16 @@ AEJ_PBS:GroundMagneticDisturbance ``B_NE`` AOB_FAC ``Latitude_QD,Longitude_QD,MLT_QD,Boundary_Flag,Quality,Pair_Indicator`` ================================= ================================================================================ +AUX_OBS products: + +=============== ========================================= +Collection type Available measurement names +=============== ========================================= +AUX_OBSH ``B_NEC,F,IAGA_code,Quality,SensorIndex`` +AUX_OBSM ``B_NEC,F,IAGA_code,Quality`` +AUX_OBSS ``B_NEC,F,IAGA_code,Quality`` +=============== ========================================= + ---- ``models`` diff --git a/viresclient/_client_swarm.py b/viresclient/_client_swarm.py index 64baa5e..86243cd 100644 --- a/viresclient/_client_swarm.py +++ b/viresclient/_client_swarm.py @@ -3,6 +3,10 @@ from collections import OrderedDict import os import sys +from io import StringIO +from pandas import read_csv +from tqdm import tqdm +from textwrap import dedent from ._wps.environment import JINJA2_ENVIRONMENT from ._wps.time_util import parse_datetime @@ -15,7 +19,8 @@ 'sync': "vires_fetch_filtered_data.xml", 'async': "vires_fetch_filtered_data_async.xml", 'model_info': "vires_get_model_info.xml", - 'times_from_orbits': "vires_times_from_orbits.xml" + 'times_from_orbits': "vires_times_from_orbits.xml", + 'get_observatories': 'vires_get_observatories.xml' } REFERENCES = { @@ -152,9 +157,20 @@ "EEF": (" https://earth.esa.int/web/guest/missions/esa-eo-missions/swarm/data-handbook/level-2-product-definitions#EEFxTMS_2F ", " https://earth.esa.int/documents/10174/1514862/Swarm-Level-2-EEF-Product-Description "), "IPD": (" https://earth.esa.int/web/guest/missions/esa-eo-missions/swarm/data-handbook/level-2-product-definitions#IPDxIPR_2F ", - ) + ), + "AUX_OBSH": ("https://doi.org/10.5047/eps.2013.07.011",), + "AUX_OBSM": ("https://doi.org/10.5047/eps.2013.07.011",), + "AUX_OBSS": ("https://doi.org/10.5047/eps.2013.07.011",), +} + +DATA_CITATIONS = { + "AUX_OBSH": "ftp://ftp.nerc-murchison.ac.uk/geomag/Swarm/AUX_OBS/hour/README", + "AUX_OBSM": "ftp://ftp.nerc-murchison.ac.uk/geomag/Swarm/AUX_OBS/minute/README", + "AUX_OBSS": "ftp://ftp.nerc-murchison.ac.uk/geomag/Swarm/AUX_OBS/second/README", } +IAGA_CODES = ['AAA', 'AAE', 'ABG', 'ABK', 'AIA', 'ALE', 'AMS', 'API', 'AQU', 'ARS', 'ASC', 'ASP', 'BDV', 'BEL', 'BFE', 'BFO', 'BGY', 'BJN', 'BLC', 'BMT', 'BNG', 'BOU', 'BOX', 'BRD', 'BRW', 'BSL', 'CBB', 'CBI', 'CDP', 'CKI', 'CLF', 'CMO', 'CNB', 'CNH', 'COI', 'CPL', 'CSY', 'CTA', 'CTS', 'CYG', 'CZT', 'DED', 'DLR', 'DLT', 'DMC', 'DOB', 'DOU', 'DRV', 'DUR', 'EBR', 'ELT', 'ESA', 'ESK', 'EYR', 'FCC', 'FRD', 'FRN', 'FUQ', 'FUR', 'GAN', 'GCK', 'GDH', 'GLM', 'GLN', 'GNA', 'GNG', 'GUA', 'GUI', 'GZH', 'HAD', 'HBK', 'HER', 'HLP', 'HON', 'HRB', 'HRN', 'HUA', 'HYB', 'IPM', 'IQA', 'IRT', 'IZN', 'JAI', 'JCO', 'KAK', 'KDU', 'KEP', 'KHB', 'KIR', 'KIV', 'KMH', 'KNY', 'KNZ', 'KOU', 'KSH', 'LER', 'LIV', 'LMM', 'LNP', 'LON', 'LOV', 'LRM', 'LRV', 'LVV', 'LYC', 'LZH', 'MAB', 'MAW', 'MBC', 'MBO', 'MCQ', 'MEA', 'MGD', 'MID', 'MIZ', 'MMB', 'MZL', 'NAQ', 'NCK', 'NEW', 'NGK', 'NGP', 'NMP', 'NUR', 'NVS', 'ORC', 'OTT', 'PAF', 'PAG', 'PBQ', 'PEG', 'PET', 'PHU', 'PIL', 'PND', 'PPT', 'PST', 'QGZ', 'QIX', 'QSB', 'QZH', 'RES', 'SBA', 'SBL', 'SFS', 'SHE', 'SHL', 'SHU', 'SIL', 'SIT', 'SJG', 'SOD', 'SPG', 'SPT', 'STJ', 'SUA', 'TAM', 'TAN', 'TDC', 'TEO', 'THJ', 'THL', 'THY', 'TIR', 'TND', 'TRO', 'TRW', 'TSU', 'TUC', 'UPS', 'VAL', 'VIC', 'VNA', 'VOS', 'VSK', 'VSS', 'WHN', 'WIC', 'WIK', 'WNG', 'YAK', 'YKC'] + class SwarmWPSInputs(WPSInputs): """Holds the set of inputs to be passed to the request template for Swarm @@ -209,11 +225,17 @@ def collection_ids(self, collection_ids): @staticmethod def _spacecraft_from_collection(collection): - """Identify spacecraft from collection name.""" - # 12th character in name, e.g. SW_OPER_MAGx_LR_1B - sc = collection[11] - sc_to_name = {"A": "Alpha", "B": "Bravo", "C": "Charlie", "_": "NSC"} - return sc_to_name[sc] + """Identify spacecraft (or ground observatory name) from collection name.""" + if "AUX_OBS" in collection: + name = "AUX_OBS" + if ":" in collection: + name = f"{name}:{collection[19:22]}" + else: + # 12th character in name, e.g. SW_OPER_MAGx_LR_1B + sc = collection[11] + sc_to_name = {"A": "Alpha", "B": "Bravo", "C": "Charlie", "_": "NSC"} + name = sc_to_name[sc] + return name def set_collections(self, collections): """Restructure given list of collections as dict required by VirES.""" @@ -388,7 +410,19 @@ class SwarmRequest(ClientRequest): "SW_OPER_AEJ{}PBS_2F:GroundMagneticDisturbance".format(x) for x in "ABC" ], "AOB_FAC": ["SW_OPER_AOB{}FAC_2F".format(x) for x in "ABC"], - } + "AUX_OBSH": [ + "SW_OPER_AUX_OBSH2_", + *[f"SW_OPER_AUX_OBSH2_:{code}" for code in IAGA_CODES] + ], + "AUX_OBSM": [ + "SW_OPER_AUX_OBSM2_", + *[f"SW_OPER_AUX_OBSM2_:{code}" for code in IAGA_CODES] + ], + "AUX_OBSS": [ + "SW_OPER_AUX_OBSS2_", + *[f"SW_OPER_AUX_OBSS2_:{code}" for code in IAGA_CODES] + ] + } # These are not necessarily real sampling steps, but are good enough to use # for splitting long requests into chunks @@ -403,6 +437,9 @@ class SwarmRequest(ClientRequest): "IPD": "PT1S", "AEJ_LPL": "PT15.6S", "AEJ_LPS": "PT1S", + "AUX_OBSH": "PT60M", + "AUX_OBSM": "PT60S", + "AUX_OBSS": "PT1S" } PRODUCT_VARIABLES = { @@ -464,7 +501,10 @@ class SwarmRequest(ClientRequest): "Latitude_QD", "Longitude_QD", "MLT_QD", "Boundary_Flag", "Quality", "Pair_Indicator" ], - } + "AUX_OBSH": ["B_NEC", "F", "IAGA_code", "Quality", "SensorIndex"], + "AUX_OBSM": ["B_NEC", "F", "IAGA_code", "Quality"], + "AUX_OBSS": ["B_NEC", "F", "IAGA_code", "Quality"], + } AUXILIARY_VARIABLES = [ "Timestamp", "Latitude", "Longitude", "Radius", "Spacecraft", @@ -475,7 +515,7 @@ class SwarmRequest(ClientRequest): "SunRightAscension", "SunAzimuthAngle", "SunZenithAngle", "SunLongitude", "SunVector", "DipoleAxisVector", "NGPLatitude", "NGPLongitude", "DipoleTiltAngle", - ] + ] MAGNETIC_MODEL_VARIABLES = ["F", "B_NEC"] @@ -490,7 +530,7 @@ class SwarmRequest(ClientRequest): "MIO_SHA_2D-Primary", "MIO_SHA_2D-Secondary", "AMPS", "MCO_SHA_2X", "CHAOS", "CHAOS-MMA", "MMA_SHA_2C", "MMA_SHA_2F", "MIO_SHA_2C", "MIO_SHA_2D", "SwarmCI", - ] + ] def __init__(self, url=None, username=None, password=None, token=None, config=None, logging_level="NO_LOGGING"): @@ -582,17 +622,27 @@ def available_collections(self, groupname=None, details=True): If False then return a dict of available collections. """ + # Shorter form of the available collections + collections_short = self._available["collections"].copy() + collections_short["AUX_OBSS"] = ['SW_OPER_AUX_OBSS2_'] + collections_short["AUX_OBSM"] = ['SW_OPER_AUX_OBSM2_'] + collections_short["AUX_OBSH"] = ['SW_OPER_AUX_OBSH2_'] + def _filter_collections(groupname): - groups = list(self._available["collections"].keys()) - if groupname in groups: - return {groupname: - self._available["collections"][groupname]} + """ Reduce the full list to just one group, e.g. "MAG """ + if groupname: + groups = list(collections_short.keys()) + if groupname in groups: + return { + groupname: + collections_short[groupname] + } + else: + raise ValueError("Invalid collection group name") else: - raise ValueError("Invalid collection group name") - if groupname: - collections_filtered = _filter_collections(groupname) - else: - collections_filtered = self._available["collections"] + return collections_short + + collections_filtered = _filter_collections(groupname) if details: print("General References:") for i in REFERENCES["General Swarm"]: @@ -705,6 +755,98 @@ def available_auxiliaries(self): """ return self._available["auxiliaries"] + def available_observatories( + self, collection=None, start_time=None, end_time=None, details=False + ): + """Get list of available observatories from server. + + Search availability by collection, one of:: + + "SW_OPER_AUX_OBSH2_" + "SW_OPER_AUX_OBSM2_" + "SW_OPER_AUX_OBSS2_" + + Example usage:: + + from viresclient import SwarmRequest + request = SwarmRequest() + # For a list of observatories available: + request.available_observatories("SW_OPER_AUX_OBSM2_") + # For a DataFrame also containing availability start and end times: + request.available_observatories("SW_OPER_AUX_OBSM2_", details=True) + # For available observatories during a given time period: + request.available_observatories( + "SW_OPER_AUX_OBSM2_", "2013-01-01", "2013-02-01" + ) + + Args: + collection (str): collection name (e.g. `"SW_OPER_AUX_OBSM2_"`) + custom_model (str): as with set_products + details (bool): returns DataFrame if True + + Returns: + list or DataFrame: IAGA codes (and start/end times) + + """ + def _request_get_observatories(collection=None, start_time=None, end_time=None): + """ Make the get_observatories request to the server """ + templatefile = TEMPLATE_FILES["get_observatories"] + template = JINJA2_ENVIRONMENT.get_template(templatefile) + request = template.render( + collection_id=collection, + begin_time=start_time, + end_time=end_time, + response_type="text/csv" + ).encode('UTF-8') + response = self._get( + request, asynchronous=False, show_progress=False + ) + return response + + def _csv_to_df(csv_data): + """ Convert bytes data to pandas dataframe """ + return read_csv( + StringIO(str(csv_data, 'utf-8')) + ) + + obs_collections = [ + "SW_OPER_AUX_OBSH2_", + "SW_OPER_AUX_OBSM2_", + "SW_OPER_AUX_OBSS2_" + ] + if collection not in obs_collections: + raise ValueError(f"Invalid collection: {collection}") + if start_time and end_time: + start_time = parse_datetime(start_time) + end_time = parse_datetime(end_time) + else: + start_time, end_time = None, None + + response = _request_get_observatories(collection, start_time, end_time) + df = _csv_to_df(response) + if details: + return df + else: + return list(df["IAGACode"]) + + def _detect_AUX_OBS(self, collections): + # Identify collection types present + collection_types_requested = { + self._available["collections_to_keys"].get(collection) + for collection in collections + } + # Output notification for each of aux_type + for aux_type in ["AUX_OBSH", "AUX_OBSM", "AUX_OBSS"]: + if aux_type in collection_types_requested: + tqdm.write( + dedent( + f""" + Accessing INTERMAGNET and/or WDC data + Check usage terms at {DATA_CITATIONS.get(aux_type)} + """ + ) + ) + def set_collection(self, *args): """Set the collection(s) to use. @@ -719,13 +861,13 @@ def set_collection(self, *args): "{} invalid. Must be string." .format(collection) ) - for collection in collections: if collection not in self._available["collections_to_keys"]: raise ValueError( "Invalid collection: {}. " "Check available with SwarmRequest().available_collections()" .format(collection) ) + self._detect_AUX_OBS(collections) self._collection_list = collections self._request_inputs.set_collections(collections) return self diff --git a/viresclient/_wps/templates/vires_get_observatories.xml b/viresclient/_wps/templates/vires_get_observatories.xml new file mode 100644 index 0000000..21977ee --- /dev/null +++ b/viresclient/_wps/templates/vires_get_observatories.xml @@ -0,0 +1,31 @@ + + + vires:get_observatories + + + collection_id + + {{ collection_id }} + + + {% if begin_time -%} + + begin_time + + {{ begin_time| d2s }} + + + + end_time + + {{ end_time|d2s }} + + + {% endif -%} + + + + output + + + \ No newline at end of file