From cfd7ea190128688a806b4738cd96fa1a77d6694a Mon Sep 17 00:00:00 2001 From: Martin Paces Date: Sat, 23 Jan 2021 12:45:41 +0100 Subject: [PATCH 01/15] Various fixes needed to support the long-term VOBS data collections. --- viresclient/_client.py | 7 +++++-- viresclient/_client_swarm.py | 24 ++++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/viresclient/_client.py b/viresclient/_client.py index 2b58172..075a367 100644 --- a/viresclient/_client.py +++ b/viresclient/_client.py @@ -77,6 +77,9 @@ # Maximum selectable time interval ~25 years MAX_TIME_SELECTION = timedelta(days=25*365.25) +# Maximum time-chunk size ~50 years +MAX_CHUNK_DURATION = 2 * MAX_TIME_SELECTION + TEMPLATE_FILES = { 'list_jobs': "vires_list_jobs.xml" @@ -392,9 +395,9 @@ def _chunkify_request(start_time, end_time, sampling_step, nrecords_limit): e.g. [(start1, end1), (start2, end2)] """ # maximum chunk duration as a timedelta object - chunk_duration = timedelta(seconds=( + chunk_duration = min(timedelta(seconds=( nrecords_limit * parse_duration(sampling_step).total_seconds() - )) + )), MAX_CHUNK_DURATION) # calculate the chunk intervals ... request_intervals = [] diff --git a/viresclient/_client_swarm.py b/viresclient/_client_swarm.py index de15fbd..540b3c3 100644 --- a/viresclient/_client_swarm.py +++ b/viresclient/_client_swarm.py @@ -233,8 +233,8 @@ def _spacecraft_from_collection(collection): else: # 12th character in name, e.g. SW_OPER_MAGx_LR_1B sc = collection[11] - sc_to_name = {"A": "Alpha", "B": "Bravo", "C": "Charlie", "_": "NSC"} - name = sc_to_name[sc] + sc_to_name = {"A": "Alpha", "B": "Bravo", "C": "Charlie"} + name = sc_to_name.get(sc, "NSC") return name def set_collections(self, collections): @@ -424,6 +424,13 @@ class SwarmRequest(ClientRequest): ] } + OBS_COLLECTIONS = [ + "SW_OPER_AUX_OBSH2_", + "SW_OPER_AUX_OBSM2_", + "SW_OPER_AUX_OBSS2_" + ] + + # These are not necessarily real sampling steps, but are good enough to use # for splitting long requests into chunks COLLECTION_SAMPLING_STEPS = { @@ -811,14 +818,9 @@ def _csv_to_df(csv_data): StringIO(str(csv_data, 'utf-8')) ) - obs_collections = [ - "SW_OPER_AUX_OBSH2_", - "SW_OPER_AUX_OBSM2_", - "SW_OPER_AUX_OBSS2_" - ] - if collection not in obs_collections: + if collection not in self.OBS_COLLECTIONS: raise ValueError( - f"Invalid collection: {collection}. Must be one of: {obs_collections}." + f"Invalid collection: {collection}. Must be one of: {self.OBS_COLLECTIONS}." ) if start_time and end_time: start_time = parse_datetime(start_time) @@ -832,7 +834,9 @@ def _csv_to_df(csv_data): if details: return df else: - return list(df["IAGACode"]) + # note: "IAGACode" has been renamed to "site" in VirES 3.5 + key = "IAGACode" if "IAGACode" in df.keys() else "site" + return list(df[key]) def _detect_AUX_OBS(self, collections): # Identify collection types present From 0e9b260356ac0ca188c9b79ee51e2a97cd2ae997 Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Mon, 25 Jan 2021 22:53:30 +0000 Subject: [PATCH 02/15] Add config for VOBS & .as_xarray(reshape=True) opt --- viresclient/__init__.py | 3 +- viresclient/_client_swarm.py | 63 +++++++++++++++++++++++++---- viresclient/_data/__init__.py | 5 +++ viresclient/_data/config_swarm.json | 5 +++ viresclient/_data_handling.py | 59 ++++++++++++++++++++++++--- 5 files changed, 120 insertions(+), 15 deletions(-) create mode 100644 viresclient/_data/__init__.py create mode 100644 viresclient/_data/config_swarm.json diff --git a/viresclient/__init__.py b/viresclient/__init__.py index bfae4ed..779eac7 100644 --- a/viresclient/__init__.py +++ b/viresclient/__init__.py @@ -34,5 +34,6 @@ from ._data_handling import ReturnedDataFile from ._api.upload import DataUpload from ._api.token import TokenManager +from . import _data -__version__ = "0.7.1" +__version__ = "0.8.0-alpha" diff --git a/viresclient/_client_swarm.py b/viresclient/_client_swarm.py index 540b3c3..95e6234 100644 --- a/viresclient/_client_swarm.py +++ b/viresclient/_client_swarm.py @@ -12,7 +12,10 @@ from ._wps.time_util import parse_datetime from ._client import WPSInputs, ClientRequest, TEMPLATE_FILES from ._data_handling import ReturnedDataFile +from ._data import CONFIG_SWARM +with open(CONFIG_SWARM, "r") as f: + CONFIG_SWARM = json.load(f) TEMPLATE_FILES = { **TEMPLATE_FILES, @@ -161,6 +164,7 @@ "AUX_OBSH": ("https://doi.org/10.5047/eps.2013.07.011",), "AUX_OBSM": ("https://doi.org/10.5047/eps.2013.07.011",), "AUX_OBSS": ("https://doi.org/10.5047/eps.2013.07.011",), + "VOBS_SW_1M": ("https://www.space.dtu.dk/english/research/projects/project-descriptions/geomagnetic-virtual-observatories",), } DATA_CITATIONS = { @@ -169,8 +173,10 @@ "AUX_OBSS": "ftp://ftp.nerc-murchison.ac.uk/geomag/Swarm/AUX_OBS/second/README", } -IAGA_CODES = ['AAA', 'AAE', 'ABG', 'ABK', 'AIA', 'ALE', 'AMS', 'API', 'AQU', 'ARS', 'ASC', 'ASP', 'BDV', 'BEL', 'BFE', 'BFO', 'BGY', 'BJN', 'BLC', 'BMT', 'BNG', 'BOU', 'BOX', 'BRD', 'BRW', 'BSL', 'CBB', 'CBI', 'CDP', 'CKI', 'CLF', 'CMO', 'CNB', 'CNH', 'COI', 'CPL', 'CSY', 'CTA', 'CTS', 'CYG', 'CZT', 'DED', 'DLR', 'DLT', 'DMC', 'DOB', 'DOU', 'DRV', 'DUR', 'EBR', 'ELT', 'ESA', 'ESK', 'EYR', 'FCC', 'FRD', 'FRN', 'FUQ', 'FUR', 'GAN', 'GCK', 'GDH', 'GLM', 'GLN', 'GNA', 'GNG', 'GUA', 'GUI', 'GZH', 'HAD', 'HBK', 'HER', 'HLP', 'HON', 'HRB', 'HRN', 'HUA', 'HYB', 'IPM', 'IQA', 'IRT', 'IZN', 'JAI', 'JCO', 'KAK', 'KDU', 'KEP', 'KHB', 'KIR', 'KIV', 'KMH', 'KNY', 'KNZ', 'KOU', 'KSH', 'LER', 'LIV', 'LMM', 'LNP', 'LON', 'LOV', 'LRM', 'LRV', 'LVV', 'LYC', 'LZH', 'MAB', 'MAW', 'MBC', 'MBO', 'MCQ', 'MEA', 'MGD', 'MID', 'MIZ', 'MMB', 'MZL', 'NAQ', 'NCK', 'NEW', 'NGK', 'NGP', 'NMP', 'NUR', 'NVS', 'ORC', 'OTT', 'PAF', 'PAG', 'PBQ', 'PEG', 'PET', 'PHU', 'PIL', 'PND', 'PPT', 'PST', 'QGZ', 'QIX', 'QSB', 'QZH', 'RES', 'SBA', 'SBL', 'SFS', 'SHE', 'SHL', 'SHU', 'SIL', 'SIT', 'SJG', 'SOD', 'SPG', 'SPT', 'STJ', 'SUA', 'TAM', 'TAN', 'TDC', 'TEO', 'THJ', 'THL', 'THY', 'TIR', 'TND', 'TRO', 'TRW', 'TSU', 'TUC', 'UPS', 'VAL', 'VIC', 'VNA', 'VOS', 'VSK', 'VSS', 'WHN', 'WIC', 'WIK', 'WNG', 'YAK', 'YKC'] +# IAGA_CODES = ['AAA', 'AAE', 'ABG', 'ABK', 'AIA', 'ALE', 'AMS', 'API', 'AQU', 'ARS', 'ASC', 'ASP', 'BDV', 'BEL', 'BFE', 'BFO', 'BGY', 'BJN', 'BLC', 'BMT', 'BNG', 'BOU', 'BOX', 'BRD', 'BRW', 'BSL', 'CBB', 'CBI', 'CDP', 'CKI', 'CLF', 'CMO', 'CNB', 'CNH', 'COI', 'CPL', 'CSY', 'CTA', 'CTS', 'CYG', 'CZT', 'DED', 'DLR', 'DLT', 'DMC', 'DOB', 'DOU', 'DRV', 'DUR', 'EBR', 'ELT', 'ESA', 'ESK', 'EYR', 'FCC', 'FRD', 'FRN', 'FUQ', 'FUR', 'GAN', 'GCK', 'GDH', 'GLM', 'GLN', 'GNA', 'GNG', 'GUA', 'GUI', 'GZH', 'HAD', 'HBK', 'HER', 'HLP', 'HON', 'HRB', 'HRN', 'HUA', 'HYB', 'IPM', 'IQA', 'IRT', 'IZN', 'JAI', 'JCO', 'KAK', 'KDU', 'KEP', 'KHB', 'KIR', 'KIV', 'KMH', 'KNY', 'KNZ', 'KOU', 'KSH', 'LER', 'LIV', 'LMM', 'LNP', 'LON', 'LOV', 'LRM', 'LRV', 'LVV', 'LYC', 'LZH', 'MAB', 'MAW', 'MBC', 'MBO', 'MCQ', 'MEA', 'MGD', 'MID', 'MIZ', 'MMB', 'MZL', 'NAQ', 'NCK', 'NEW', 'NGK', 'NGP', 'NMP', 'NUR', 'NVS', 'ORC', 'OTT', 'PAF', 'PAG', 'PBQ', 'PEG', 'PET', 'PHU', 'PIL', 'PND', 'PPT', 'PST', 'QGZ', 'QIX', 'QSB', 'QZH', 'RES', 'SBA', 'SBL', 'SFS', 'SHE', 'SHL', 'SHU', 'SIL', 'SIT', 'SJG', 'SOD', 'SPG', 'SPT', 'STJ', 'SUA', 'TAM', 'TAN', 'TDC', 'TEO', 'THJ', 'THL', 'THY', 'TIR', 'TND', 'TRO', 'TRW', 'TSU', 'TUC', 'UPS', 'VAL', 'VIC', 'VNA', 'VOS', 'VSK', 'VSS', 'WHN', 'WIC', 'WIK', 'WNG', 'YAK', 'YKC'] +IAGA_CODES = CONFIG_SWARM.get("IAGA_CODES") +VOBS_SITES = CONFIG_SWARM.get("VOBS_SITES") class SwarmWPSInputs(WPSInputs): """Holds the set of inputs to be passed to the request template for Swarm @@ -421,13 +427,38 @@ class SwarmRequest(ClientRequest): "AUX_OBSS": [ "SW_OPER_AUX_OBSS2_", *[f"SW_OPER_AUX_OBSS2_:{code}" for code in IAGA_CODES] - ] + ], + "VOBS_SW_1M": [ + "SW_OPER_VOBS_1M_2_", + *[f"SW_OPER_VOBS_1M_2_:{site}" for site in VOBS_SITES] + ], + "VOBS_SW_4M": [ + "SW_OPER_VOBS_4M_2_", + *[f"SW_OPER_VOBS_4M_2_:{site}" for site in VOBS_SITES] + ], + "VOBS_CH_1M": [ + "CH_OPER_VOBS_1M_2_", + *[f"CH_OPER_VOBS_1M_2_:{site}" for site in VOBS_SITES] + ], + "VOBS_CH_4M": [ + "CH_OPER_VOBS_4M_2_", + *[f"CH_OPER_VOBS_4M_2_:{site}" for site in VOBS_SITES] + ], + "VOBS_CR_4M": [ + "CR_OPER_VOBS_4M_2_", + *[f"CR_OPER_VOBS_4M_2_:{site}" for site in VOBS_SITES] + ], } OBS_COLLECTIONS = [ "SW_OPER_AUX_OBSH2_", "SW_OPER_AUX_OBSM2_", - "SW_OPER_AUX_OBSS2_" + "SW_OPER_AUX_OBSS2_", + "SW_OPER_VOBS_1M_2_", + "SW_OPER_VOBS_4M_2_", + "CH_OPER_VOBS_1M_2_", + "CH_OPER_VOBS_4M_2_", + "CR_OPER_VOBS_4M_2_", ] @@ -446,7 +477,12 @@ class SwarmRequest(ClientRequest): "AEJ_LPS": "PT1S", "AUX_OBSH": "PT60M", "AUX_OBSM": "PT60S", - "AUX_OBSS": "PT1S" + "AUX_OBSS": "PT1S", + "VOBS_SW_1M": "P31D", + "VOBS_CH_1M": "P31D", + "VOBS_SW_4M": "P122D", + "VOBS_CH_4M": "P122D", + "VOBS_CR_4M": "P122D", } PRODUCT_VARIABLES = { @@ -511,6 +547,11 @@ class SwarmRequest(ClientRequest): "AUX_OBSH": ["B_NEC", "F", "IAGA_code", "Quality", "SensorIndex"], "AUX_OBSM": ["B_NEC", "F", "IAGA_code", "Quality"], "AUX_OBSS": ["B_NEC", "F", "IAGA_code", "Quality"], + "VOBS_SW_1M": ["SiteCode", "B_CF", "B_OB", "B_SV", "sigma_CF", "sigma_OB", "sigma_SV"], + "VOBS_CH_1M": ["SiteCode", "B_CF", "B_OB", "B_SV", "sigma_CF", "sigma_OB", "sigma_SV"], + "VOBS_SW_4M": ["SiteCode", "B_CF", "B_OB", "B_SV", "sigma_CF", "sigma_OB", "sigma_SV"], + "VOBS_CH_4M": ["SiteCode", "B_CF", "B_OB", "B_SV", "sigma_CF", "sigma_OB", "sigma_SV"], + "VOBS_CR_4M": ["SiteCode", "B_CF", "B_OB", "B_SV", "sigma_CF", "sigma_OB", "sigma_SV"], } AUXILIARY_VARIABLES = [ @@ -629,11 +670,17 @@ def available_collections(self, groupname=None, details=True): If False then return a dict of available collections. """ - # Shorter form of the available collections + # Shorter form of the available collections, + # without all the individual SiteCodes collections_short = self._available["collections"].copy() - collections_short["AUX_OBSS"] = ['SW_OPER_AUX_OBSS2_'] - collections_short["AUX_OBSM"] = ['SW_OPER_AUX_OBSM2_'] - collections_short["AUX_OBSH"] = ['SW_OPER_AUX_OBSH2_'] + collections_short["AUX_OBSS"] = ["SW_OPER_AUX_OBSS2_"] + collections_short["AUX_OBSM"] = ["SW_OPER_AUX_OBSM2_"] + collections_short["AUX_OBSH"] = ["SW_OPER_AUX_OBSH2_"] + collections_short["VOBS_SW_1M"] = ["SW_OPER_VOBS_1M_2_"] + collections_short["VOBS_SW_4M"] = ["SW_OPER_VOBS_4M_2_"] + collections_short["VOBS_CH_1M"] = ["CH_OPER_VOBS_1M_2_"] + collections_short["VOBS_CH_4M"] = ["CH_OPER_VOBS_4M_2_"] + collections_short["VOBS_CR_4M"] = ["CR_OPER_VOBS_4M_2_"] def _filter_collections(groupname): """ Reduce the full list to just one group, e.g. "MAG """ diff --git a/viresclient/_data/__init__.py b/viresclient/_data/__init__.py new file mode 100644 index 0000000..483d691 --- /dev/null +++ b/viresclient/_data/__init__.py @@ -0,0 +1,5 @@ +from os.path import join, dirname + +_DIRNAME = dirname(__file__) + +CONFIG_SWARM = join(_DIRNAME, "config_swarm.json") diff --git a/viresclient/_data/config_swarm.json b/viresclient/_data/config_swarm.json new file mode 100644 index 0000000..ea56cdf --- /dev/null +++ b/viresclient/_data/config_swarm.json @@ -0,0 +1,5 @@ +{ + "IAGA_CODES": ["AAA", "AAE", "ABG", "ABK", "AIA", "ALE", "AMS", "API", "AQU", "ARS", "ASC", "ASP", "BDV", "BEL", "BFE", "BFO", "BGY", "BJN", "BLC", "BMT", "BNG", "BOU", "BOX", "BRD", "BRW", "BSL", "CBB", "CBI", "CDP", "CKI", "CLF", "CMO", "CNB", "CNH", "COI", "CPL", "CSY", "CTA", "CTS", "CYG", "CZT", "DED", "DLR", "DLT", "DMC", "DOB", "DOU", "DRV", "DUR", "EBR", "ELT", "ESA", "ESK", "EYR", "FCC", "FRD", "FRN", "FUQ", "FUR", "GAN", "GCK", "GDH", "GLM", "GLN", "GNA", "GNG", "GUA", "GUI", "GZH", "HAD", "HBK", "HER", "HLP", "HON", "HRB", "HRN", "HUA", "HYB", "IPM", "IQA", "IRT", "IZN", "JAI", "JCO", "KAK", "KDU", "KEP", "KHB", "KIR", "KIV", "KMH", "KNY", "KNZ", "KOU", "KSH", "LER", "LIV", "LMM", "LNP", "LON", "LOV", "LRM", "LRV", "LVV", "LYC", "LZH", "MAB", "MAW", "MBC", "MBO", "MCQ", "MEA", "MGD", "MID", "MIZ", "MMB", "MZL", "NAQ", "NCK", "NEW", "NGK", "NGP", "NMP", "NUR", "NVS", "ORC", "OTT", "PAF", "PAG", "PBQ", "PEG", "PET", "PHU", "PIL", "PND", "PPT", "PST", "QGZ", "QIX", "QSB", "QZH", "RES", "SBA", "SBL", "SFS", "SHE", "SHL", "SHU", "SIL", "SIT", "SJG", "SOD", "SPG", "SPT", "STJ", "SUA", "TAM", "TAN", "TDC", "TEO", "THJ", "THL", "THY", "TIR", "TND", "TRO", "TRW", "TSU", "TUC", "UPS", "VAL", "VIC", "VNA", "VOS", "VSK", "VSS", "WHN", "WIC", "WIK", "WNG", "YAK", "YKC"], + "VOBS_SITES": ["N90E000", "N77W026", "N77W077", "N77W129", "N77E026", "N77E077", "N77E129", "N77E180", "N65W024", "N65W051", "N65W079", "N65W107", "N65W135", "N65W162", "N65E004", "N65E032", "N65E059", "N65E087", "N65E115", "N65E142", "N65E170", "N54W003", "N54W023", "N54W043", "N54W063", "N54W083", "N54W103", "N54W123", "N54W143", "N54W163", "N54E017", "N54E037", "N54E057", "N54E077", "N54E097", "N54E117", "N54E137", "N54E157", "N54E177", "N42W007", "N42W023", "N42W038", "N42W054", "N42W069", "N42W085", "N42W101", "N42W116", "N42W132", "N42W148", "N42W163", "N42W179", "N42E009", "N42E024", "N42E040", "N42E056", "N42E071", "N42E087", "N42E103", "N42E118", "N42E134", "N42E150", "N42E165", "N30W009", "N30W022", "N30W036", "N30W049", "N30W062", "N30W076", "N30W089", "N30W102", "N30W116", "N30W129", "N30W142", "N30W156", "N30W169", "N30E004", "N30E018", "N30E031", "N30E044", "N30E058", "N30E071", "N30E084", "N30E098", "N30E111", "N30E124", "N30E138", "N30E151", "N30E164", "N30E178", "N18W010", "N18W022", "N18W034", "N18W046", "N18W058", "N18W070", "N18W082", "N18W094", "N18W106", "N18W118", "N18W130", "N18W142", "N18W154", "N18W166", "N18W178", "N18E002", "N18E014", "N18E026", "N18E038", "N18E050", "N18E062", "N18E074", "N18E086", "N18E098", "N18E110", "N18E122", "N18E134", "N18E146", "N18E158", "N18E170", "N06W010", "N06W021", "N06W033", "N06W045", "N06W056", "N06W068", "N06W079", "N06W091", "N06W103", "N06W114", "N06W126", "N06W138", "N06W149", "N06W161", "N06W172", "N06E002", "N06E013", "N06E025", "N06E037", "N06E048", "N06E060", "N06E072", "N06E083", "N06E095", "N06E106", "N06E118", "N06E130", "N06E141", "N06E153", "N06E164", "N06E176", "S06W004", "S06W016", "S06W027", "S06W039", "S06W050", "S06W062", "S06W074", "S06W085", "S06W097", "S06W108", "S06W120", "S06W132", "S06W143", "S06W155", "S06W167", "S06W178", "S06E008", "S06E019", "S06E031", "S06E042", "S06E054", "S06E066", "S06E077", "S06E089", "S06E101", "S06E112", "S06E124", "S06E135", "S06E147", "S06E159", "S06E170", "S18W003", "S18W015", "S18W027", "S18W039", "S18W051", "S18W063", "S18W075", "S18W087", "S18W099", "S18W111", "S18W123", "S18W135", "S18W147", "S18W159", "S18W171", "S18E009", "S18E021", "S18E033", "S18E045", "S18E057", "S18E069", "S18E081", "S18E093", "S18E105", "S18E117", "S18E129", "S18E141", "S18E153", "S18E165", "S18E177", "S30W001", "S30W015", "S30W028", "S30W041", "S30W055", "S30W068", "S30W081", "S30W095", "S30W108", "S30W121", "S30W135", "S30W148", "S30W161", "S30W175", "S30E012", "S30E025", "S30E039", "S30E052", "S30E065", "S30E079", "S30E092", "S30E105", "S30E119", "S30E132", "S30E145", "S30E159", "S30E172", "S42W014", "S42W030", "S42W046", "S42W061", "S42W077", "S42W093", "S42W108", "S42W124", "S42W140", "S42W155", "S42W171", "S42E001", "S42E017", "S42E033", "S42E048", "S42E064", "S42E079", "S42E095", "S42E111", "S42E126", "S42E142", "S42E158", "S42E173", "S54W014", "S54W034", "S54W054", "S54W074", "S54W094", "S54W114", "S54W134", "S54W154", "S54W174", "S54E006", "S54E026", "S54E046", "S54E066", "S54E086", "S54E106", "S54E126", "S54E146", "S54E166", "S65W041", "S65W069", "S65W096", "S65W124", "S65W152", "S65W179", "S65E014", "S65E042", "S65E070", "S65E098", "S65E125", "S65E153", "S65W013", "S77W063", "S77W114", "S77W166", "S77E040", "S77E092", "S77E143", "S77W011", "S90E000"] +} + diff --git a/viresclient/_data_handling.py b/viresclient/_data_handling.py index 18ae723..363fd9b 100644 --- a/viresclient/_data_handling.py +++ b/viresclient/_data_handling.py @@ -42,7 +42,7 @@ # Frame names to use as xarray dimension names FRAME_NAMES = { - "NEC": ["B_NEC"], + "NEC": ["B_NEC", "B_OB", "B_CF", "B_SV", "sigma_OB", "sigma_CF", "sigma_SV"], "VFM": ["B_VFM", "dB_Sun", "dB_AOCS", "dB_other", "B_error"], "quaternion": ["q_NEC_CRF"], "WGS84": ["GPS_Position", "LEO_Position"], @@ -202,7 +202,7 @@ def as_pandas_dataframe(self, expand=False): df[column + "_" + str(suffix)] = vector_data[:, i] return df - def as_xarray_dataset(self): + def as_xarray_dataset(self, reshape=False): # NB currrently does not set the global metadata (attrs) # (avoids issues with concatenating them) # (this is done in ReturnedData) @@ -260,6 +260,50 @@ def as_xarray_dataset(self): ds[dataname].attrs["units"] = self.get_variable_units(dataname) ds[dataname].attrs["description"] = self.get_variable_description( dataname) + # Reshape to a sensible higher dimensional structure + # Currently only for GVO data, and without magnetic model values or auxiliaries + # Inefficient as it is duplicating the data (ds -> ds2) + if reshape: + if "SiteCode" not in ds.data_vars: + raise NotImplementedError( + """ + Only available for GVO dataset where the "SiteCode" + parameter has been requested + """ + ) + if len(set(ds["SiteCode"].values)) != 300: + raise NotImplementedError( + """ + Only available when all 300 GVO's have been requested + """ + ) + ds = ds.drop("Spacecraft") + # TODO: extend to account for all data_vars + remaining_vars = set(ds.data_vars) - {"Latitude", "Longitude", "Radius", "SiteCode"} + incompatible = remaining_vars - {"B_OB", "B_CF", "B_SV", "sigma_OB", "sigma_CF", "sigma_SV"} + if len(incompatible) != 0: + raise NotImplementedError(f"Parameters: {incompatible} not supported") + # Extract unique coordinate values + t = ds["Timestamp"][0:-1:300] + lat = ds["Latitude"][0:300].values + lon = ds["Longitude"][0:300].values + rad = ds["Radius"][0:300].values + sitecodes = ds["SiteCode"][0:300].values + # Create new dataset based on reshaped variables + ds2 = xarray.Dataset( + coords={ + "Timestamp": t, "SiteCode": (("Site"), sitecodes), + "Latitude": ("Site", lat), "Longitude": ("Site", lon), "Radius": ("Site", rad), + "NEC": ["N", "E", "C"] + }, + ) + len_t = len(t) + for var in remaining_vars: + ds2[var] = ( + ("Timestamp", "Site", "NEC"), + numpy.reshape(ds[var].values, (len_t, 300, 3)) + ) + return ds2 return ds @@ -428,7 +472,7 @@ def as_dataframe(self, expand=False): df = f.as_pandas_dataframe(expand=expand) return df - def as_xarray(self, group=None): + def as_xarray(self, group=None, reshape=False): """Convert the data to an xarray Dataset. Note: @@ -444,7 +488,7 @@ def as_xarray(self, group=None): raise NotImplementedError("csv to xarray is not supported") elif self.filetype == 'cdf': with FileReader(self._file) as f: - ds = f.as_xarray_dataset() + ds = f.as_xarray_dataset(reshape=reshape) elif self.filetype == 'nc': ds = xarray.open_dataset(self._file.name, group=group) return ds @@ -573,9 +617,12 @@ def as_dataframe(self, expand=False): return pandas.concat( [d.as_dataframe(expand=expand) for d in self.contents]) - def as_xarray(self): + def as_xarray(self, reshape=False): """Convert the data to an xarray Dataset. + Args: + reshape (bool): Reshape to a convenient higher dimensional form + Returns: xarray.Dataset @@ -586,7 +633,7 @@ def as_xarray(self): # and the filtering that has been applied. ds_list = [] for i, data in enumerate(self.contents): - ds_part = data.as_xarray() + ds_part = data.as_xarray(reshape=reshape) if ds_part is None: print("Warning: ", "Unable to create dataset from part {} of {}".format( From 61a9b5ccbee13cccebeb6a27375227396cab33bd Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Tue, 26 Jan 2021 11:00:27 +0000 Subject: [PATCH 03/15] Fix package installation --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9931ae0..6e13d82 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,8 @@ def read(fname): scripts=[], package_data={ 'viresclient': [ - '_wps/templates/*' + '_wps/templates/*', + '_data/*' ], }, python_requires='>=3.6', From 4dc3772ff2bd52473ce9c7002dcd65712f5a8da0 Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Tue, 26 Jan 2021 15:17:56 +0000 Subject: [PATCH 04/15] Add model evaluation for data without B_NEC/F vars --- viresclient/_client_swarm.py | 45 ++++++++++++++++++++++------------- viresclient/_data_handling.py | 3 --- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/viresclient/_client_swarm.py b/viresclient/_client_swarm.py index 95e6234..7fd5b93 100644 --- a/viresclient/_client_swarm.py +++ b/viresclient/_client_swarm.py @@ -544,7 +544,7 @@ class SwarmRequest(ClientRequest): "Latitude_QD", "Longitude_QD", "MLT_QD", "Boundary_Flag", "Quality", "Pair_Indicator" ], - "AUX_OBSH": ["B_NEC", "F", "IAGA_code", "Quality", "SensorIndex"], + "AUX_OBSH": ["B_NEC", "F", "IAGA_code", "Quality", "ObsIndex"], "AUX_OBSM": ["B_NEC", "F", "IAGA_code", "Quality"], "AUX_OBSS": ["B_NEC", "F", "IAGA_code", "Quality"], "VOBS_SW_1M": ["SiteCode", "B_CF", "B_OB", "B_SV", "sigma_CF", "sigma_OB", "sigma_SV"], @@ -999,25 +999,38 @@ def set_products(self, measurements=None, models=None, custom_model=None, raise OSError("Custom model .shc file not found") else: custom_shc = None + # Set up the variables that actually get passed to the WPS request + + def _model_datavar_names(variable, residuals=False): + """Give the list of allowable variable names containing model evaluations""" + if variable not in model_variables: + raise ValueError(f"Expected one of {model_variables}; got '{variable}'") + affix = "_res_" if residuals else "_" + return [f"{variable}{affix}{model_name}" for model_name in model_ids] + + # Identify which (if any) of ["F", "B_NEC", ...] are requested + model_variables_present = set(measurements).intersection(set(model_variables)) + # Create the list of variable names to request variables = [] - for variable in measurements: - if variable in model_variables: - if residuals: - variables.extend( - "%s_res_%s" % (variable, model_name) - for model_name in model_ids - ) - else: - variables.append(variable) - variables.extend( - "%s_%s" % (variable, model_name) - for model_name in model_ids - ) - else: # not a model variable + for variable in model_variables_present: + if not residuals: + # Include "F" / "B_NEC" as requested... variables.append(variable) + # Include e.g. "F_IGRF" / "B_NEC_IGRF" / "B_NEC_res_IGRF" etc. + variables.extend(_model_datavar_names(variable, residuals=residuals)) + if models and (len(model_variables_present) == 0): + if residuals: + raise ValueError( + f""" + Residuals requested without one of {model_variables} set as measurements + """ + ) + # If "F" / "B_NEC" have not been requested, include e.g. "B_NEC_IGRF" etc. + variables.extend(_model_datavar_names("B_NEC")) + # Include all the non-model-related variables + variables.extend(list(set(measurements) - model_variables_present)) variables.extend(auxiliaries) - # Set these in the SwarmWPSInputs object self._request_inputs.model_expression = model_expression_string self._request_inputs.variables = variables self._request_inputs.sampling_step = sampling_step diff --git a/viresclient/_data_handling.py b/viresclient/_data_handling.py index 363fd9b..5a51391 100644 --- a/viresclient/_data_handling.py +++ b/viresclient/_data_handling.py @@ -280,9 +280,6 @@ def as_xarray_dataset(self, reshape=False): ds = ds.drop("Spacecraft") # TODO: extend to account for all data_vars remaining_vars = set(ds.data_vars) - {"Latitude", "Longitude", "Radius", "SiteCode"} - incompatible = remaining_vars - {"B_OB", "B_CF", "B_SV", "sigma_OB", "sigma_CF", "sigma_SV"} - if len(incompatible) != 0: - raise NotImplementedError(f"Parameters: {incompatible} not supported") # Extract unique coordinate values t = ds["Timestamp"][0:-1:300] lat = ds["Latitude"][0:300].values From de2d9e308e44237743f2d7fad66db544d6b21600 Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Tue, 26 Jan 2021 15:42:47 +0000 Subject: [PATCH 05/15] Fix xarray coord attrs --- viresclient/_data_handling.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/viresclient/_data_handling.py b/viresclient/_data_handling.py index 5a51391..d0e55d3 100644 --- a/viresclient/_data_handling.py +++ b/viresclient/_data_handling.py @@ -252,14 +252,10 @@ def as_xarray_dataset(self, reshape=False): for dimname, dimlabels in FRAME_LABELS.items(): if dimname in dims_used: ds[dimname] = numpy.array(dimlabels) - ds[dimname].attrs["description"] = FRAME_DESCRIPTIONS.get( - dimname, None) ds = ds.set_coords(dimname) - # Add metadata of each variable - for dataname in ds: - ds[dataname].attrs["units"] = self.get_variable_units(dataname) - ds[dataname].attrs["description"] = self.get_variable_description( - dataname) + # ds[dimname].attrs["description"] = FRAME_DESCRIPTIONS.get( + # dimname, None) + # ds = ds.set_coords(dimname) # Reshape to a sensible higher dimensional structure # Currently only for GVO data, and without magnetic model values or auxiliaries # Inefficient as it is duplicating the data (ds -> ds2) @@ -268,7 +264,7 @@ def as_xarray_dataset(self, reshape=False): raise NotImplementedError( """ Only available for GVO dataset where the "SiteCode" - parameter has been requested + parameter has been requested """ ) if len(set(ds["SiteCode"].values)) != 300: @@ -300,7 +296,17 @@ def as_xarray_dataset(self, reshape=False): ("Timestamp", "Site", "NEC"), numpy.reshape(ds[var].values, (len_t, 300, 3)) ) - return ds2 + ds = ds2 + # Add metadata of each variable + for var in list(ds.data_vars) + list(ds.coords): + try: + ds[var].attrs["units"] = self.get_variable_units(var) + except KeyError: + ds[var].attrs["units"] = None + try: + ds[var].attrs["description"] = self.get_variable_description(var) + except KeyError: + ds[var].attrs["description"] = FRAME_DESCRIPTIONS.get(var, None) return ds From f9ccf96cb809782fc512392270dc14e8df3e09c2 Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Tue, 26 Jan 2021 17:25:10 +0000 Subject: [PATCH 06/15] Try fix shell detection --- viresclient/_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viresclient/_client.py b/viresclient/_client.py index 075a367..8aa4588 100644 --- a/viresclient/_client.py +++ b/viresclient/_client.py @@ -35,7 +35,7 @@ try: from IPython import get_ipython IN_JUPYTER = 'zmqshell' in str(type(get_ipython())) -except ImportError: +except Exception: IN_JUPYTER = False from tqdm import tqdm From d1784d52d2bd536443982261a327102738813e6a Mon Sep 17 00:00:00 2001 From: Martin Paces Date: Wed, 3 Feb 2021 12:38:15 +0100 Subject: [PATCH 07/15] Adding VOBS secular variations. --- viresclient/_client_swarm.py | 50 ++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/viresclient/_client_swarm.py b/viresclient/_client_swarm.py index 7fd5b93..949a12a 100644 --- a/viresclient/_client_swarm.py +++ b/viresclient/_client_swarm.py @@ -448,6 +448,26 @@ class SwarmRequest(ClientRequest): "CR_OPER_VOBS_4M_2_", *[f"CR_OPER_VOBS_4M_2_:{site}" for site in VOBS_SITES] ], + "VOBS_SW_1M:SecularVariation": [ + "SW_OPER_VOBS_1M_2_:SecularVariation", + *[f"SW_OPER_VOBS_1M_2_:SecularVariation:{site}" for site in VOBS_SITES] + ], + "VOBS_SW_4M:SecularVariation": [ + "SW_OPER_VOBS_4M_2_:SecularVariation", + *[f"SW_OPER_VOBS_4M_2_:SecularVariation:{site}" for site in VOBS_SITES] + ], + "VOBS_CH_1M:SecularVariation": [ + "CH_OPER_VOBS_1M_2_:SecularVariation", + *[f"CH_OPER_VOBS_1M_2_:SecularVariation:{site}" for site in VOBS_SITES] + ], + "VOBS_CH_4M:SecularVariation": [ + "CH_OPER_VOBS_4M_2_:SecularVariation", + *[f"CH_OPER_VOBS_4M_2_:SecularVariation:{site}" for site in VOBS_SITES] + ], + "VOBS_CR_4M:SecularVariation": [ + "CR_OPER_VOBS_4M_2_:SecularVariation", + *[f"CR_OPER_VOBS_4M_2_:SecularVariation:{site}" for site in VOBS_SITES] + ], } OBS_COLLECTIONS = [ @@ -459,6 +479,11 @@ class SwarmRequest(ClientRequest): "CH_OPER_VOBS_1M_2_", "CH_OPER_VOBS_4M_2_", "CR_OPER_VOBS_4M_2_", + "SW_OPER_VOBS_1M_2_:SecularVariation", + "SW_OPER_VOBS_4M_2_:SecularVariation", + "CH_OPER_VOBS_1M_2_:SecularVariation", + "CH_OPER_VOBS_4M_2_:SecularVariation", + "CR_OPER_VOBS_4M_2_:SecularVariation", ] @@ -483,6 +508,11 @@ class SwarmRequest(ClientRequest): "VOBS_SW_4M": "P122D", "VOBS_CH_4M": "P122D", "VOBS_CR_4M": "P122D", + "VOBS_SW_1M:SecularVariation": "P31D", + "VOBS_CH_1M:SecularVariation": "P31D", + "VOBS_SW_4M:SecularVariation": "P122D", + "VOBS_CH_4M:SecularVariation": "P122D", + "VOBS_CR_4M:SecularVariation": "P122D", } PRODUCT_VARIABLES = { @@ -547,11 +577,16 @@ class SwarmRequest(ClientRequest): "AUX_OBSH": ["B_NEC", "F", "IAGA_code", "Quality", "ObsIndex"], "AUX_OBSM": ["B_NEC", "F", "IAGA_code", "Quality"], "AUX_OBSS": ["B_NEC", "F", "IAGA_code", "Quality"], - "VOBS_SW_1M": ["SiteCode", "B_CF", "B_OB", "B_SV", "sigma_CF", "sigma_OB", "sigma_SV"], - "VOBS_CH_1M": ["SiteCode", "B_CF", "B_OB", "B_SV", "sigma_CF", "sigma_OB", "sigma_SV"], - "VOBS_SW_4M": ["SiteCode", "B_CF", "B_OB", "B_SV", "sigma_CF", "sigma_OB", "sigma_SV"], - "VOBS_CH_4M": ["SiteCode", "B_CF", "B_OB", "B_SV", "sigma_CF", "sigma_OB", "sigma_SV"], - "VOBS_CR_4M": ["SiteCode", "B_CF", "B_OB", "B_SV", "sigma_CF", "sigma_OB", "sigma_SV"], + "VOBS_SW_1M": ["SiteCode", "B_CF", "B_OB", "sigma_CF", "sigma_OB"], + "VOBS_CH_1M": ["SiteCode", "B_CF", "B_OB", "sigma_CF", "sigma_OB"], + "VOBS_SW_4M": ["SiteCode", "B_CF", "B_OB", "sigma_CF", "sigma_OB"], + "VOBS_CH_4M": ["SiteCode", "B_CF", "B_OB", "sigma_CF", "sigma_OB"], + "VOBS_CR_4M": ["SiteCode", "B_CF", "B_OB", "sigma_CF", "sigma_OB"], + "VOBS_SW_1M:SecularVariation": ["SiteCode", "B_SV", "sigma_SV"], + "VOBS_CH_1M:SecularVariation": ["SiteCode", "B_SV", "sigma_SV"], + "VOBS_SW_4M:SecularVariation": ["SiteCode", "B_SV", "sigma_SV"], + "VOBS_CH_4M:SecularVariation": ["SiteCode", "B_SV", "sigma_SV"], + "VOBS_CR_4M:SecularVariation": ["SiteCode", "B_SV", "sigma_SV"], } AUXILIARY_VARIABLES = [ @@ -681,6 +716,11 @@ def available_collections(self, groupname=None, details=True): collections_short["VOBS_CH_1M"] = ["CH_OPER_VOBS_1M_2_"] collections_short["VOBS_CH_4M"] = ["CH_OPER_VOBS_4M_2_"] collections_short["VOBS_CR_4M"] = ["CR_OPER_VOBS_4M_2_"] + collections_short["VOBS_SW_1M:SecularVariation"] = ["SW_OPER_VOBS_1M_2_:SecularVariation"] + collections_short["VOBS_SW_4M:SecularVariation"] = ["SW_OPER_VOBS_4M_2_:SecularVariation"] + collections_short["VOBS_CH_1M:SecularVariation"] = ["CH_OPER_VOBS_1M_2_:SecularVariation"] + collections_short["VOBS_CH_4M:SecularVariation"] = ["CH_OPER_VOBS_4M_2_:SecularVariation"] + collections_short["VOBS_CR_4M:SecularVariation"] = ["CR_OPER_VOBS_4M_2_:SecularVariation"] def _filter_collections(groupname): """ Reduce the full list to just one group, e.g. "MAG """ From d063cfd717f5b300e4bc48678966175e3360bb23 Mon Sep 17 00:00:00 2001 From: Martin Paces Date: Thu, 18 Mar 2021 18:12:04 +0100 Subject: [PATCH 08/15] Minimal changes to work with non-Swarm MAG data. --- viresclient/_client_swarm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/viresclient/_client_swarm.py b/viresclient/_client_swarm.py index 949a12a..42fba13 100644 --- a/viresclient/_client_swarm.py +++ b/viresclient/_client_swarm.py @@ -236,11 +236,13 @@ def _spacecraft_from_collection(collection): name = "AUX_OBS" if ":" in collection: name = f"{name}:{collection[19:22]}" - else: + elif collection[:3] == "SW_": # 12th character in name, e.g. SW_OPER_MAGx_LR_1B sc = collection[11] sc_to_name = {"A": "Alpha", "B": "Bravo", "C": "Charlie"} name = sc_to_name.get(sc, "NSC") + else: + name = collection return name def set_collections(self, collections): From 354ee4bcca9a8bd31d107a95e89dbc4d638caae4 Mon Sep 17 00:00:00 2001 From: Martin Paces Date: Mon, 12 Apr 2021 17:10:21 +0200 Subject: [PATCH 09/15] Allowing model requests for B_NEC[123] variables --- viresclient/_client_swarm.py | 40 +++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/viresclient/_client_swarm.py b/viresclient/_client_swarm.py index 42fba13..71a6905 100644 --- a/viresclient/_client_swarm.py +++ b/viresclient/_client_swarm.py @@ -602,7 +602,13 @@ class SwarmRequest(ClientRequest): "NGPLongitude", "DipoleTiltAngle", ] - MAGNETIC_MODEL_VARIABLES = ["F", "B_NEC"] + MAGNETIC_MODEL_VARIABLES = { + "F": "F", + "B_NEC": "B_NEC", + "B_NEC1": "B_NEC", + "B_NEC2": "B_NEC", + "B_NEC3": "B_NEC", + } MAGNETIC_MODELS = [ "IGRF", "IGRF12", "LCS-1", "MF7", @@ -992,7 +998,7 @@ def set_products(self, measurements=None, models=None, custom_model=None, raise Exception("Must run .set_collection() first.") measurements = [] if measurements is None else measurements models = [] if models is None else models - model_variables = set(self._available["model_variables"]) + model_variables = self._available["model_variables"] auxiliaries = [] if auxiliaries is None else auxiliaries # If inputs are strings (when providing only one parameter) # put them in lists @@ -1052,16 +1058,26 @@ def _model_datavar_names(variable, residuals=False): return [f"{variable}{affix}{model_name}" for model_name in model_ids] # Identify which (if any) of ["F", "B_NEC", ...] are requested - model_variables_present = set(measurements).intersection(set(model_variables)) + model_variables_present = { + variable: model_variables[variable] + for variable in measurements if variable in model_variables + } # Create the list of variable names to request - variables = [] - for variable in model_variables_present: + variables = set() + _model_variables_requested = set() + + for variable, model_variable in model_variables_present.items(): if not residuals: # Include "F" / "B_NEC" as requested... - variables.append(variable) + variables.add(variable) # Include e.g. "F_IGRF" / "B_NEC_IGRF" / "B_NEC_res_IGRF" etc. - variables.extend(_model_datavar_names(variable, residuals=residuals)) - if models and (len(model_variables_present) == 0): + if not residuals or variable == model_variable: + _model_variables_requested.add(model_variable) + variables.update(_model_datavar_names( + model_variable, residuals=residuals + )) + _no_model_variable = False + if models and not _model_variables_requested: if residuals: raise ValueError( f""" @@ -1069,12 +1085,12 @@ def _model_datavar_names(variable, residuals=False): """ ) # If "F" / "B_NEC" have not been requested, include e.g. "B_NEC_IGRF" etc. - variables.extend(_model_datavar_names("B_NEC")) + variables.update(_model_datavar_names("B_NEC")) # Include all the non-model-related variables - variables.extend(list(set(measurements) - model_variables_present)) - variables.extend(auxiliaries) + variables.update(set(measurements) - set(model_variables_present)) + variables.update(auxiliaries) self._request_inputs.model_expression = model_expression_string - self._request_inputs.variables = variables + self._request_inputs.variables = list(variables) self._request_inputs.sampling_step = sampling_step self._request_inputs.custom_shc = custom_shc return self From aa1e9870c16d7753bd77c5fc499e3b9418a9286c Mon Sep 17 00:00:00 2001 From: Martin Paces Date: Mon, 2 Aug 2021 11:39:50 +0200 Subject: [PATCH 10/15] Support for multiple missions in SwarmRequest.get_times_for_orbits() method. --- viresclient/_client_swarm.py | 67 +++++++++++++++++-- .../templates/vires_times_from_orbits.xml | 10 +++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/viresclient/_client_swarm.py b/viresclient/_client_swarm.py index 71a6905..944e8da 100644 --- a/viresclient/_client_swarm.py +++ b/viresclient/_client_swarm.py @@ -7,6 +7,7 @@ from pandas import read_csv from tqdm import tqdm from textwrap import dedent +from warnings import warn from ._wps.environment import JINJA2_ENVIRONMENT from ._wps.time_util import parse_datetime @@ -394,6 +395,12 @@ class SwarmRequest(ClientRequest): logging_level (str): """ + MISSION_SPACECRAFTS = { + 'Swarm': ['A', 'B', 'C'], + 'GRACE': ['1', '2'], + 'GRACE-FO': ['1', '2'], + } + COLLECTIONS = { "MAG": ["SW_OPER_MAG{}_LR_1B".format(x) for x in "ABC"], "MAG_HR": ["SW_OPER_MAG{}_HR_1B".format(x) for x in "ABC"], @@ -1128,14 +1135,17 @@ def clear_range_filter(self): self._request_inputs.filters = None return self - def get_times_for_orbits(self, spacecraft, start_orbit, end_orbit): + def get_times_for_orbits(self, start_orbit, end_orbit, mission="Swarm", spacecraft=None): """Translate a pair of orbit numbers to a time interval. Args: - spacecraft (str): one of ('A','B','C') or - ("Alpha", "Bravo", "Charlie") start_orbit (int): a starting orbit number end_orbit (int): a later orbit number + spacecraft (str): + Swarm: one of ('A','B','C') or ("Alpha", "Bravo", "Charlie") + GRACE: one of ('1','2') + GRACE-FO: one of ('1','2') + mission (str): one of ('Swarm', 'GRACE', 'GRACE-FO') Returns: tuple (datetime): (start_time, end_time) The start time of the @@ -1143,12 +1153,61 @@ def get_times_for_orbits(self, spacecraft, start_orbit, end_orbit): (Based on ascending nodes of the orbits) """ + # check old function signature and print warning + if ( + isinstance(start_orbit, str) and + isinstance(mission, int) and + spacecraft is None + ): + spacecraft, start_orbit, end_orbit = start_orbit, end_orbit, mission + mission = "Swarm" + warn( + "The order of SwarmRequest.get_times_for_orbits() method's " + "parameters has changed! " + "The backward compatibility will be removed in the future. " + "Please change your code to: " + "request.get_times_for_orbits(start_orbit, end_orbit, " + "'Swarm', spacecraft)", + FutureWarning, + ) + + start_orbit = int(start_orbit) + end_orbit = int(end_orbit) + + if mission not in self.MISSION_SPACECRAFTS: + raise ValueError( + f"Invalid mission {mission}!" + f"Allowed options are: {','.join(self.MISSION_SPACECRAFTS)}" + ) + # Change to spacecraft = "A" etc. for this request - if spacecraft in ("Alpha", "Bravo", "Charlie"): + spacecraft = str(spacecraft) + if mission == "Swarm" and spacecraft in ("Alpha", "Bravo", "Charlie"): spacecraft = spacecraft[0] + + if self.MISSION_SPACECRAFTS[mission]: + # missions with required spacecraft id + if not spacecraft: + raise ValueError( + f"The {mission} spacecraft is required!" + f"Allowed options are: {','.join(self.MISSION_SPACECRAFTS[mission])}" + ) + if spacecraft not in self.MISSION_SPACECRAFTS[mission]: + raise ValueError( + f"Invalid {mission} spacecraft! " + f"Allowed options are: {','.join(self.MISSION_SPACECRAFTS[mission])}" + ) + + elif spacecraft: # mission without spacecraft id + raise ValueError( + f"No {mission} spacecraft shall be specified! " + "Set spacecraft to None." + ) + templatefile = TEMPLATE_FILES["times_from_orbits"] template = JINJA2_ENVIRONMENT.get_template(templatefile) request = template.render( + mission=mission, spacecraft=spacecraft, start_orbit=start_orbit, end_orbit=end_orbit diff --git a/viresclient/_wps/templates/vires_times_from_orbits.xml b/viresclient/_wps/templates/vires_times_from_orbits.xml index 4f8d71b..eacde63 100644 --- a/viresclient/_wps/templates/vires_times_from_orbits.xml +++ b/viresclient/_wps/templates/vires_times_from_orbits.xml @@ -2,12 +2,22 @@ vires:get_orbit_timerange + {% if mission -%} + + mission + + {{ mission }} + + + {% endif -%} + {% if spacecraft -%} spacecraft {{ spacecraft }} + {% endif -%} start_orbit From 08119a3ac60e44a690a8683332eaf276786887a3 Mon Sep 17 00:00:00 2001 From: Martin Paces Date: Thu, 5 Aug 2021 09:27:07 +0200 Subject: [PATCH 11/15] Adding GRACE and GRACE-FO spacecrafts. --- viresclient/_data_handling.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/viresclient/_data_handling.py b/viresclient/_data_handling.py index d0e55d3..49efdce 100644 --- a/viresclient/_data_handling.py +++ b/viresclient/_data_handling.py @@ -40,6 +40,8 @@ CDF_EPOCH_1970 = 62167219200000.0 +ALLOWED_SPACECRFTS = ["A", "B", "C", "1", "2", "-"] + # Frame names to use as xarray dimension names FRAME_NAMES = { "NEC": ["B_NEC", "B_OB", "B_CF", "B_SV", "sigma_OB", "sigma_CF", "sigma_SV"], @@ -212,7 +214,7 @@ def as_xarray_dataset(self, reshape=False): self._cdftime_to_datetime(self.get_variable("Timestamp"))}) # Add Spacecraft variable as Categorical to save memory ds["Spacecraft"] = (("Timestamp",), pandas.Categorical( - self.get_variable("Spacecraft"), categories=["A", "B", "C", "-"])) + self.get_variable("Spacecraft"), categories=ALLOWED_SPACECRFTS)) datanames = set(self.variables) datanames.remove("Timestamp") datanames.remove("Spacecraft") From f3d52d355832e601c301b8d3fda9cc8dae6e02c8 Mon Sep 17 00:00:00 2001 From: Martin Paces Date: Thu, 5 Aug 2021 11:06:58 +0200 Subject: [PATCH 12/15] Support for CryoSat-2 missions in SwarmRequest.get_times_for_orbits() method. --- viresclient/_client_swarm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/viresclient/_client_swarm.py b/viresclient/_client_swarm.py index 944e8da..662c601 100644 --- a/viresclient/_client_swarm.py +++ b/viresclient/_client_swarm.py @@ -1,3 +1,5 @@ +# pylint: disable=missing-docstring, invalid-name,line-too-long + import datetime import json from collections import OrderedDict @@ -399,6 +401,7 @@ class SwarmRequest(ClientRequest): 'Swarm': ['A', 'B', 'C'], 'GRACE': ['1', '2'], 'GRACE-FO': ['1', '2'], + 'CryoSat-2': None, } COLLECTIONS = { @@ -1145,7 +1148,8 @@ def get_times_for_orbits(self, start_orbit, end_orbit, mission="Swarm", spacecra Swarm: one of ('A','B','C') or ("Alpha", "Bravo", "Charlie") GRACE: one of ('1','2') GRACE-FO: one of ('1','2') - mission (str): one of ('Swarm', 'GRACE', 'GRACE-FO') + CryoSat-2: None + mission (str): one of ('Swarm', 'GRACE', 'GRACE-FO', 'CryoSat-2') Returns: tuple (datetime): (start_time, end_time) The start time of the @@ -1181,7 +1185,7 @@ def get_times_for_orbits(self, start_orbit, end_orbit, mission="Swarm", spacecra ) # Change to spacecraft = "A" etc. for this request - spacecraft = str(spacecraft) + spacecraft = str(spacecraft) if spacecraft is not None else None if mission == "Swarm" and spacecraft in ("Alpha", "Bravo", "Charlie"): spacecraft = spacecraft[0] From 4925b3abd5a0371ba76d84c5abfa356870f5eef7 Mon Sep 17 00:00:00 2001 From: Martin Paces Date: Fri, 6 Aug 2021 14:04:24 +0200 Subject: [PATCH 13/15] Implementing residuals for the CryoSat-2 B_NEC[123] variables. --- viresclient/_client_swarm.py | 72 ++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/viresclient/_client_swarm.py b/viresclient/_client_swarm.py index 662c601..a1d5a38 100644 --- a/viresclient/_client_swarm.py +++ b/viresclient/_client_swarm.py @@ -1060,45 +1060,43 @@ def set_products(self, measurements=None, models=None, custom_model=None, # Set up the variables that actually get passed to the WPS request - def _model_datavar_names(variable, residuals=False): - """Give the list of allowable variable names containing model evaluations""" - if variable not in model_variables: - raise ValueError(f"Expected one of {model_variables}; got '{variable}'") - affix = "_res_" if residuals else "_" - return [f"{variable}{affix}{model_name}" for model_name in model_ids] - - # Identify which (if any) of ["F", "B_NEC", ...] are requested - model_variables_present = { - variable: model_variables[variable] - for variable in measurements if variable in model_variables - } - # Create the list of variable names to request - variables = set() - _model_variables_requested = set() - - for variable, model_variable in model_variables_present.items(): - if not residuals: - # Include "F" / "B_NEC" as requested... - variables.add(variable) - # Include e.g. "F_IGRF" / "B_NEC_IGRF" / "B_NEC_res_IGRF" etc. - if not residuals or variable == model_variable: - _model_variables_requested.add(model_variable) - variables.update(_model_datavar_names( - model_variable, residuals=residuals - )) - _no_model_variable = False - if models and not _model_variables_requested: - if residuals: - raise ValueError( - f""" - Residuals requested without one of {model_variables} set as measurements - """ + # Requested variables, start with the measurements ... + variables = set(measurements) + + # model-related measurements + _requested_model_variables = [ + variable for variable in measurements + if variable in model_variables + ] + + if residuals: + # Remove the measurements ... + variables.difference_update(_requested_model_variables) + # ... add their residuals instead. + variables.update( + f"{variable}_res_{model_id}" + for variable in _requested_model_variables + for model_id in model_ids + ) + + else: + # If no variable is requested fall back to B_NEC. + if not _requested_model_variables: + _requested_model_variables = ["B_NEC"] + + # Add calculated model variables. + variables.update( + f"{variable}_{model_id}" + for variable in ( + model_variables[variable] + for variable in _requested_model_variables ) - # If "F" / "B_NEC" have not been requested, include e.g. "B_NEC_IGRF" etc. - variables.update(_model_datavar_names("B_NEC")) - # Include all the non-model-related variables - variables.update(set(measurements) - set(model_variables_present)) + for model_id in model_ids + ) + + # Finally, add the auxiliary variables. variables.update(auxiliaries) + self._request_inputs.model_expression = model_expression_string self._request_inputs.variables = list(variables) self._request_inputs.sampling_step = sampling_step From 7acd84a195e0e7abdf0a8e9e71e9644840599095 Mon Sep 17 00:00:00 2001 From: Ashley Smth Date: Mon, 23 Aug 2021 18:55:51 +0100 Subject: [PATCH 14/15] Add configuration for Cryosat-2, GRACE, GRACE-FO --- viresclient/_client_swarm.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/viresclient/_client_swarm.py b/viresclient/_client_swarm.py index e5503c6..5067598 100644 --- a/viresclient/_client_swarm.py +++ b/viresclient/_client_swarm.py @@ -242,6 +242,8 @@ def _spacecraft_from_collection(collection): sc = collection[11] sc_to_name = {"A": "Alpha", "B": "Bravo", "C": "Charlie"} name = sc_to_name.get(sc, "NSC") + else: + name = collection return name def set_collections(self, collections): @@ -498,6 +500,10 @@ class SwarmRequest(ClientRequest): "MIT_TEC:ID": [f"SW_OPER_MIT{x}TEC_2F:ID" for x in "ABC"], "PPI_FAC": [f"SW_OPER_PPI{x}FAC_2F" for x in "ABC"], "PPI_FAC:ID": [f"SW_OPER_PPI{x}FAC_2F:ID" for x in "ABC"], + # Multi-mission magnetic products + "MAG_CS": ["CS_OPER_MAG",], + "MAG_GRACE": ["GRACE_A_MAG", "GRACE_B_MAG"], + "MAG_GFO": ["GF1_OPER_FGM_ACAL_CORR", "GF2_OPER_FGM_ACAL_CORR"], } OBS_COLLECTIONS = [ @@ -660,6 +666,18 @@ class SwarmRequest(ClientRequest): "Counter", "Latitude_QD", "Longitude_QD", "MLT_QD", "L_value", "SZA", "Position_Quality", "PointType", ], + "MAG_CS": [ + "F", "B_NEC", "B_mod_NEC", "B_NEC1", "B_NEC2", "B_NEC3", + "B_FGM1", "B_FGM2", "B_FGM3", "q_NEC_CRF", "q_error", + ], + "MAG_GRACE": [ + "F", "B_NEC", "B_NEC_raw", "B_FGM", "B_mod_NEC", + "q_NEC_CRF", "q_error", + ], + "MAG_GFO": [ + "B_NEC", "B_FGM", "dB_MTQ_FGM", "dB_XI_FGM", "dB_NY_FGM", "dB_BT_FGM", + "dB_ST_FGM", "dB_SA_FGM", "dB_BAT_FGM", "q_NEC_FGM", "B_FLAG", + ] } AUXILIARY_VARIABLES = [ From c4de7b2cabb036086759582e7dfdf0426b44aef5 Mon Sep 17 00:00:00 2001 From: Ashley Smth Date: Mon, 23 Aug 2021 20:19:32 +0100 Subject: [PATCH 15/15] Add multi-mission mag products to docs --- docs/available_parameters.rst | 10 ++++++++++ docs/release_notes.rst | 1 + 2 files changed, 11 insertions(+) diff --git a/docs/available_parameters.rst b/docs/available_parameters.rst index 5e52d5c..32cccce 100644 --- a/docs/available_parameters.rst +++ b/docs/available_parameters.rst @@ -105,6 +105,16 @@ SW_OPER_VOBS_1M_2\_:SecularVariation VOBS_SW_1M:SecularVariation Secular variati Each VOBS product (e.g. Swarm 1-monthly) is split into two collections (e.g. ``SW_OPER_VOBS_1M_2_`` (containing ``B_OB`` & ``B_CF``) and ``SW_OPER_VOBS_1M_2_:SecularVariation`` (containing ``B_SV``)) because of the different temporal sampling points (i.e. differing ``Timestamp``) of these measurements. Data can also be requested for a specific virtual observatory alone (distinguishable by the ``SiteCode`` variable) with special collection names like ``SW_OPER_VOBS_1M_2_:N65W051`` and ``SW_OPER_VOBS_1M_2_:SecularVariation:N65W051``. +Calibrated magnetic data are also available from external missions: Cryosat-2, GRACE (A+B), GRACE-FO (1+2): + +=============================== ================ ========================================================================================================= +Collection full name Collection type Available measurement names +=============================== ================ ========================================================================================================= +CS_OPER_MAG MAG_CS ``F,B_NEC,B_mod_NEC,B_NEC1,B_NEC2,B_NEC3,B_FGM1,B_FGM2,B_FGM3,q_NEC_CRF,q_error`` +GRACE_x_MAG (x=A/B) MAG_GRACE ``F,B_NEC,B_NEC_raw,B_FGM,B_mod_NEC,q_NEC_CRF,q_error`` +GFx_OPER_FGM_ACAL_CORR (x=1/2) MAG_GFO ``B_NEC,B_FGM,dB_MTQ_FGM,dB_XI_FGM,dB_NY_FGM,dB_BT_FGM,dB_ST_FGM,dB_SA_FGM,dB_BAT_FGM,q_NEC_FGM,B_FLAG`` +=============================== ================ ========================================================================================================= + The ``measurements``, ``models``, and ``auxiliaries`` chosen will match the cadence of the ``collection`` chosen. ---- diff --git a/docs/release_notes.rst b/docs/release_notes.rst index b20b3fd..f8cb8fb 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -10,6 +10,7 @@ Changes from 0.8.0 to 0.9.0 - Added support for: - PRISM products (``SW_OPER_MITx_LP_2F``, ``SW_OPER_MITxTEC_2F``, ``SW_OPER_PPIxFAC_2F``) + - Multi-mission magnetic products (``CS_OPER_MAG``, ``GRACE_x_MAG``, ``GFx_OPER_FGM_ACAL_CORR``) - Fixed missing auxiliary "dDst" - Fixed fetching longer time series of hourly observatory products