Add support for external mission MAG products

docs/available_parameters.rst

@@ -105,6 +105,16 @@ SW_OPER_VOBS_1M_2\_:SecularVariation VOBS_SW_1M:SecularVariation Secular variati
 
 Each VOBS product (e.g. Swarm 1-monthly) is split into two collections (e.g. ``SW_OPER_VOBS_1M_2_`` (containing ``B_OB`` & ``B_CF``) and ``SW_OPER_VOBS_1M_2_:SecularVariation`` (containing ``B_SV``)) because of the different temporal sampling points (i.e. differing ``Timestamp``) of these measurements. Data can also be requested for a specific virtual observatory alone (distinguishable by the ``SiteCode`` variable) with special collection names like ``SW_OPER_VOBS_1M_2_:N65W051`` and ``SW_OPER_VOBS_1M_2_:SecularVariation:N65W051``.
 
+Calibrated magnetic data are also available from external missions: Cryosat-2, GRACE (A+B), GRACE-FO (1+2):
+
+=============================== ================ =========================================================================================================
+Collection full name            Collection type  Available measurement names
+=============================== ================ =========================================================================================================
+CS_OPER_MAG                     MAG_CS           ``F,B_NEC,B_mod_NEC,B_NEC1,B_NEC2,B_NEC3,B_FGM1,B_FGM2,B_FGM3,q_NEC_CRF,q_error``
+GRACE_x_MAG (x=A/B)             MAG_GRACE        ``F,B_NEC,B_NEC_raw,B_FGM,B_mod_NEC,q_NEC_CRF,q_error``
+GFx_OPER_FGM_ACAL_CORR (x=1/2)  MAG_GFO          ``B_NEC,B_FGM,dB_MTQ_FGM,dB_XI_FGM,dB_NY_FGM,dB_BT_FGM,dB_ST_FGM,dB_SA_FGM,dB_BAT_FGM,q_NEC_FGM,B_FLAG``
+=============================== ================ =========================================================================================================
+
 The ``measurements``, ``models``, and ``auxiliaries`` chosen will match the cadence of the ``collection`` chosen.
 
 ----

docs/release_notes.rst

@@ -10,6 +10,7 @@ Changes from 0.8.0 to 0.9.0
 - Added support for:
 
   - PRISM products (``SW_OPER_MITx_LP_2F``, ``SW_OPER_MITxTEC_2F``, ``SW_OPER_PPIxFAC_2F``)
+  - Multi-mission magnetic products (``CS_OPER_MAG``, ``GRACE_x_MAG``, ``GFx_OPER_FGM_ACAL_CORR``)
 
 - Fixed missing auxiliary "dDst"
 - Fixed fetching longer time series of hourly observatory products

viresclient/_client_swarm.py

@@ -1,3 +1,5 @@
+# pylint: disable=missing-docstring, invalid-name,line-too-long
+
 import datetime
 import json
 from collections import OrderedDict
@@ -7,6 +9,7 @@
 from pandas import read_csv
 from tqdm import tqdm
 from textwrap import dedent
+from warnings import warn
 
 from ._wps.environment import JINJA2_ENVIRONMENT
 from ._wps.time_util import parse_datetime
@@ -234,11 +237,13 @@ def _spacecraft_from_collection(collection):
             name = "AUX_OBS"
             if ":" in collection:
                 name = f"{name}:{collection[19:22]}"
-        else:
+        elif collection[:3] == "SW_":
             # 12th character in name, e.g. SW_OPER_MAGx_LR_1B
             sc = collection[11]
             sc_to_name = {"A": "Alpha", "B": "Bravo", "C": "Charlie"}
             name = sc_to_name.get(sc, "NSC")
+        else:
+            name = collection
         return name
 
     def set_collections(self, collections):
@@ -390,6 +395,13 @@ class SwarmRequest(ClientRequest):
         logging_level (str):
 
     """
+    MISSION_SPACECRAFTS = {
+        'Swarm': ['A', 'B', 'C'],
+        'GRACE': ['1', '2'],
+        'GRACE-FO': ['1', '2'],
+        'CryoSat-2': None,
+    }
+
     COLLECTIONS = {
         "MAG": ["SW_OPER_MAG{}_LR_1B".format(x) for x in "ABC"],
         "MAG_HR": ["SW_OPER_MAG{}_HR_1B".format(x) for x in "ABC"],
@@ -488,6 +500,10 @@ class SwarmRequest(ClientRequest):
         "MIT_TEC:ID": [f"SW_OPER_MIT{x}TEC_2F:ID" for x in "ABC"],
         "PPI_FAC": [f"SW_OPER_PPI{x}FAC_2F" for x in "ABC"],
         "PPI_FAC:ID": [f"SW_OPER_PPI{x}FAC_2F:ID" for x in "ABC"],
+        # Multi-mission magnetic products
+        "MAG_CS": ["CS_OPER_MAG",],
+        "MAG_GRACE": ["GRACE_A_MAG", "GRACE_B_MAG"],
+        "MAG_GFO": ["GF1_OPER_FGM_ACAL_CORR", "GF2_OPER_FGM_ACAL_CORR"],
     }
 
     OBS_COLLECTIONS = [
@@ -650,6 +666,18 @@ class SwarmRequest(ClientRequest):
             "Counter", "Latitude_QD", "Longitude_QD", "MLT_QD", "L_value", "SZA",
             "Position_Quality", "PointType",
         ],
+        "MAG_CS": [
+            "F", "B_NEC", "B_mod_NEC", "B_NEC1", "B_NEC2", "B_NEC3",
+            "B_FGM1", "B_FGM2", "B_FGM3", "q_NEC_CRF", "q_error",
+        ],
+        "MAG_GRACE": [
+            "F", "B_NEC", "B_NEC_raw", "B_FGM", "B_mod_NEC",
+            "q_NEC_CRF", "q_error",
+        ],
+        "MAG_GFO": [
+            "B_NEC", "B_FGM", "dB_MTQ_FGM", "dB_XI_FGM", "dB_NY_FGM", "dB_BT_FGM",
+            "dB_ST_FGM", "dB_SA_FGM", "dB_BAT_FGM", "q_NEC_FGM", "B_FLAG",
+        ]
     }
 
     AUXILIARY_VARIABLES = [
@@ -663,7 +691,13 @@ class SwarmRequest(ClientRequest):
         "NGPLongitude", "DipoleTiltAngle", "dDst"
     ]
 
-    MAGNETIC_MODEL_VARIABLES = ["F", "B_NEC"]
+    MAGNETIC_MODEL_VARIABLES = {
+        "F": "F",
+        "B_NEC": "B_NEC",
+        "B_NEC1": "B_NEC",
+        "B_NEC2": "B_NEC",
+        "B_NEC3": "B_NEC",
+    }
 
     MAGNETIC_MODELS = [
         "IGRF", "IGRF12", "LCS-1", "MF7",
@@ -1057,7 +1091,7 @@ def set_products(self, measurements=None, models=None, custom_model=None,
             raise Exception("Must run .set_collection() first.")
         measurements = [] if measurements is None else measurements
         models = [] if models is None else models
-        model_variables = set(self._available["model_variables"])
+        model_variables = self._available["model_variables"]
         auxiliaries = [] if auxiliaries is None else auxiliaries
         # If inputs are strings (when providing only one parameter)
         #  put them in lists
@@ -1109,37 +1143,45 @@ def set_products(self, measurements=None, models=None, custom_model=None,
 
         # Set up the variables that actually get passed to the WPS request
 
-        def _model_datavar_names(variable, residuals=False):
-            """Give the list of allowable variable names containing model evaluations"""
-            if variable not in model_variables:
-                raise ValueError(f"Expected one of {model_variables}; got '{variable}'")
-            affix = "_res_" if residuals else "_"
-            return [f"{variable}{affix}{model_name}" for model_name in model_ids]
-
-        # Identify which (if any) of ["F", "B_NEC", ...] are requested
-        model_variables_present = set(measurements).intersection(set(model_variables))
-        # Create the list of variable names to request
-        variables = []
-        for variable in model_variables_present:
-            if not residuals:
-                # Include "F" / "B_NEC" as requested...
-                variables.append(variable)
-            # Include e.g. "F_IGRF" / "B_NEC_IGRF" / "B_NEC_res_IGRF" etc.
-            variables.extend(_model_datavar_names(variable, residuals=residuals))
-        if models and (len(model_variables_present) == 0):
-            if residuals:
-                raise ValueError(
-                    f"""
-                    Residuals requested without one of {model_variables} set as measurements
-                    """
+        # Requested variables, start with the measurements ...
+        variables = set(measurements)
+
+        # model-related measurements
+        _requested_model_variables = [
+            variable for variable in measurements
+            if variable in model_variables
+        ]
+
+        if residuals:
+            # Remove the measurements ...
+            variables.difference_update(_requested_model_variables)
+            # ... add their residuals instead.
+            variables.update(
+                f"{variable}_res_{model_id}"
+                for variable in _requested_model_variables
+                for model_id in model_ids
+            )
+
+        else:
+            # If no variable is requested fall back to B_NEC.
+            if not _requested_model_variables:
+                _requested_model_variables = ["B_NEC"]
+
+            # Add calculated model variables.
+            variables.update(
+                f"{variable}_{model_id}"
+                for variable in (
+                    model_variables[variable]
+                    for variable in _requested_model_variables
                 )
-            # If "F" / "B_NEC" have not been requested, include e.g. "B_NEC_IGRF" etc.
-            variables.extend(_model_datavar_names("B_NEC"))
-        # Include all the non-model-related variables
-        variables.extend(list(set(measurements) - model_variables_present))
-        variables.extend(auxiliaries)
+                for model_id in model_ids
+            )
+
+        # Finally, add the auxiliary variables.
+        variables.update(auxiliaries)
+
         self._request_inputs.model_expression = model_expression_string
-        self._request_inputs.variables = variables
+        self._request_inputs.variables = list(variables)
         self._request_inputs.sampling_step = sampling_step
         self._request_inputs.custom_shc = custom_shc
         return self
@@ -1177,27 +1219,80 @@ def clear_range_filter(self):
         self._request_inputs.filters = None
         return self
 
-    def get_times_for_orbits(self, spacecraft, start_orbit, end_orbit):
+    def get_times_for_orbits(self, start_orbit, end_orbit, mission="Swarm", spacecraft=None):
         """Translate a pair of orbit numbers to a time interval.
 
         Args:
-            spacecraft (str): one of ('A','B','C') or
-                                ("Alpha", "Bravo", "Charlie")
             start_orbit (int): a starting orbit number
             end_orbit (int): a later orbit number
+            spacecraft (str):
+                    Swarm: one of ('A','B','C') or ("Alpha", "Bravo", "Charlie")
+                    GRACE: one of ('1','2')
+                    GRACE-FO: one of ('1','2')
+                    CryoSat-2: None
+            mission (str): one of ('Swarm', 'GRACE', 'GRACE-FO', 'CryoSat-2')
 
         Returns:
             tuple (datetime): (start_time, end_time) The start time of the
             start_orbit and the ending time of the end_orbit.
             (Based on ascending nodes of the orbits)
 
         """
+        # check old function signature and print warning
+        if (
+            isinstance(start_orbit, str) and
+            isinstance(mission, int) and
+            spacecraft is None
+        ):
+            spacecraft, start_orbit, end_orbit = start_orbit, end_orbit, mission
+            mission = "Swarm"
+            warn(
+                "The order of SwarmRequest.get_times_for_orbits() method's "
+                "parameters has changed!  "
+                "The backward compatibility will be removed in the future.  "
+                "Please change your code to:  "
+                "request.get_times_for_orbits(start_orbit, end_orbit, "
+                "'Swarm', spacecraft)",
+                FutureWarning,
+            )
+
+        start_orbit = int(start_orbit)
+        end_orbit = int(end_orbit)
+
+        if mission not in self.MISSION_SPACECRAFTS:
+            raise ValueError(
+                f"Invalid mission {mission}!"
+                f"Allowed options are: {','.join(self.MISSION_SPACECRAFTS)}"
+            )
+
         # Change to spacecraft = "A" etc. for this request
-        if spacecraft in ("Alpha", "Bravo", "Charlie"):
+        spacecraft = str(spacecraft) if spacecraft is not None else None
+        if mission == "Swarm" and spacecraft in ("Alpha", "Bravo", "Charlie"):
             spacecraft = spacecraft[0]
+
+        if self.MISSION_SPACECRAFTS[mission]:
+            # missions with required spacecraft id
+            if not spacecraft:
+                raise ValueError(
+                    f"The {mission} spacecraft is required!"
+                    f"Allowed options are: {','.join(self.MISSION_SPACECRAFTS[mission])}"
+                )
+            if spacecraft not in self.MISSION_SPACECRAFTS[mission]:
+                raise ValueError(
+                    f"Invalid {mission} spacecraft! "
+                    f"Allowed options are: {','.join(self.MISSION_SPACECRAFTS[mission])}"
+                )
+
+        elif spacecraft: # mission without spacecraft id
+            raise ValueError(
+                f"No {mission} spacecraft shall be specified! "
+                "Set spacecraft to None."
+            )
+
         templatefile = TEMPLATE_FILES["times_from_orbits"]
         template = JINJA2_ENVIRONMENT.get_template(templatefile)
         request = template.render(
+            mission=mission,
             spacecraft=spacecraft,
             start_orbit=start_orbit,
             end_orbit=end_orbit

viresclient/_data_handling.py

@@ -42,6 +42,8 @@
 
 CDF_EPOCH_1970 = 62167219200000.0
 
+ALLOWED_SPACECRFTS = ["A", "B", "C", "1", "2", "-"]
+
 # Frame names to use as xarray dimension names
 FRAME_NAMES = {
     "NEC": ["B_NEC", "B_OB", "B_CF", "B_SV", "sigma_OB", "sigma_CF", "sigma_SV"],
@@ -215,7 +217,7 @@ def as_xarray_dataset(self, reshape=False):
                     self._cdftime_to_datetime(self.get_variable("Timestamp"))})
         # Add Spacecraft variable as Categorical to save memory
         ds["Spacecraft"] = (("Timestamp",), pandas.Categorical(
-            self.get_variable("Spacecraft"), categories=["A", "B", "C", "-"]))
+            self.get_variable("Spacecraft"), categories=ALLOWED_SPACECRFTS))
         datanames = set(self.variables)
         datanames.remove("Timestamp")
         datanames.remove("Spacecraft")

viresclient/_wps/templates/vires_times_from_orbits.xml

@@ -2,12 +2,22 @@
 <wps:Execute xmlns:wps="http://www.opengis.net/wps/1.0.0" xmlns:ows="http://www.opengis.net/ows/1.1" version="1.0.0" service="WPS">
   <ows:Identifier>vires:get_orbit_timerange</ows:Identifier>
   <wps:DataInputs>
+    {% if mission -%}
+    <wps:Input>
+      <ows:Identifier>mission</ows:Identifier>
+      <wps:Data>
+        <wps:LiteralData>{{ mission }}</wps:LiteralData>
+      </wps:Data>
+    </wps:Input>
+    {% endif -%}
+    {% if spacecraft -%}
     <wps:Input>
       <ows:Identifier>spacecraft</ows:Identifier>
       <wps:Data>
         <wps:LiteralData>{{ spacecraft }}</wps:LiteralData>
       </wps:Data>
     </wps:Input>
+    {% endif -%}
     <wps:Input>
       <ows:Identifier>start_orbit</ows:Identifier>
       <wps:Data>