diff --git a/pyvo/mivot/features/sky_coord_builder.py b/pyvo/mivot/features/sky_coord_builder.py index 2d039a4c..2e5d0bd9 100644 --- a/pyvo/mivot/features/sky_coord_builder.py +++ b/pyvo/mivot/features/sky_coord_builder.py @@ -2,11 +2,12 @@ """ Utility transforming MIVOT annotation into SkyCoord instances """ - +import numbers from astropy.coordinates import SkyCoord from astropy import units as u from astropy.coordinates import ICRS, Galactic, FK4, FK5 -from pyvo.mivot.utils.exceptions import NoMatchingDMTypeError +from astropy.time.core import Time +from pyvo.mivot.utils.exceptions import NoMatchingDMTypeError, MappingError class MangoRoles: @@ -62,7 +63,9 @@ def __init__(self, mivot_instance_dict): def build_sky_coord(self): """ Build a SkyCoord instance from the MivotInstance dictionary. - The operation requires the dictionary to have ``mango:EpochPosition`` as dmtype + The operation requires the dictionary to have ``mango:EpochPosition`` as dmtype. + This instance can be either the root of the dictionary or it can be one + of the Mango properties if the root object is a mango:MangoObject instance This is a public method which could be extended to support other dmtypes. returns @@ -75,15 +78,28 @@ def build_sky_coord(self): NoMatchingDMTypeError if the SkyCoord instance cannot be built. """ - if self._mivot_instance_dict and self._mivot_instance_dict["dmtype"] == "mango:EpochPosition": + + if self._mivot_instance_dict and self._mivot_instance_dict["dmtype"] == "mango:MangoObject": + property_dock = self._mivot_instance_dict["propertyDock"] + for mango_property in property_dock: + if mango_property["dmtype"] == "mango:EpochPosition": + self._mivot_instance_dict = mango_property + return self._build_sky_coord_from_mango() + raise NoMatchingDMTypeError( + "No INSTANCE with dmtype='mango:EpochPosition' has been found:" + " in the property dock of the MangoObject, " + "cannot build a SkyCoord from annotations") + + elif self._mivot_instance_dict and self._mivot_instance_dict["dmtype"] == "mango:EpochPosition": return self._build_sky_coord_from_mango() raise NoMatchingDMTypeError( "No INSTANCE with dmtype='mango:EpochPosition' has been found:" " cannot build a SkyCoord from annotations") - def _set_year_time_format(self, hk_field, besselian=False): + def _get_time_instance(self, hk_field, besselian=False): """ Format a date expressed in year as [scale]year + - Exception possibly risen by Astropy are not caught parameters ---------- @@ -94,33 +110,96 @@ def _set_year_time_format(self, hk_field, besselian=False): returns ------- - string or None - attribute value formatted as [scale]year + Time instance or None + + raise + ----- + MappingError: if the Time instance cannot be built for some reason """ - scale = "J" if not besselian else "B" # Process complex type "mango:DateTime - # only "year" representation are supported yet if hk_field['dmtype'] == "mango:DateTime": representation = hk_field['representation']['value'] timestamp = hk_field['dateTime']['value'] - if representation == "year": - return f"{scale}{timestamp}" + # Process simple attribute + else: + representation = hk_field.get("unit") + timestamp = hk_field.get("value") + + if not representation or not timestamp: + raise MappingError(f"Cannot interpret field {hk_field} " + f"as a {('besselian' if besselian else 'julian')} timestamp") + + time_instance = self. _build_time_instance(timestamp, representation, besselian) + if not time_instance: + raise MappingError(f"Cannot build a Time instance from {hk_field}") + + return time_instance + + def _build_time_instance(self, timestamp, representation, besselian=False): + """ + Build a Time instance matching the input parameters. + - Returns None if the parameters do not allow any Time setup + - Exception possibly risen by Astropy are not caught at this level + + parameters + ---------- + timestamp: string or number + The timestamp must comply with the given representation + representation: string + year, iso, ... (See MANGO primitive types derived from ivoa:timeStamp) + besselian: boolean (optional) + Flag telling to use the besselain calendar. We assume it to only be + relevant for FK5 frame + returns + ------- + Time instance or None + """ + if representation in ["year", "yr", "y"]: + # it the timestamp is numeric, we infer its format from the besselian flag + if isinstance(timestamp, numbers.Number): + return Time(f"{('B' if besselian else 'J')}{timestamp}", + format=("byear_str" if besselian else "jyear_str")) + if besselian: + if timestamp.startswith("B"): + return Time(f"{timestamp}", format="byear_str") + elif timestamp.startswith("J"): + # a besselain year cannot be given as "Jxxxx" + return None + elif timestamp.isnumeric(): + # we force the string representation not to break the test assertions + return Time(f"B{timestamp}", format="byear_str") + else: + if timestamp.startswith("J"): + return Time(f"{timestamp}", format="jyear_str") + elif timestamp.startswith("B"): + # a julian year cannot be given as "Bxxxx" + return None + elif timestamp.isnumeric(): + # we force the string representation not to break the test assertions + return Time(f"J{timestamp}", format="jyear_str") + # no case matches return None - return (f"{scale}{hk_field['value']}" if hk_field["unit"] in ("yr", "year") - else hk_field["value"]) + # in the following cases, the calendar (B or J) is givent by the besselian flag + # We force to use the string representation to avoid breaking unit tests. + elif representation == "mjd": + time = Time(f"{timestamp}", format="mjd") + return (Time(time.byear_str) if besselian else time) + elif representation == "jd": + time = Time(f"{timestamp}", format="jd") + return (Time(time.byear_str) if besselian else time) + elif representation == "iso": + time = Time(f"{timestamp}", format="iso") + return (Time(time.byear_str) if besselian else time) + + return None - def _get_space_frame(self, obstime=None): + def _get_space_frame(self): """ Build an astropy space frame instance from the MIVOT annotations. - Equinox are supported for FK4/5 - Reference location is not supported - parameters - ---------- - obstime: str - Observation time is given to the space frame builder (this method) because - it must be set by the coordinate system constructor in case of FK4 frame. returns ------- FK2, FK5, ICRS or Galactic @@ -133,14 +212,15 @@ def _get_space_frame(self, obstime=None): if frame == 'fk4': self._map_coord_names = skycoord_param_default if "equinox" in coo_sys: - equinox = self._set_year_time_format(coo_sys["equinox"], True) - return FK4(equinox=equinox, obstime=obstime) + equinox = self._get_time_instance(coo_sys["equinox"], True) + # by FK4 takes obstime=equinox by default + return FK4(equinox=equinox) return FK4() if frame == 'fk5': self._map_coord_names = skycoord_param_default if "equinox" in coo_sys: - equinox = self._set_year_time_format(coo_sys["equinox"]) + equinox = self._get_time_instance(coo_sys["equinox"]) return FK5(equinox=equinox) return FK5() @@ -153,9 +233,7 @@ def _get_space_frame(self, obstime=None): def _build_sky_coord_from_mango(self): """ - Build silently a SkyCoord instance from the ``mango:EpochPosition instance``. - No error is trapped, unconsistencies in the ``mango:EpochPosition`` instance will - raise Astropy errors. + Build a SkyCoord instance from the ``mango:EpochPosition instance``. - The epoch (obstime) is meant to be given in year. - ICRS frame is taken by default @@ -170,26 +248,29 @@ def _build_sky_coord_from_mango(self): kwargs = {} kwargs["frame"] = self._get_space_frame() - for key, value in self._map_coord_names.items(): - # ignore not set parameters - if key not in self._mivot_instance_dict: + for mango_role, skycoord_field in self._map_coord_names.items(): + # ignore not mapped parameters + if mango_role not in self._mivot_instance_dict: continue - hk_field = self._mivot_instance_dict[key] - # format the observation time (J-year by default) - if value == "obstime": - # obstime must be set into the KK4 frame but not as an input parameter - fobstime = self._set_year_time_format(hk_field) - if isinstance(kwargs["frame"], FK4): - kwargs["frame"] = self._get_space_frame(obstime=fobstime) + hk_field = self._mivot_instance_dict[mango_role] + if mango_role == "obsDate": + besselian = isinstance(kwargs["frame"], FK4) + fobstime = self._get_time_instance(hk_field, + besselian=besselian) + # FK4 class has an obstime attribute which must be set at instanciation time + if besselian: + kwargs["frame"] = FK4(equinox=kwargs["frame"].equinox, obstime=fobstime) + # This is not the case for any other space frames else: - kwargs[value] = fobstime - # Convert the parallax (mango) into a distance - elif value == "distance": - kwargs[value] = (hk_field["value"] - * u.Unit(hk_field["unit"]).to(u.parsec, equivalencies=u.parallax())) - kwargs[value] = kwargs[value] * u.parsec - elif "unit" in hk_field and hk_field["unit"]: - kwargs[value] = hk_field["value"] * u.Unit(hk_field["unit"]) - else: - kwargs[value] = hk_field["value"] + kwargs[skycoord_field] = fobstime + # ignore not set parameters + elif (hk_value := hk_field["value"]) is not None: + # Convert the parallax (mango) into a distance + if skycoord_field == "distance": + kwargs[skycoord_field] = ( + (hk_value * u.Unit(hk_field["unit"])).to(u.parsec, equivalencies=u.parallax())) + elif "unit" in hk_field and hk_field["unit"]: + kwargs[skycoord_field] = hk_value * u.Unit(hk_field["unit"]) + else: + kwargs[skycoord_field] = hk_value return SkyCoord(**kwargs) diff --git a/pyvo/mivot/seekers/annotation_seeker.py b/pyvo/mivot/seekers/annotation_seeker.py index 5c0d718a..307843f6 100644 --- a/pyvo/mivot/seekers/annotation_seeker.py +++ b/pyvo/mivot/seekers/annotation_seeker.py @@ -155,9 +155,11 @@ def get_templates_tableref(self): def get_templates(self): """ Return a list of TEMPLATES @tableref. + Returns ------- - list: TEMPLATES tablerefs + [string] + tablerefs of all TEMPLATES elements """ templates_found = [] eset = XPath.x_path(self._xml_block, ".//" + Ele.TEMPLATES) @@ -170,19 +172,26 @@ def get_templates(self): def get_templates_block(self, tableref): """ - Return the TEMPLATES mapping block of the table matching @tableref. - If tableref is None returns all values of templates_blocks. + Return the TEMPLATES mapping block of the table identified @tableref. + If tableref is None or equals to Constant.FIRST_TABLE, return the first TEMPLATES. + Parameters ---------- tableref (str): @tableref of the searched TEMPLATES + Returns ------- - dict: TEMPLATES tablerefs and their mapping blocks {'tableref': mapping_block, ...} + XML element: matching TEMPLATES block or None """ # one table: name forced to DEFAULT or take the first if tableref is None or tableref == Constant.FIRST_TABLE: for _, tmpl in self._templates_blocks.items(): return tmpl + + if tableref not in self._templates_blocks: + raise MivotError( + "No TEMPLATES with tableref=" + tableref) + return self._templates_blocks[tableref] """ @@ -191,6 +200,7 @@ def get_templates_block(self, tableref): def get_instance_dmtypes(self): """ Get @dmtypes of all mapped instances + Returns ------- dict: @dmtypes of all mapped instances {GLOBALS: [], TEMPLATES: {}} diff --git a/pyvo/mivot/tests/__init__.py b/pyvo/mivot/tests/__init__.py index eb1d89d1..37810e4f 100644 --- a/pyvo/mivot/tests/__init__.py +++ b/pyvo/mivot/tests/__init__.py @@ -31,7 +31,8 @@ def check_output(self, want, got): bool True if the two XML outputs are equal, False otherwise. """ - return self._format_xml(want.strip()) == self._format_xml(got.strip()) + return (self._format_xml(want.strip()) + == self._format_xml(got.strip())) def output_difference(self, want, got): """ @@ -121,7 +122,8 @@ def assertXmltreeEqualsFile(xmltree1, xmltree2_file): The path to the file containing the second XML tree. """ xmltree2 = XMLOutputChecker.xmltree_from_file(xmltree2_file).getroot() - xml_str1 = etree.tostring(xmltree1).decode("utf-8") - xml_str2 = etree.tostring(xmltree2).decode("utf-8") + xml_str1 = etree.tostring(xmltree1).decode("utf-8").strip() + xml_str2 = etree.tostring(xmltree2).decode("utf-8").strip() checker = XMLOutputChecker() + assert checker.check_output(xml_str1, xml_str2), f"XML trees differ:\n{xml_str1}\n---\n{xml_str2}" diff --git a/pyvo/mivot/tests/data/reference/annotation_seeker.0.2.xml b/pyvo/mivot/tests/data/reference/annotation_seeker.0.2.xml index 8b60bf29..d7f39109 100644 --- a/pyvo/mivot/tests/data/reference/annotation_seeker.0.2.xml +++ b/pyvo/mivot/tests/data/reference/annotation_seeker.0.2.xml @@ -1,6 +1,6 @@ - + diff --git a/pyvo/mivot/tests/data/simbad-cone-mivot.xml b/pyvo/mivot/tests/data/simbad-cone-mivot.xml new file mode 100644 index 00000000..e101aa0e --- /dev/null +++ b/pyvo/mivot/tests/data/simbad-cone-mivot.xml @@ -0,0 +1,160 @@ + + + +IVOID of the service specification +Used access protocol and version +Data publisher +HTTP request URL +Query execution date +Publisher email address +Successful query + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Distance (in degrees) between this object and the cone-search's target. + + +Main identifier for an object + + + +Right ascension + + +Declination + + +Object type + + + + +Coordinate error major axis + + +Coordinate error minor axis + + +Coordinate error angle + + +Proper motion in RA + + +Proper motion in DEC + + +Proper motion error major axis + + +Proper motion error minor axis + + +Proper motion error angle + + +Parallax + + +Radial Velocity + + +Angular size major axis + + +Angular size minor axis + + +Galaxy ellipse angle + + +MK spectral type + + +Morphological type + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
1.0409523503457668E-5NAME Barnard's Star c269.452076958619004.69336496657667Planet 17 57 48.4984700683 +04 41 36.1138796750.02620.029090-801.55110362.3940.0320.03690546.97591
+Truncated result + + diff --git a/pyvo/mivot/tests/data/test.instance_multiple.xml b/pyvo/mivot/tests/data/test.instance_multiple.xml new file mode 100644 index 00000000..de0349e6 --- /dev/null +++ b/pyvo/mivot/tests/data/test.instance_multiple.xml @@ -0,0 +1,186 @@ + + + + + + + + + Automatically annotated by XTAPDB + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the flux of the energy band number 1 of the ep camera in sc + + + the flux of the energy band number 2 of the ep camera in sc + + + the flux of the energy band number 3 of the ep camera in sc + + + + + + + + + + + + + + + +
0.00.10.2
1.02.13.2
+ + diff --git a/pyvo/mivot/tests/data/test.mivot_viewer.first_instance.xml b/pyvo/mivot/tests/data/test.mivot_viewer.first_instance.xml index 2e90f7f4..406af0e9 100644 --- a/pyvo/mivot/tests/data/test.mivot_viewer.first_instance.xml +++ b/pyvo/mivot/tests/data/test.mivot_viewer.first_instance.xml @@ -12,20 +12,11 @@ - - + - - - - - - - - - + diff --git a/pyvo/mivot/tests/test_annotation_seeker.py b/pyvo/mivot/tests/test_annotation_seeker.py index fd741eed..26d2a6d1 100644 --- a/pyvo/mivot/tests/test_annotation_seeker.py +++ b/pyvo/mivot/tests/test_annotation_seeker.py @@ -12,7 +12,7 @@ from pyvo.mivot.utils.dict_utils import DictUtils from pyvo.mivot.version_checker import check_astropy_version from pyvo.mivot.viewer import MivotViewer -from . import XMLOutputChecker +from pyvo.mivot.tests import XMLOutputChecker @pytest.fixture @@ -24,6 +24,14 @@ def a_seeker(): return AnnotationSeeker(m_viewer._mapping_block) +@pytest.fixture +def a_multiple_seeker(): + m_viewer = MivotViewer( + get_pkg_data_filename("data/test.instance_multiple.xml"), + tableref="Results") + return AnnotationSeeker(m_viewer._mapping_block) + + @pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") def test_multiple_templates(): """ @@ -87,3 +95,9 @@ def test_all_reverts(a_seeker): assert a_seeker.get_globals_instance_from_collection( "_CoordinateSystems", "ICRS").get("dmtype") == "coords:SpaceSys" assert a_seeker.get_globals_instance_from_collection("wrong_dmid", "ICRS") is None + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_multiple_seeker(a_multiple_seeker): + assert (a_multiple_seeker.get_instance_dmtypes()["TEMPLATES"] + == {"Results": ["mango:Brightness", "mango:Brightness", "mango:Brightness"]}) diff --git a/pyvo/mivot/tests/test_mango_annoter.py b/pyvo/mivot/tests/test_mango_annoter.py index 79d4c4cf..f01ccbda 100644 --- a/pyvo/mivot/tests/test_mango_annoter.py +++ b/pyvo/mivot/tests/test_mango_annoter.py @@ -152,8 +152,7 @@ def test_all_properties(): add_color(builder) add_photometry(builder) add_epoch_positon(builder) - builder.pack_into_votable() - XmlUtils.pretty_print(builder._annotation.mivot_block) + builder.pack_into_votable(schema_check=False) assert XmlUtils.strip_xml(builder._annotation.mivot_block) == ( XmlUtils.strip_xml(get_pkg_data_contents("data/reference/mango_object.xml")) ) @@ -171,7 +170,7 @@ def test_extraction_from_votable_header(): builder.extract_data_origin() epoch_position_mapping = builder.extract_epoch_position_parameters() builder.add_mango_epoch_position(**epoch_position_mapping) - builder.pack_into_votable() + builder.pack_into_votable(schema_check=False) assert XmlUtils.strip_xml(builder._annotation.mivot_block) == ( XmlUtils.strip_xml(get_pkg_data_contents("data/reference/test_header_extraction.xml")) ) diff --git a/pyvo/mivot/tests/test_mivot_instance.py b/pyvo/mivot/tests/test_mivot_instance.py index 3cc3035b..e3f89438 100644 --- a/pyvo/mivot/tests/test_mivot_instance.py +++ b/pyvo/mivot/tests/test_mivot_instance.py @@ -4,14 +4,9 @@ @author: michel ''' -import os import pytest from astropy.table import Table -from astropy.utils.data import get_pkg_data_filename -from pyvo.mivot.version_checker import check_astropy_version from pyvo.mivot.viewer.mivot_instance import MivotInstance -from pyvo.mivot.utils.mivot_utils import MivotUtils -from pyvo.mivot.viewer import MivotViewer fake_hk_dict = { "dmtype": "EpochPosition", @@ -103,22 +98,6 @@ } -@pytest.fixture -def m_viewer(): - data_path = get_pkg_data_filename(os.path.join("data", - "test.mivot_viewer.xml") - ) - return MivotViewer(data_path, tableref="Results") - - -@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") -def test_xml_viewer(m_viewer): - - xml_instance = m_viewer.xml_viewer.view - dm_instance = MivotInstance(**MivotUtils.xml_to_dict(xml_instance)) - assert dm_instance.to_dict() == test_dict - - def test_mivot_instance_constructor(): """Test the class generation from a dict.""" mivot_object = MivotInstance(**fake_hk_dict) diff --git a/pyvo/mivot/tests/test_mivot_instance_generation.py b/pyvo/mivot/tests/test_mivot_instance_generation.py deleted file mode 100644 index 847005cf..00000000 --- a/pyvo/mivot/tests/test_mivot_instance_generation.py +++ /dev/null @@ -1,183 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -""" -Test for mivot.viewer.model_viewer_level3.py and mivot.viewer.mivot_time.py -""" -import os -import pytest -from urllib.request import urlretrieve -from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot.viewer import MivotViewer -from pyvo.mivot.utils.mivot_utils import MivotUtils - - -@pytest.mark.remote_data -@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") -def test_model_viewer3(votable_test, simple_votable): - """ - Recursively compare an XML element with an element of MIVOT - class with the function recursive_xml_check. - This test run on 2 votables : votable_test and simple_votable. - """ - m_viewer_simple_votable = MivotViewer(votable_path=simple_votable) - MivotInstance = m_viewer_simple_votable.dm_instance - xml_simple_votable = m_viewer_simple_votable.xml_view - assert xml_simple_votable.tag == 'TEMPLATES' - recusive_xml_check(xml_simple_votable, MivotInstance) - m_viewer_votable_test = MivotViewer(votable_path=votable_test) - m_viewer_votable_test.next_row_view() - mivot_instance = m_viewer_votable_test.dm_instance - xml_votable_test = m_viewer_votable_test.xml_view - assert xml_simple_votable.tag == 'TEMPLATES' - recusive_xml_check(xml_votable_test, mivot_instance) - - -@pytest.mark.remote_data -@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") -def recusive_xml_check(xml_simple_votable, MivotInstance): - if xml_simple_votable.tag == 'TEMPLATES': - recusive_xml_check(xml_simple_votable[0], MivotInstance) - else: - for child in xml_simple_votable: - if child.tag == 'INSTANCE': - for key, value in child.attrib.items(): - if key == 'dmrole': - if value == '': - if child.tag == 'ATTRIBUTE': - recusive_xml_check(child, - getattr(MivotInstance, - MivotInstance._remove_model_name( - child.get('dmrole')))) - elif child.tag == 'INSTANCE': - recusive_xml_check(child, getattr(MivotInstance, - MivotInstance._remove_model_name - (child.get('dmrole')))) - else: - if child.tag == 'ATTRIBUTE': - recusive_xml_check(child, getattr(MivotInstance, - MivotInstance._remove_model_name( - child.get('dmrole')))) - elif child.tag == 'INSTANCE': - recusive_xml_check(child, getattr(MivotInstance, - MivotInstance._remove_model_name( - child.get('dmrole')))) - elif child.tag == 'COLLECTION': - recusive_xml_check(child, getattr(MivotInstance, - MivotInstance._remove_model_name( - child.get('dmrole')))) - elif child.tag == 'COLLECTION': - for key, value in child.attrib.items(): - assert len(getattr(MivotInstance, - MivotInstance._remove_model_name(child.get('dmrole')))) == len(child) - i = 0 - for child2 in child: - recusive_xml_check(child2, getattr(MivotInstance, MivotInstance._remove_model_name - (child.get('dmrole')))[i]) - i += 1 - elif child.tag == 'ATTRIBUTE': - MivotInstance_attribute = getattr(MivotInstance, - MivotInstance._remove_model_name(child.get('dmrole'))) - for key, value in child.attrib.items(): - if key == 'dmtype': - assert MivotInstance_attribute.dmtype in value - elif key == 'value': - if (MivotInstance_attribute.value is not None - and not isinstance(MivotInstance_attribute.value, bool)): - if isinstance(MivotInstance_attribute.value, float): - pytest.approx(float(value), MivotInstance_attribute.value, 0.0001) - else: - assert value == MivotInstance_attribute.value - elif child.tag.startswith("REFERENCE"): - # Viewer not in resolve_ref mode: REFRENCEs are not filtered - pass - else: - print(child.tag) - assert False - - -@pytest.mark.remote_data -@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") -def test_dict_model_viewer3(votable_test, simple_votable): - """ - To test the generation of the MIVOT class, the function builds a ModelViewerLevel3 - with his MIVOT class and his previous dictionary from XML. - Then, it calls the function recursive_check which recursively compares an element of MIVOT class - with the dictionary on which it was built. - MIVOT class is itself a dictionary with only essential information of the ModelViewerLevel3._dict. - This test run on 2 votables : votable_test and simple_votable. - """ - m_viewer_votable_test = MivotViewer(votable_path=votable_test) - m_viewer_votable_test.next_row_view() - mivot_instance = m_viewer_votable_test.dm_instance - _dict = MivotUtils.xml_to_dict(m_viewer_votable_test.xml_viewer.view) - recursive_check(mivot_instance, **_dict) - mivot_instance = m_viewer_votable_test.dm_instance - _dict = MivotUtils.xml_to_dict(m_viewer_votable_test.xml_view) - recursive_check(mivot_instance, **_dict) - - -def recursive_check(MivotInstance, **kwargs): - for key, value in kwargs.items(): - # the root instance ha no role: this makes an empty value in the unpacked dict - if key == '': - continue - if isinstance(value, list): - nbr_item = 0 - for item in value: - if isinstance(item, dict): - assert 'dmtype' in item.keys() - recursive_check(getattr(MivotInstance, - MivotInstance._remove_model_name(key))[nbr_item], - **item - ) - nbr_item += 1 - elif isinstance(value, dict) and 'value' not in value: - # for INSTANCE of INSTANCEs dmrole needs model_name - assert MivotInstance._remove_model_name(key, True) in vars(MivotInstance).keys() - recursive_check(getattr(MivotInstance, MivotInstance._remove_model_name(key, True)), **value) - else: - if isinstance(value, dict) and MivotInstance._is_leaf(**value): - assert value.keys().__contains__('dmtype' and 'value' and 'unit' and 'ref') - lower_dmtype = value['dmtype'].lower() - if "real" in lower_dmtype or "double" in lower_dmtype or "float" in lower_dmtype: - assert isinstance(value['value'], float) - elif "bool" in lower_dmtype: - assert isinstance(value['value'], bool) - elif value['dmtype'] is None: - assert (value['value'] in - ('notset', 'noset', 'null', 'none', 'NotSet', 'NoSet', 'Null', 'None')) - else: - if value['value'] is not None: - assert isinstance(value['value'], str) - recursive_check(getattr(MivotInstance, MivotInstance._remove_model_name(key)), **value) - else: - assert key == 'dmtype' or 'value' - - -@pytest.fixture -def votable_test(data_path, data_sample_url): - votable_name = "vizier_csc2_gal.annot.xml" - votable_path = os.path.join(data_path, "data", votable_name) - urlretrieve(data_sample_url + votable_name, - votable_path) - yield votable_path - os.remove(votable_path) - - -@pytest.fixture -def simple_votable(data_path, data_sample_url): - votable_name = "simple-annotation-votable.xml" - votable_path = os.path.join(data_path, "data", votable_name) - urlretrieve(data_sample_url + votable_name, - votable_path) - yield votable_path - os.remove(votable_path) - - -@pytest.fixture -def data_path(): - return os.path.dirname(os.path.realpath(__file__)) - - -@pytest.fixture -def data_sample_url(): - return "https://raw.githubusercontent.com/ivoa/dm-usecases/main/pyvo-ci-sample/" diff --git a/pyvo/mivot/tests/test_mivot_viewer.py b/pyvo/mivot/tests/test_mivot_viewer.py index ff79da42..1447f5e6 100644 --- a/pyvo/mivot/tests/test_mivot_viewer.py +++ b/pyvo/mivot/tests/test_mivot_viewer.py @@ -13,6 +13,153 @@ from pyvo.mivot.viewer import MivotViewer from astropy import version as astropy_version +dm_raw_instances = [ + { + "dmrole": "", + "dmtype": "mango:Brightness", + "value": { + "dmtype": "ivoa:RealQuantity", + "value": None, + "unit": None, + "ref": "SC_EP_1_FLUX", + }, + }, + { + "dmrole": "", + "dmtype": "mango:Brightness", + "value": { + "dmtype": "ivoa:RealQuantity", + "value": None, + "unit": None, + "ref": "SC_EP_2_FLUX", + }, + }, + { + "dmrole": "", + "dmtype": "mango:Brightness", + "value": { + "dmtype": "ivoa:RealQuantity", + "value": None, + "unit": None, + "ref": "SC_EP_3_FLUX", + }, + }, +] + +globals_photcal = { + "dmid": "CoordSystem_XMM_EB1_id", + "dmtype": "Phot:PhotCal", + "identifier": { + "dmtype": "ivoa:string", + "value": "XMM/EPIC/EB1", + "unit": None, + "ref": None, + }, + "magnitudeSystem": { + "dmrole": "Phot:PhotCal.magnitudeSystem", + "dmtype": "Phot:MagnitudeSystem", + "type": { + "dmtype": "Phot:TypeOfMagSystem", + "value": "XMM", + "unit": None, + "ref": None, + }, + "referenceSpectrum": { + "dmtype": "ivoa:anyURI", + "value": "https://xmm-tools.cosmos.esa.int/external" + "/xmm_user_support/documentation/sas_usg/USG/SASUSG.html", + "unit": None, + "ref": None, + }, + }, + "photometryFilter": { + "dmid": "CoordSystem_XMM_FILTER_EB1_id", + "dmtype": "Phot:PhotometryFilter", + "dmrole": "Phot:PhotCal.photometryFilter", + "identifier": { + "dmtype": "ivoa:string", + "value": "XMM/EPIC/EB1", + "unit": None, + "ref": None, + }, + "name": { + "dmtype": "ivoa:string", + "value": "XMM EPIC EB1", + "unit": None, + "ref": None, + }, + "description": { + "dmtype": "ivoa:string", + "value": "Soft", + "unit": None, + "ref": None, + }, + "bandName": { + "dmtype": "ivoa:string", + "value": "EB1", + "unit": None, + "ref": None, + }, + "spectralLocation": { + "dmrole": "Phot:PhotometryFilter.spectralLocation", + "dmtype": "Phot:SpectralLocation", + "ucd": { + "dmtype": "Phot:UCD", + "value": "em.wl.effective", + "unit": None, + "ref": None, + }, + "unitexpression": { + "dmtype": "ivoa:Unit", + "value": "keV", + "unit": None, + "ref": None, + }, + "value": {"dmtype": "ivoa:real", "value": 0.35, "unit": None, "ref": None}, + }, + "bandwidth": { + "dmrole": "Phot:PhotometryFilter.bandwidth", + "dmtype": "Phot:Bandwidth", + "ucd": { + "dmtype": "Phot:UCD", + "value": "instr.bandwidth;stat.fwhm", + "unit": None, + "ref": None, + }, + "unitexpression": { + "dmtype": "ivoa:Unit", + "value": "keV", + "unit": None, + "ref": None, + }, + "extent": {"dmtype": "ivoa:real", "value": 0.3, "unit": None, "ref": None}, + "start": {"dmtype": "ivoa:real", "value": 0.2, "unit": None, "ref": None}, + "stop": {"dmtype": "ivoa:real", "value": 0.5, "unit": None, "ref": None}, + }, + "transmissionCurve": { + "dmrole": "Phot:PhotometryFilter.transmissionCurve", + "dmtype": "Phot:TransmissionCurve", + "access": { + "dmrole": "Phot:TransmissionCurve.access", + "dmtype": "Phot:Access", + "reference": { + "dmtype": "ivoa:anyURI", + "value": "https://xmm-tools.cosmos.esa.int/external/xmm_user_support" + "/documentation/sas_usg/USG/SASUSG.html", + "unit": None, + "ref": None, + }, + "format": { + "dmtype": "ivoa:string", + "value": "text/html", + "unit": None, + "ref": None, + }, + }, + }, + }, +} + @pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") def test_get_first_instance_dmtype(path_to_first_instance): @@ -22,11 +169,11 @@ def test_get_first_instance_dmtype(path_to_first_instance): """ m_viewer = MivotViewer(votable_path=path_to_first_instance) assert m_viewer.get_first_instance_dmtype("one_instance") == "one_instance" - assert m_viewer.get_first_instance_dmtype("coll_and_instances") == "first" - assert m_viewer.get_first_instance_dmtype("one_collection") == Constant.ROOT_COLLECTION - assert m_viewer.get_first_instance_dmtype("only_collection") == Constant.ROOT_COLLECTION - with pytest.raises(Exception, match="Can't find the first INSTANCE/COLLECTION in TEMPLATES"): + assert m_viewer.get_first_instance_dmtype("some_instances") == "first" + with pytest.raises(Exception, match="Can't find the first INSTANCE in TEMPLATES"): m_viewer.get_first_instance_dmtype("empty") + with pytest.raises(Exception, match="No TEMPLATES with tableref=not_existing_tableref"): + m_viewer.get_first_instance_dmtype("not_existing_tableref") @pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") @@ -77,7 +224,7 @@ def test_global_getters(m_viewer): @pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") def test_no_mivot(path_no_mivot): """ - Test each getter for GLOBALS of the model_viewer specific . + Test the viewer behavior when there is no mapping """ m_viewer = MivotViewer(path_no_mivot) assert m_viewer.get_table_ids() is None @@ -92,6 +239,57 @@ def test_no_mivot(path_no_mivot): assert m_viewer.next_table_row() is None +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_instance_mutiple_in_templates(path_to_multiple_instance): + """ + Test case with a TEMPLATES containing multiple instances + """ + m_viewer = MivotViewer(votable_path=path_to_multiple_instance) + instance_dict = [] + # test the DM instances children of TEMPLATES before their values are set + for dmi in m_viewer.dm_instances: + instance_dict.append(dmi.to_hk_dict()) + assert instance_dict == dm_raw_instances + + # test the DM instances children of TEMPLATES set with the values of the first row + m_viewer.next_row_view() + row_values = [] + for dmi in m_viewer.dm_instances: + row_values.append(dmi.value.value) + assert row_values == pytest.approx([0.0, 0.1, 0.2], rel=1e-3) + + # test the DM instances children of TEMPLATES set with the values of the second row + m_viewer.next_row_view() + row_values = [] + for dmi in m_viewer.dm_instances: + row_values.append(dmi.value.value) + assert row_values == pytest.approx([1.0, 2.1, 3.2], rel=1e-3) + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_globals_instances(path_to_multiple_instance): + """ + Test case for the GLOBALS instance access as MivotInstances + """ + m_viewer = MivotViewer(votable_path=path_to_multiple_instance) + instance_dict = [] + photcals = 0 + photfilters = 0 + # test the DM instances children of TEMPLATES before their values are set + for dmi in m_viewer.dm_globals_instances: + if dmi.dmtype == "Phot:PhotCal": + photcals += 1 + elif dmi.dmtype == "Phot:PhotometryFilter": + photfilters += 1 + else: + assert False, f"Unexpected dmtype {dmi.dmtype} in GLOBALS " + instance_dict.append(dmi.to_hk_dict()) + assert photcals == 3 + assert photfilters == 3 + # just check the first one + assert instance_dict[0] == globals_photcal + + def test_check_version(path_to_viewer): if not check_astropy_version(): with pytest.raises(Exception, @@ -125,8 +323,16 @@ def path_to_viewer(): return get_pkg_data_filename(os.path.join("data", votable_name)) +@pytest.fixture +def path_to_multiple_instance(): + + votable_name = "test.instance_multiple.xml" + return get_pkg_data_filename(os.path.join("data", votable_name)) + + @pytest.fixture def path_to_first_instance(): + votable_name = "test.mivot_viewer.first_instance.xml" return get_pkg_data_filename(os.path.join("data", votable_name)) diff --git a/pyvo/mivot/tests/test_sky_coord_builder.py b/pyvo/mivot/tests/test_sky_coord_builder.py index b38062a4..5ddbb04c 100644 --- a/pyvo/mivot/tests/test_sky_coord_builder.py +++ b/pyvo/mivot/tests/test_sky_coord_builder.py @@ -7,10 +7,16 @@ for the output of this service. ''' import pytest +from astropy.utils.data import get_pkg_data_filename from pyvo.mivot.version_checker import check_astropy_version from pyvo.mivot.viewer.mivot_instance import MivotInstance from pyvo.mivot.features.sky_coord_builder import SkyCoordBuilder from pyvo.mivot.utils.exceptions import NoMatchingDMTypeError +from pyvo.mivot.viewer.mivot_viewer import MivotViewer +from pyvo.utils import activate_features + +# Enable MIVOT-specific features in the pyvo library +activate_features("MIVOT") # annotations generated by Vizier as given to the MivotInstance vizier_dict = { @@ -218,7 +224,7 @@ def test_vizier_output(): @pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") def test_vizier_output_with_equinox_and_parallax(): - """Test the SkyCoord issued from the modofier Vizier response * + """Test the SkyCoord issued from the modified Vizier response * (parallax added and FK5 + Equinox frame) """ mivot_instance = MivotInstance(**vizier_equin_dict) @@ -226,13 +232,30 @@ def test_vizier_output_with_equinox_and_parallax(): scoo = scb.build_sky_coord() assert (str(scoo).replace("\n", "").replace(" ", "") == "") vizier_equin_dict["spaceSys"]["frame"]["spaceRefFrame"]["value"] = "FK4" mivot_instance = MivotInstance(**vizier_equin_dict) scoo = mivot_instance.get_SkyCoord() assert (str(scoo).replace("\n", "").replace(" ", "") - == "") + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_simad_cs_output(): + """Test the SkyCoord issued from a Simbad SCS response + """ + filename = get_pkg_data_filename('data/simbad-cone-mivot.xml') + m_viewer = MivotViewer(filename, resolve_ref=True) + mivot_instance = m_viewer.dm_instance + scb = SkyCoordBuilder(mivot_instance.to_dict()) + scoo = scb.build_sky_coord() + + assert (str(scoo).replace("\n", "").replace(" ", "") + == "") + assert str(scoo.obstime) == "J2000.000" diff --git a/pyvo/mivot/tests/test_user_api.py b/pyvo/mivot/tests/test_user_api.py index 44b010b4..39de509f 100644 --- a/pyvo/mivot/tests/test_user_api.py +++ b/pyvo/mivot/tests/test_user_api.py @@ -15,7 +15,6 @@ from pyvo.mivot.viewer import MivotViewer from astropy.io.votable import parse - ref_ra = [ 0.04827189, 0.16283175, diff --git a/pyvo/mivot/tests/test_xml_viewer.py b/pyvo/mivot/tests/test_xml_viewer.py deleted file mode 100644 index 1a0cd713..00000000 --- a/pyvo/mivot/tests/test_xml_viewer.py +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -""" -Test for mivot.viewer.model_viewer_level2.py -""" -import pytest -try: - from defusedxml.ElementTree import Element as element -except ImportError: - from xml.etree.ElementTree import Element as element -from astropy.utils.data import get_pkg_data_filename -from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot.viewer import MivotViewer -from pyvo.mivot.utils.exceptions import MivotError - - -@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") -def test_xml_viewer(m_viewer): - - m_viewer.next_row_view() - xml_viewer = m_viewer.xml_viewer - with pytest.raises(MivotError, - match="Cannot find dmrole wrong_role in any instances of the VOTable"): - xml_viewer.get_instance_by_role("wrong_role") - - with pytest.raises(MivotError, - match="Cannot find dmrole wrong_role in any instances of the VOTable"): - xml_viewer.get_instance_by_role("wrong_role", all_instances=True) - - with pytest.raises(MivotError, - match="Cannot find dmtype wrong_dmtype in any instances of the VOTable"): - xml_viewer.get_instance_by_type("wrong_dmtype") - - with pytest.raises(MivotError, - match="Cannot find dmtype wrong_dmtype in any instances of the VOTable"): - xml_viewer.get_instance_by_type("wrong_dmtype", all_instances=True) - - with pytest.raises(MivotError, - match="Cannot find dmrole wrong_role in any collections of the VOTable"): - xml_viewer.get_collection_by_role("wrong_role") - - with pytest.raises(MivotError, - match="Cannot find dmrole wrong_role in any collections of the VOTable"): - xml_viewer.get_collection_by_role("wrong_role", all_instances=True) - - instances_list_role = xml_viewer.get_instance_by_role("cube:MeasurementAxis.measure") - assert isinstance(instances_list_role, element) - - instances_list_role = xml_viewer.get_instance_by_role("cube:MeasurementAxis.measure", all_instances=True) - assert len(instances_list_role) == 3 - - instances_list_type = xml_viewer.get_instance_by_type("cube:Observable") - assert isinstance(instances_list_type, element) - - instances_list_type = xml_viewer.get_instance_by_type("cube:Observable", all_instances=True) - assert len(instances_list_type) == 3 - - collections_list_role = xml_viewer.get_collection_by_role("cube:NDPoint.observable") - assert isinstance(collections_list_role, element) - - collections_list_role = xml_viewer.get_collection_by_role("cube:NDPoint.observable", all_instances=True) - assert len(collections_list_role) == 1 - - -@pytest.fixture -def m_viewer(): - return MivotViewer(get_pkg_data_filename("data/test.mivot_viewer.xml"), - tableref="Results") diff --git a/pyvo/mivot/utils/vocabulary.py b/pyvo/mivot/utils/vocabulary.py index a7ec58b0..b03a7802 100644 --- a/pyvo/mivot/utils/vocabulary.py +++ b/pyvo/mivot/utils/vocabulary.py @@ -13,8 +13,6 @@ class Constant: FIRST_TABLE = "first_table" FIELD_UNIT = "field_unit" COL_INDEX = "col_index" - ROOT_COLLECTION = "root_collection" - ROOT_OBJECT = "root_object" NOT_SET = "NotSet" ANONYMOUS_TABLE = "AnonymousTable" diff --git a/pyvo/mivot/viewer/mivot_viewer.py b/pyvo/mivot/viewer/mivot_viewer.py index 1a87f328..739d2d70 100644 --- a/pyvo/mivot/viewer/mivot_viewer.py +++ b/pyvo/mivot/viewer/mivot_viewer.py @@ -42,7 +42,6 @@ from pyvo.mivot.viewer.mivot_instance import MivotInstance from pyvo.utils.prototype import prototype_feature from pyvo.mivot.utils.mivot_utils import MivotUtils -from pyvo.mivot.viewer.xml_viewer import XMLViewer # Use defusedxml only if already present in order to avoid a new depency. try: from defusedxml import ElementTree as etree @@ -99,7 +98,8 @@ def __init__(self, votable_path, tableref=None, resolve_ref=False): self._mapping_block = None self._mapped_tables = [] self._resource_seeker = None - self._dm_instance = None + self._dm_instances = [] + self._dm_globals_instances = [] self._resolve_ref = resolve_ref try: self._set_resource() @@ -107,7 +107,8 @@ def __init__(self, votable_path, tableref=None, resolve_ref=False): self._resource_seeker = ResourceSeeker(self._resource) self._set_mapped_tables() self._connect_table(tableref) - self._init_instance() + self._init_instances() + self._init_globals_instances() except MappingError as mnf: logging.error(str(mnf)) @@ -162,34 +163,34 @@ def dm_instance(self): """ returns ------- - A Python object (MivotInstance) built from the XML view of - the mapped model with attribute values set from the last values - of the last read data rows + MivotInstance: The Python object (MivotInstance) built from the XML view of the + first 'TEMPLATES' child, with the attribute values set according + to the values of the current read data row. """ - return self._dm_instance + return self._dm_instances[0] @property - def xml_view(self): + def dm_instances(self): """ - returns + Returns ------- - The XML view on the current data row + [MivotInstance]: The list of Python objects (MivotInstance) built from the XML views of + the TEMPLATES children, whose attribute values are set from the values + of the current read data row. """ - return self.xml_viewer.view + return self._dm_instances @property - def xml_viewer(self): + def dm_globals_instances(self): """ - returns - XMLViewer tuned to browse the TEMPLATES content + @@@@@@@@@ + Returns + ------- + [MivotInstance]: The list of Python objects (MivotInstance) built from the XML views of + the TEMPLATES children, whose attribute values are set from the values + of the current read data row. """ - # build a first XMLViewer for extract the content of the TEMPLATES element - model_view = XMLViewer(self._get_model_view()) - first_instance_dmype = self.get_first_instance_dmtype(tableref=self.connected_table_ref) - model_view.get_instance_by_type(first_instance_dmype) - - # return an XMLViewer tuned to process the TEMPLATES content - return XMLViewer(model_view._xml_view) + return self._dm_globals_instances @property def table_row(self): @@ -198,23 +199,23 @@ def table_row(self): def next_row_view(self): """ - jump to the next table row and update the MivotInstance instance + jump to the next table row and update the MivotInstance instance with the row values - returns + Returns ------- - MivotInstance: the updated instance or None - it he able end has been reached + [MivotInstance] + List of updated instances or None + it he able end has been reached """ self.next_table_row() if self._current_data_row is None: return None + self._init_instances() - if self._dm_instance is None: - xml_instance = self.xml_viewer.view - self._dm_instance = MivotInstance(**MivotUtils.xml_to_dict(xml_instance)) - self._dm_instance.update(self._current_data_row) - return self._dm_instance + for dm_instance in self._dm_instances: + dm_instance.update(self._current_data_row) + return self._dm_instances def get_table_ids(self): """ @@ -293,48 +294,47 @@ def rewind(self): if self._table_iterator: self._table_iterator.rewind() - def get_first_instance_dmtype(self, tableref=None): + def get_templates_child_instances(self, tableref=None): """ - Return the dmtype of the head INSTANCE (first TEMPLATES child). - If no INSTANCE is found, take the first COLLECTION. + Returns + ------- + [`xml.etree.ElementTree.Element`] + List of all INSTANCES elements children of the current TEMPLATES block + """ + if self._annotation_seeker is None: + return None + templates_block = self._annotation_seeker.get_templates_block(tableref) + return XPath.x_path(templates_block, ".//" + Ele.INSTANCE) + + def get_first_instance_dmtype(self, tableref): + """ + Return the dmtype of the head INSTANCE (first TEMPLATES child) of the + TEMPLATES block mapping the data table identified by tableref. Parameters ---------- - tableref : str or None, optional - Identifier of the table. + tableref : str or None + Identifier of the data table. + Returns ------- - ~`xml.etree.ElementTree.Element` - The first child of TEMPLATES. + string + dmtype of the selected INSTANCE + + Raises + ------ + MivotError + if no INSTANCE can be found """ - if self._annotation_seeker is None: - return None - child_template = self._annotation_seeker.get_templates_block(tableref) - child = child_template.findall("*") - collection = XPath.x_path(self._annotation_seeker.get_templates_block(tableref), - ".//" + Ele.COLLECTION) - instance = XPath.x_path(self._annotation_seeker.get_templates_block(tableref), ".//" + Ele.INSTANCE) - if len(collection) >= 1: - collection[0].set(Att.dmtype, Constant.ROOT_COLLECTION) - (self._annotation_seeker.get_templates_block(tableref).find(".//" + Ele.COLLECTION) - .set(Att.dmtype, Constant.ROOT_COLLECTION)) - if len(child) > 1: - if len(instance) >= 1: - for inst in instance: - if inst in child: - return inst.get(Att.dmtype) - elif len(collection) >= 1: - for coll in collection: - if coll in child: - return coll.get(Att.dmtype) - elif len(child) == 1: - if child[0] in instance: - return child[0].get(Att.dmtype) - elif child[0] in collection: - return collection[0].get(Att.dmtype) - else: - raise MivotError("Can't find the first " + Ele.INSTANCE - + "/" + Ele.COLLECTION + " in " + Ele.TEMPLATES) + templates_block = self._annotation_seeker.get_templates_block(tableref) + instances = XPath.x_path(templates_block, ".//" + Ele.INSTANCE) + for instance in instances: + return instance.get(Att.dmtype) + + raise MivotError( + "Can't find the first " + Ele.INSTANCE + + " in " + Ele.TEMPLATES + ) def _connect_table(self, tableref=None): """ @@ -379,12 +379,20 @@ def _connect_table(self, tableref=None): self._set_column_indices() self._set_column_units() - def _get_model_view(self): + def _get_model_view(self, xml_instance): """ Return an XML model view of the last read row. - This function resolves references by default. + + - References are possibly resolved here. + - ``ATTRIBUTE@value`` are set with actual data row values + + Returns + ------- + `xml.etree.ElementTree.Element` + XML model view of the last read row. + """ - templates_copy = deepcopy(self._templates) + templates_copy = deepcopy(xml_instance) if self._resolve_ref is True: while StaticReferenceResolver.resolve(self._annotation_seeker, self._connected_tableref, templates_copy) > 0: @@ -397,7 +405,6 @@ def _get_model_view(self): XmlUtils.add_column_units(templates_copy, self._resource_seeker .get_id_unit_mapping(self._connected_tableref)) - # for ele in templates_copy.xpath("//ATTRIBUTE"): for ele in XPath.x_path(templates_copy, ".//ATTRIBUTE"): ref = ele.get(Att.ref) if ref is not None and ref != Constant.NOT_SET and Constant.COL_INDEX in ele.attrib: @@ -405,25 +412,45 @@ def _get_model_view(self): ele.attrib[Att.value] = str(self._current_data_row[int(index)]) return templates_copy - def _init_instance(self): + def _init_instances(self): """ - Read the first table row and build the MivotInstance (_instance attribute) from it. - The table row iterator in rewind at he end to make sure we won't lost the first data row. + Read the first table row and build all MivotInstances (_dm_instances attribute) from it. + The table row iterator in rewind at the end to make sure we won't lost the first data row. """ - if self._dm_instance is None: + if not self._dm_instances: self.next_table_row() - first_instance = self.get_first_instance_dmtype(tableref=self.connected_table_ref) - xml_instance = self.xml_viewer.get_instance_by_type(first_instance) - self._dm_instance = MivotInstance(**MivotUtils.xml_to_dict(xml_instance)) + xml_instances = self.get_templates_child_instances(self.connected_table_ref) + self._dm_instances = [] + for xml_instance in xml_instances: + self._dm_instances.append( + MivotInstance( + **MivotUtils.xml_to_dict(self._get_model_view(xml_instance)) + )) self.rewind() - return self._dm_instance + + def _init_globals_instances(self): + """ + Build one MivotInstance for each GLOBALS/INSTANCE. Internal references are always resolved + Globals MivotInstance are stored in the _dm_globals_instances list + """ + if not self._dm_globals_instances: + globals_copy = deepcopy(self._annotation_seeker.globals_block) + + while StaticReferenceResolver.resolve(self._annotation_seeker, None, + globals_copy) > 0: + pass + for ele in XPath.x_path(globals_copy, "./" + Ele.INSTANCE): + self._dm_globals_instances.append( + MivotInstance( + **MivotUtils.xml_to_dict(ele) + )) def _set_mapped_tables(self): """ - Set the mapped tables with a list of the TEMPLATES tablerefs. + Set the _mapped_tables list with the TEMPLATES tablerefs. """ if not self.resource_seeker: - self._mapped_table = [] + self._mapped_tables = [] else: self._mapped_tables = self._annotation_seeker.get_templates() @@ -478,7 +505,7 @@ def _squash_join_and_references(self): def _set_column_indices(self): """ Add column ranks to attribute having a ref. - Using ranks allow identifying columns even numpy raw have been serialised as [] + Using ranks allow identifying columns even when numpy raw have been serialised as [] """ index_map = self._resource_seeker.get_id_index_mapping(self._connected_tableref) XmlUtils.add_column_indices(self._templates, index_map) diff --git a/pyvo/mivot/viewer/xml_viewer.py b/pyvo/mivot/viewer/xml_viewer.py deleted file mode 100644 index fd69e79c..00000000 --- a/pyvo/mivot/viewer/xml_viewer.py +++ /dev/null @@ -1,145 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -""" -XMLViewer provides several getters on XML instances built by - `pyvo.mivot.viewer.mivot_viewer`. -""" -from pyvo.mivot.utils.exceptions import MivotError -from pyvo.mivot.utils.xpath_utils import XPath -from pyvo.utils.prototype import prototype_feature - - -@prototype_feature('MIVOT') -class XMLViewer: - """ - The XMLViewer is used by `~pyvo.mivot.viewer.mivot_viewer` - to extract from the XML serialization of the model, - elements that will be used to build the dictionary from which - the Python class holding the mapped model will be generated. - """ - def __init__(self, xml_view): - self._xml_view = xml_view - - @property - def view(self): - """ - getter returning the XML model view - - returns - ------- - XML model view to be parsed - by different methods - """ - return self._xml_view - - def get_instance_by_role(self, dmrole, all_instances=False): - """ - If all_instances is False, return the first INSTANCE matching with @dmrole. - If all_instances is True, return a list of all instances matching with @dmrole. - Parameters - ---------- - dmrole : str - The @dmrole to look for. - all_instances : bool, optional - If True, returns a list of all instances, otherwise returns the first instance. - Default is False. - Returns - ------- - Union[`xml.etree.ElementTree.Element`, List[`xml.etree.ElementTree.Element`], None] - If all_instances is False, returns the instance matching with @dmrole. - If all_instances is True, returns a list of all instances matching with @dmrole. - If no matching instance is found, returns None. - Raises - ------ - MivotElementNotFound - If dmrole is not found. - """ - instances = XPath.select_elements_by_atttribute( - self._xml_view, - "INSTANCE", - "dmrole", - dmrole) - - if len(instances) == 0: - raise MivotError( - f"Cannot find dmrole {dmrole} in any instances of the VOTable") - - if all_instances is False: - return instances[0] - else: - return instances - - def get_instance_by_type(self, dmtype, all_instances=False): - """ - Return the instance matching with @dmtype. - If all_instances is False, returns the first INSTANCE matching with @dmtype. - If all_instances is True, returns a list of all instances matching with @dmtype. - Parameters - ---------- - dmtype : str - The @dmtype to look for. - all : bool, optional - If True, returns a list of all instances, otherwise returns the first instance. - Default is False. - Returns - ------- - Union[~`xml.etree.ElementTree.Element`, List[~`xml.etree.ElementTree.Element`], None] - If all_instances is False, returns the instance matching with @dmtype. - If all_instances is True, returns a list of all instances matching with @dmtype. - If no matching instance is found, returns None. - Raises - ------ - MivotElementNotFound - If dmtype is not found. - """ - instances = XPath.select_elements_by_atttribute( - self._xml_view, - "INSTANCE", - "dmtype", - dmtype) - - if len(instances) == 0: - raise MivotError( - f"Cannot find dmtype {dmtype} in any instances of the VOTable") - - if all_instances is False: - return instances[0] - else: - return instances - - def get_collection_by_role(self, dmrole, all_instances=False): - """ - Return the collection matching with @dmrole. - If all_instances is False, returns the first COLLECTION matching with @dmrole. - If all_instances is True, returns a list of all COLLECTION matching with @dmrole. - Parameters - ---------- - dmrole : str - The @dmrole to look for. - all_instances : bool, optional - If True, returns a list of all COLLECTION, otherwise returns the first COLLECTION. - Default is False. - Returns - ------- - Union[~`xml.etree.ElementTree.Element`, List[~`xml.etree.ElementTree.Element`], None] - If all_instances is False, returns the collection matching with @dmrole. - If all_instances is True, returns a list of all collections matching with @dmrole. - If no matching collection is found, returns None. - Raises - ------ - MivotElementNotFound - If dmrole is not found. - """ - collections = XPath.select_elements_by_atttribute( - self._xml_view, - "COLLECTION", - "dmrole", - dmrole) - - if len(collections) == 0: - raise MivotError( - f"Cannot find dmrole {dmrole} in any collections of the VOTable") - - if all_instances is False: - return collections[0] - else: - return collections diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py index d1a2e57b..a85279ef 100644 --- a/pyvo/mivot/writer/annotations.py +++ b/pyvo/mivot/writer/annotations.py @@ -170,7 +170,7 @@ def build_mivot_block(self, *, templates_id=None, schema_check=True): ---------- templates_id : str, optional The ID to associate with the block. Defaults to None. - schema_check : boolean, optional + schema_check : boolean, optional (default True) Skip the XSD validation if False (use to make test working in local mode). Raises diff --git a/pyvo/mivot/writer/instances_from_models.py b/pyvo/mivot/writer/instances_from_models.py index f69a5105..bc7fd50b 100644 --- a/pyvo/mivot/writer/instances_from_models.py +++ b/pyvo/mivot/writer/instances_from_models.py @@ -696,7 +696,7 @@ def add_query_origin(self, mapping={}): self._annotation._dmids.append("_origin") return query_origin_instance - def pack_into_votable(self, *, report_msg="", sparse=False): + def pack_into_votable(self, *, report_msg="", sparse=False, schema_check=True): """ Pack all mapped objects in the annotation block and put it in the VOTable. @@ -707,6 +707,9 @@ def pack_into_votable(self, *, report_msg="", sparse=False): sparse: boolean, optional (default to False) If True, all properties are added in a independent way to the the TEMPLATES. They are packed in a MangoObject otherwise. + schema_check: boolean, optional (default to True) + If True the MIVOT block is validated against its schema. + This may test failing due to remote accesses. """ self._annotation.set_report(True, report_msg) if sparse is True: @@ -718,5 +721,5 @@ def pack_into_votable(self, *, report_msg="", sparse=False): self._annotation.add_templates(self._mango_instance.get_mango_object( with_origin=("_origin" in self._annotation._dmids))) - self._annotation.build_mivot_block() + self._annotation.build_mivot_block(schema_check=schema_check) self._annotation.insert_into_votable(self._votable, override=True)