diff --git a/README.md b/README.md index 6559b930..ab3866da 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,13 @@ www.simphony-project.eu). ## Installation -See [https://simphony.readthedocs.io/en/latest/installation.html](https://simphony.readthedocs.io/en/latest/installation.html). +OSP-core is available on PyPI, so it can be installed using `pip` + +```shell +pip install osp-core +``` + +For more detailed instructions, see [https://simphony.readthedocs.io/en/latest/installation.html](https://simphony.readthedocs.io/en/latest/installation.html). ## Visualization of ontologies diff --git a/osp/core/ontology/namespace.py b/osp/core/ontology/namespace.py index 3377bb70..17654983 100644 --- a/osp/core/ontology/namespace.py +++ b/osp/core/ontology/namespace.py @@ -361,7 +361,7 @@ def _get_labels_for_iri(self, iri, lang=None, _return_literal=False, Args: iri (rdflib.URIRef): the target iri. - lang (str): retrieve labels only on a speific language. + lang (str): retrieve labels only on a specific language. _return_literal: return rdflib.Literal instead of str, so that the language of the labels is known to the caller. diff --git a/osp/core/ontology/namespace_registry.py b/osp/core/ontology/namespace_registry.py index 005c43d2..56216cec 100644 --- a/osp/core/ontology/namespace_registry.py +++ b/osp/core/ontology/namespace_registry.py @@ -252,7 +252,7 @@ def _get_entity_name(self, entity_iri, ns_iri): _return_literal=True, _return_label_property=True)) if not labels: - logger.warning(f"No label for {entity_iri}") + logger.debug(f"No label for {entity_iri}") else: labels = sorted(labels, key=lambda x: diff --git a/osp/core/ontology/ontology.py b/osp/core/ontology/ontology.py index d5346588..9e1c89da 100644 --- a/osp/core/ontology/ontology.py +++ b/osp/core/ontology/ontology.py @@ -183,7 +183,7 @@ def _overlay_add_cuba_triples(parser: Union[OntologyParser, 'Ontology'], logger.warning(f"Specified relationship {iri} as " f"active relationship, which is not " f"a valid object property in the ontology." - f"If such relationship belongs to another" + f"If such relationship belongs to another " f"ontology, and such ontology is installed, " f"then you may safely ignore this warning.") # This requirement is checked later on in @@ -233,12 +233,13 @@ def labels_for_iri(iri): lang=None, _return_literal=True) - # Finally check for the duplicate labels. + # Finally, check for the duplicate labels. subjects = set(subject for subject in graph.subjects() if in_namespace(subject)) - results = sorted(((label.toPython(), label.language), iri) - for iri in subjects for label - in labels_for_iri(iri)) + results = set(((label.toPython(), label.language or ''), iri) + for iri in subjects for label + in labels_for_iri(iri)) + results = sorted(results) labels, iris = tuple(result[0] for result in results), \ tuple(result[1] for result in results) coincidence_search = tuple(i @@ -248,12 +249,13 @@ def labels_for_iri(iri): for i in coincidence_search: conflicting_labels[labels[i]] |= {iris[i - 1], iris[i]} if len(conflicting_labels) > 0: - texts = (f'{label[0]}, language {label[1]}: ' + texts = (f'{label[0]}, language ' + f'{label[1] if label[1] != "" else None}: ' f'{", ".join(tuple(str(iri) for iri in iris))}' for label, iris in conflicting_labels.items()) raise KeyError(f'The following labels are assigned to more than ' f'one entity in namespace {namespace}; ' - f'{"; ".join(texts)}.') + f'{"; ".join(texts)} .') def _check_namespaces(namespace_iris: Iterable[URIRef], diff --git a/osp/core/session/db/db_wrapper_session.py b/osp/core/session/db/db_wrapper_session.py index b474813c..a85c34d0 100644 --- a/osp/core/session/db/db_wrapper_session.py +++ b/osp/core/session/db/db_wrapper_session.py @@ -1,16 +1,19 @@ """An abstract session containing method useful for all database backends.""" -from abc import abstractmethod -from typing import Union import itertools import logging -import rdflib import uuid -from osp.core.utils.general import uid_from_iri, CUDS_IRI_PREFIX +from abc import abstractmethod +from typing import Union + +import rdflib + +import osp.core.warnings as warning_settings from osp.core.ontology.namespace_registry import namespace_registry from osp.core.session.wrapper_session import consumes_buffers, WrapperSession from osp.core.session.result import returns_query_result from osp.core.session.buffers import BufferContext, EngineContext +from osp.core.utils.general import uid_from_iri, CUDS_IRI_PREFIX logger = logging.getLogger(__name__) @@ -27,11 +30,12 @@ def commit(self): try: root_obj = self._registry.get(self.root) added, updated, deleted = self._buffers[BufferContext.USER] + if warning_settings.unreachable_cuds_objects: + self._unreachable_warning(root_obj) self._apply_added(root_obj, added) self._apply_updated(root_obj, updated) self._apply_deleted(root_obj, deleted) self._reset_buffers(BufferContext.USER) - self._unreachable_warning(root_obj) self._commit() except Exception as e: self._rollback_transaction() @@ -176,24 +180,112 @@ def _unreachable_warning(self, root_obj: Union[rdflib.URIRef, uuid.UUID]): raises a warning that lists some of the unreachable CUDS objects. Args: - root_obj (Union[URIRef, UUID]): The root object with respect to - which objects are deemed reachable or unreachable. + root_obj: The root object with respect to which objects are + deemed reachable or unreachable. """ - unreachable = self._registry._get_not_reachable(root_obj, rel=None) + large_dataset_warning = LargeDatasetWarning() + unreachable, reachable = self._registry._get_not_reachable( + root_obj, rel=None, return_reachable=True, + warning=large_dataset_warning + ) + + # Warn about unreachable CUDS max_cuds_on_warning = 5 if len(unreachable) > 0: - warning = "Some CUDS objects are unreachable " \ - "from the wrapper object: " \ - "{cuds}{more}." \ - "\n" \ - "If you want to be able to retrieve those CUDS " \ - "objects later, either add them to the wrapper object " \ - "or to any other CUDS that is reachable from it." \ - .format(cuds=', '.join(str(x) for x in itertools - .islice(unreachable, - max_cuds_on_warning)), - more=" and " + str(len(unreachable) - - max_cuds_on_warning) - + " more" if len(unreachable) > 5 - else "") - logger.warning(warning) + unreachable_cuds_warning = ( + "Some CUDS objects are unreachable from the wrapper object: " + "{cuds}{more}. \n" + "If you want to be able to retrieve those CUDS objects later, " + "either add them to the wrapper object or to any other CUDS " + "that is reachable from it." + ).format( + cuds=', '.join(str(x) for x in itertools + .islice(unreachable, max_cuds_on_warning)), + more=" and " + str(len(unreachable) - max_cuds_on_warning) + + " more" if len(unreachable) > 5 else "") + # A filter is applied to the logger that attaches the warning + # type to the log records. + logger_filter = UnreachableCUDSWarningFilter() + logger.addFilter(logger_filter) + logger.warning(unreachable_cuds_warning) + logger.removeFilter(logger_filter) + + # Inform the large dataset warning that the unreachable CUDS + # warning was raised (so that it changes its text). + large_dataset_warning.unreachable_cuds_warning = True + + # Warn about large datasets and recommend disabling the unreachable + # CUDS warning for large datasets. + if len(reachable) + len(unreachable) >= \ + warning_settings.unreachable_cuds_objects_large_dataset_size: + # Recommend disabling the warning for large datasets. + large_dataset_warning.warn() + + +class UnreachableCUDSWarning(UserWarning): + """Shown when CUDS are unreachable from the wrapper. + + Used by `DbWrapperSession._unreachable_warning` during the commit + operation. + """ + + +class UnreachableCUDSWarningFilter(logging.Filter): + """Attaches the `UnreachableCUDSWarning` class to the records.""" + + def filter(self, record): + """Attaches the `UnreachableCUDSWarning` to the records.""" + record.warning_class = UnreachableCUDSWarning + return True + + +class LargeDatasetWarning(UserWarning): + """Shown while working with a large dataset. + + Used by `DbWrapperSession._unreachable_warning`, during the commit + operation. + """ + warned: bool = False + unreachable_cuds_warning: bool = False + + def warn(self) -> None: + """Show the warning. + + The warning will be only shown once. If you want to show the warning + again, you must create a new instance of `LargeDatasetWarning`. + """ + if self.warned: + return + + # Recommend disabling the `UnreachableCUDSWarning` for large datasets. + warning = ( + "You are working with a large dataset. When committing " + "changes, OSP-core looks for objects that are unreachable " + "from the wrapper object to generate {reference_to_warning}. " + "Generating such warning is very expensive in computational " + "terms when small changes are applied to existing, " + "large datasets. You will notice that the changes may take a " + "lot of time to be committed. Please turn off such warning " + "when working with large datasets. You can turn off the " + "warning by running `import osp.core.warnings as " + "warning_settings; " + "warning_settings.unreachable_cuds_objects = False`.") + reference = ("a warning" if not self.unreachable_cuds_warning else + "the previous warning") + warning = warning.format(reference_to_warning=reference) + # A filter is applied to the logger that attaches the warning + # type to the log records. + logger_filter = LargeDatasetWarningFilter() + logger.addFilter(logger_filter) + logger.warning(warning) + logger.removeFilter(logger_filter) + self.warned = True + + +class LargeDatasetWarningFilter(logging.Filter): + """Filter that attaches the `LargeDatasetWarning` class to the records.""" + + def filter(self, record): + """Attaches the `LargeDatasetWarning` to the records.""" + record.warning_class = LargeDatasetWarning + return True diff --git a/osp/core/session/registry.py b/osp/core/session/registry.py index e83eb0af..5f07621d 100644 --- a/osp/core/session/registry.py +++ b/osp/core/session/registry.py @@ -4,6 +4,8 @@ from rdflib import URIRef import logging +import osp.core.warnings as warning_settings + logger = logging.getLogger(__name__) @@ -58,7 +60,8 @@ def get(self, uid): message = '{!r} is not a proper uid' raise ValueError(message.format(uid)) - def get_subtree(self, root, rel=None, skip=None): + def get_subtree(self, root, subtree=None, rel=None, skip=None, + warning=None): """Get all the elements in the subtree rooted at given root. Only use the given relationship for traversal. @@ -67,23 +70,59 @@ def get_subtree(self, root, rel=None, skip=None): root (Union[UUID, URIRef, Cuds]): The root of the subtree. rel (Relationship, optional): The relationship used for traversal. Defaults to None. Defaults to None. + subtree (Set[Cuds]): Currently calculated subtree (this is a + recursive algorithm). skip (Set[Cuds], optional): The elements to skip. Defaults to None. Defaults to None. + warning (LargeDatasetWarning, optional): Raise a + `LargeDatasetWarning` when the subtree is large. When `None`, + no warning is raised. If you wish to raise the warning, a + `LargeDatasetWarning` object must be provided. Returns: Set[Cuds]: The set of elements in the subtree rooted in the given uid. """ - from osp.core.cuds import Cuds - skip = skip or set() - if not isinstance(root, Cuds): + if isinstance(root, (UUID, URIRef)): root = super().__getitem__(root) assert root.uid in self - subtree = {root} - for child in root.iter(rel=rel): - if child not in (skip | subtree): - subtree |= self.get_subtree(child.uid, rel, - skip=(skip | subtree)) + skip = skip or set() | {root} + skip |= {root} + subtree = subtree or {root} + + subclasses = set() if rel is None else rel.subclasses + subclass_check = (lambda r: True) \ + if not subclasses else (lambda r: r in subclasses) + """Checks whether relationship `x` should be considered. + + - When no `rel` is provided, `subclass_check` should always return + True, as all relationships should be considered. + + - When `rel` is provided, it should return true only if the + relationship `x` is a subclass of the provided relationship (`rel`). + """ + + # Load neighbors connected through the relationship + filtered_neighbors = ( + neighbor + for r, dict_target in root._neighbors.items() + if subclass_check(r) + for neighbor in dict_target + ) + filtered_neighbors = set(root.session.load(*filtered_neighbors)) + + subtree |= filtered_neighbors + + # Optional: raise a `LargeDatasetWarning` if the subtree is too large. + if warning is not None and len(subtree) \ + > warning_settings \ + .unreachable_cuds_objects_large_dataset_size: + warning.warn() + warning = None + + for neighbor in filter(lambda x: x not in skip, filtered_neighbors): + self.get_subtree(neighbor, subtree=subtree, rel=rel, skip=skip, + warning=warning) return subtree def prune(self, *roots, rel=None): @@ -103,7 +142,11 @@ def prune(self, *roots, rel=None): super().__delitem__(x.uid) return not_reachable - def _get_not_reachable(self, *roots, rel=None): + def _get_not_reachable(self, + *roots, + rel=None, + return_reachable=False, + warning=None): """Get all elements in the registry that are not reachable. Use the given rel for traversal. @@ -113,14 +156,21 @@ def _get_not_reachable(self, *roots, rel=None): from these root elements. rel (Relationship, optional): Only use this relationship for traversal. Defaults to None. + return_reachable (bool): Returns also the uids of the reachable + cuds. Returns: - List[Cuds]: The set of non reachable elements. + Union[List[Cuds], + Tuple[List[Cuds], Set[Union[UUID, URIRef]]]]: Either a + list of the unreachable CUDS when `return_reachable` is False + or a tuple whose first element is such list, and second + element a set with the uids of the reachable cuds. """ # Get all reachable Cuds objects reachable = set() for root in roots: - reachable |= self.get_subtree(root, rel=rel, skip=reachable) + reachable |= self.get_subtree( + root, rel=rel, skip=reachable, warning=warning) reachable_uids = set([r.uid for r in reachable]) # Get all the Cuds objects that are not reachable @@ -128,7 +178,7 @@ def _get_not_reachable(self, *roots, rel=None): for uid in self.keys(): if uid not in reachable_uids: delete.append(super().__getitem__(uid)) - return delete + return delete if not return_reachable else (delete, reachable_uids) def reset(self): """Delete the contents of the registry.""" diff --git a/osp/core/session/transport/communication_engine.py b/osp/core/session/transport/communication_engine.py index f79af976..1dd13489 100644 --- a/osp/core/session/transport/communication_engine.py +++ b/osp/core/session/transport/communication_engine.py @@ -162,6 +162,12 @@ def __init__(self, uri, handle_response, **kwargs): """ self.uri = uri self.kwargs = kwargs + # The default `ping_timeout` is 20s. The pings are not sent during a + # transfer. Thus, if the transfer takes more than 20s, then the + # default value causes the websockets connection to close + # unexpectedly. Hence, we chose to never close the connection due to + # ping timeouts unless the user wishes to do so. + self.kwargs['ping_timeout'] = self.kwargs.get('ping_timeout', None) self.handle_response = handle_response self.websocket = None diff --git a/osp/core/warnings.py b/osp/core/warnings.py index 534d51e6..ac5ab84a 100644 --- a/osp/core/warnings.py +++ b/osp/core/warnings.py @@ -6,3 +6,14 @@ For example `fr = city.City(name='Freiburg', coordinates=[1, 2]); fr.coordinates`. """ + +unreachable_cuds_objects = True +unreachable_cuds_objects_large_dataset_size = 1000 +"""Warns when a commit is performed and unreachable CUDS exist. + +Disabling this warning can greatly improve the performance of commits when +working with large datasets. + +The second parameter `unreachable_cuds_objects_large_dataset_size` controls +the minimum size of a dataset needs to be in order to be considered large. +""" diff --git a/packageinfo.py b/packageinfo.py index 89d2dec0..aafbe672 100644 --- a/packageinfo.py +++ b/packageinfo.py @@ -1,4 +1,4 @@ """Information about the package.""" NAME = "osp-core" -VERSION = "3.6.1" +VERSION = "3.7.0" diff --git a/setup.py b/setup.py index 02eb44af..7adbf242 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -"""Install osp-core.""" +"""Install OSP-core.""" import os @@ -7,10 +7,11 @@ from packageinfo import VERSION, NAME -# Read description +# Read `README.md` file. with open('README.md', 'r', encoding="utf8") as readme: README_TEXT = readme.read() +# Copy `packageinfo.py` to the `osp.core` module. with open("packageinfo.py", "r", encoding="utf8") as packageinfo: with open(os.path.join("osp", "core", "packageinfo.py"), "w", encoding="utf8") as f: @@ -21,32 +22,44 @@ file=f) -# main setup configuration class setup( name=NAME, version=VERSION, - author='SimPhoNy, EU FP7 Project (Nr. 604005)', - url='www.simphony-project.eu', - description='The native implementation of the SimPhoNy cuds objects', - keywords='simphony, cuds, Fraunhofer IWM', + description='The native implementation of the SimPhoNy CUDS objects', long_description=README_TEXT, + long_description_content_type='text/markdown', + url='https://github.com/simphony', + author='SimPhoNy, EU FP7 Project (Nr. 604005)', + maintainer='Fraunhofer IWM', + maintainer_email='simphony@iwm.fraunhofer.de', + license='BSD-3-Clause', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + 'Topic :: Scientific/Engineering', + 'Topic :: Software Development :: Libraries', + 'License :: OSI Approved :: BSD License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: MacOS', + 'Operating System :: Unix', + ], + keywords='owl ontology interoperability materials-science osp simphony ' + 'wrappers cuds open-simulation-platform', + download_url="https://pypi.python.org/pypi/osp-core", + project_urls={ + "Tracker": "https://github.com/simphony/osp-core/issues", + "Documentation": "https://simphony.readthedocs.io", + "Source": "https://github.com/simphony/osp-core", + }, packages=find_packages( exclude=("examples", "tests")), - package_data={ - "osp.core.ontology.docs": ["*.yml", "*.ttl", "*.xml", "EMMO/*.owl"], - }, - include_package_data=True, - python_requires=">=3.6", - entry_points={ - 'wrappers': 'osp-core = osp.core.session.core_session:CoreSession', - 'console_scripts': { - 'owl2yml = osp.core.tools.owl2yml:run_from_terminal', - 'pico = osp.core.pico:install_from_terminal', - 'ontology2dot = osp.core.tools.ontology2dot:run_from_terminal', - 'yaml2camelcase = osp.core.tools.yaml2camelcase:run_from_terminal' - } - }, install_requires=[ "graphviz", "numpy", @@ -62,4 +75,19 @@ # 🠕 Required by rdflib >= 5.0.0, < 6.0.0, otherwise no SPARQL support. # ↑ --- Python 3.6 support. --- ↑ # ], + python_requires=">=3.6", + package_data={ + "osp.core.ontology.docs": ["*.yml", "*.ttl", "*.xml"], + }, + data_files=[('.', ['packageinfo.py'])], + include_package_data=True, + entry_points={ + 'wrappers': 'osp-core = osp.core.session.core_session:CoreSession', + 'console_scripts': { + 'owl2yml = osp.core.tools.owl2yml:run_from_terminal', + 'pico = osp.core.pico:install_from_terminal', + 'ontology2dot = osp.core.tools.ontology2dot:run_from_terminal', + 'yaml2camelcase = osp.core.tools.yaml2camelcase:run_from_terminal' + } + }, ) diff --git a/tests/test_registry_city.py b/tests/test_registry_city.py index eb531b1c..bcd7d36c 100644 --- a/tests/test_registry_city.py +++ b/tests/test_registry_city.py @@ -40,16 +40,46 @@ def test_get_subtree(self): registry = c.session._registry self.assertEqual( registry.get_subtree(c.uid), - set([c, p, n, s])) + {c, p, n, s}) self.assertEqual( registry.get_subtree(c.uid, rel=cuba.activeRelationship), - set([c, p, n, s])) + {c, p, n, s}) self.assertEqual( registry.get_subtree(n.uid), - set([c, p, n, s])) + {c, p, n, s}) self.assertEqual( registry.get_subtree(n.uid, rel=cuba.activeRelationship), - set([n, s])) + {n, s}) + + c_o = city.City(name="other city") + n.add(c_o, rel=city.isPartOf) + self.assertEqual( + registry.get_subtree(c.uid), + {c, p, n, s, c_o}) + self.assertEqual( + registry.get_subtree(c.uid, rel=cuba.activeRelationship), + {c, p, n, s}) + self.assertEqual( + registry.get_subtree(n.uid), + {c, p, n, s, c_o}) + self.assertEqual( + registry.get_subtree(n.uid, rel=cuba.activeRelationship), + {n, s}) + + # Test whether cycles are a problem + c_o.add(c, rel=cuba.relationship) + self.assertEqual( + registry.get_subtree(c_o.uid), + {c, p, n, s, c_o}) + + # Disconnected items + c_f = city.City(name="far city") + self.assertEqual( + registry.get_subtree(c.uid), + {c, p, n, s, c_o}) + self.assertEqual( + registry.get_subtree(c_f.uid), + {c_f}) def test_prune(self): """Test the pruning method.""" diff --git a/tests/test_sqlite_city.py b/tests/test_sqlite_city.py index bb578d40..579c7d8e 100644 --- a/tests/test_sqlite_city.py +++ b/tests/test_sqlite_city.py @@ -1,11 +1,18 @@ """Test the Sqlite Wrapper with the CITY ontology.""" +import logging import os -from osp.core.utils.general import iri_from_uid -import uuid -import unittest2 as unittest import sqlite3 +import uuid +from typing import Iterable, Tuple, Type, Union + import numpy as np +import unittest2 as unittest + +from osp.core.utils.general import iri_from_uid +from osp.core.session.db.db_wrapper_session import LargeDatasetWarning, \ + UnreachableCUDSWarning, logger +import osp.core.warnings from osp.wrappers.sqlite import SqliteSession try: @@ -379,6 +386,195 @@ def test_cuds_with_iri(self): self.assertEqual(wrapper.get(c.uid).name, "Freiburg") self.assertEqual(wrapper.get(c.uid).oclass, city.City) + def test_big_dataset_warning(self): + """Tests the warnings emitted when committing big datasets.""" + # Save the original warning settings to restore them at the end of + # the test. + original_warning_setting = \ + osp.core.warnings.unreachable_cuds_objects + original_large_dataset_size = \ + osp.core.warnings.unreachable_cuds_objects_large_dataset_size + + def count_warnings_by_class(records: Iterable[logging.LogRecord], + classes: Union[Type, Tuple[Type, ...]]) \ + -> int: + """Given log records, count their "classes" if attached. + + For each record, checks if it has a `warning_class` attribute, + and checks whether its value is a subclass of the classes + provided. + """ + return sum( + bool(issubclass(record.warning_class, classes) + if hasattr(record, 'warning_class') else False) + for record in records + ) + + osp.core.warnings.unreachable_cuds_objects_large_dataset_size = 5 + osp.core.warnings.unreachable_cuds_objects = True + + large_dataset_size = \ + osp.core.warnings.unreachable_cuds_objects_large_dataset_size + + try: + # No warning: small dataset, all CUDS reachable + with SqliteSession(DB) as session: + wrapper = city.CityWrapper(session=session) + wrapper.add(city.Citizen(name='citizen'), + rel=city.hasInhabitant) + with self.assertLogs(logger=logger) as captured: + logger.warning('At least log entry is needed for ' + '`assertLogs`.') + session.commit() + self.assertEqual( + count_warnings_by_class( + captured.records, + (LargeDatasetWarning, UnreachableCUDSWarning)), + 0 + ) + if os.path.exists(DB): + os.remove(DB) + + # Unreachable CUDS warning: small dataset, some CUDS not reachable + with SqliteSession(DB) as session: + wrapper = city.CityWrapper(session=session) + wrapper.add(city.Citizen(name='citizen'), + rel=city.hasInhabitant) + city.Citizen(name='citizen') + with self.assertLogs(logger=logger) as captured: + logger.warning('At least log entry is needed for ' + '`assertLogs`.') + session.commit() + self.assertEqual( + count_warnings_by_class( + captured.records, + LargeDatasetWarning), + 0 + ) + self.assertEqual( + count_warnings_by_class( + captured.records, + UnreachableCUDSWarning), + 1 + ) + if os.path.exists(DB): + os.remove(DB) + + # Large dataset warning: large dataset, all CUDS reachable + with SqliteSession(DB) as session: + wrapper = city.CityWrapper(session=session) + wrapper.add( + *[city.Citizen(name='citizen') + for _ in range(0, large_dataset_size + 1)], + rel=city.hasInhabitant) + with self.assertLogs(logger=logger) as captured: + logger.warning('At least log entry is needed for ' + '`assertLogs`.') + session.commit() + self.assertEqual( + count_warnings_by_class( + captured.records, + LargeDatasetWarning), + 1 + ) + self.assertEqual( + count_warnings_by_class( + captured.records, + UnreachableCUDSWarning), + 0 + ) + if os.path.exists(DB): + os.remove(DB) + + # Both warnings: large dataset, some CUDS not reachable + with SqliteSession(DB) as session: + wrapper = city.CityWrapper(session=session) + wrapper.add( + *[city.Citizen(name='citizen') + for _ in range(0, large_dataset_size + 1)], + rel=city.hasInhabitant) + city.Citizen(name='citizen') + with self.assertLogs(logger=logger) as captured: + logger.warning('At least log entry is needed for ' + '`assertLogs`.') + session.commit() + self.assertEqual( + count_warnings_by_class( + captured.records, + LargeDatasetWarning), + 1 + ) + self.assertEqual( + count_warnings_by_class( + captured.records, + UnreachableCUDSWarning), + 1 + ) + if os.path.exists(DB): + os.remove(DB) + + # Edge case: large dataset warning after unreachable warning + with SqliteSession(DB) as session: + wrapper = city.CityWrapper(session=session) + wrapper.add(city.Citizen(name='citizen'), + rel=city.hasInhabitant) + [city.Citizen(name='citizen') + for i in range(0, large_dataset_size)] + with self.assertLogs(logger=logger) as captured: + logger.warning('At least log entry is needed for ' + '`assertLogs`.') + session.commit() + self.assertEqual( + count_warnings_by_class( + captured.records, + LargeDatasetWarning), + 1 + ) + self.assertEqual( + count_warnings_by_class( + captured.records, + UnreachableCUDSWarning), + 1 + ) + large_dataset_warning = set( + record + for record in captured.records + if hasattr(record, 'warning_class') + if issubclass(record.warning_class, + LargeDatasetWarning)).pop() + self.assertIn('the previous warning', + large_dataset_warning.msg) + if os.path.exists(DB): + os.remove(DB) + + # Test warning turned off for the both warnings case + osp.core.warnings.unreachable_cuds_objects = False + with SqliteSession(DB) as session: + wrapper = city.CityWrapper(session=session) + wrapper.add( + *[city.Citizen(name='citizen') + for _ in range(0, large_dataset_size + 1)], + rel=city.hasInhabitant) + city.Citizen(name='citizen') + with self.assertLogs(logger=logger) as captured: + logger.warning('At least log entry is needed for ' + '`assertLogs`.') + session.commit() + self.assertEqual( + count_warnings_by_class( + captured.records, + (LargeDatasetWarning, UnreachableCUDSWarning)), + 0 + ) + if os.path.exists(DB): + os.remove(DB) + + finally: + osp.core.warnings.unreachable_cuds_objects = \ + original_warning_setting + osp.core.warnings.unreachable_cuds_objects_large_dataset_size = \ + original_large_dataset_size + def check_state(test_case, c, p1, p2, db=DB): """Check if the sqlite tables are in the correct state."""