Add some local tests

comparejsonld.py

@@ -74,7 +74,8 @@
         Downloads the entity from wikidata and assigns the json to self._entities
         """
         url: str = f"https://www.wikidata.org/wiki/Special:EntityData/{self._entity}.json"
-        response: Response = requests.get(url)
+        response: Response = requests.get(url=url,
+                                          headers={'User-Agent': 'Userscript Entityshape by User:Teester'})
         self._entities = response.json()
 
     def _get_props(self, claims: dict) -> None:
@@ -108,9 +109,13 @@
                                         for i in range((len(self._props) + 48) // 48)]
         for element in wikidata_property_list:
             required_properties: str = "|".join(element)
-            url: str = f"https://www.wikidata.org/w/api.php?action=wbgetentities&ids=" \
-                       f"{required_properties}&props=labels&languages={language}&format=json"
-            response: Response = requests.get(url)
+            response: Response = requests.get(url="https://www.wikidata.org/w/api.php",
+                                              params={"action": "wbgetentities",
+                                                      "ids": required_properties,
+                                                      "props": "labels",
+                                                      "languages": language,
+                                                      "format": "json"},
+                                              headers={'User-Agent': 'Entityshape API by User:Teester'})
             json_text: dict = response.json()
             for item in element:
                 try:
@@ -167,6 +172,13 @@
 
         :return:
         """
+        if "entities" not in self._entities:
+            return {}
+        if self._entity not in self._entities["entities"]:
+            return {}
+        if "claims" not in self._entities["entities"][self._entity]:
+            return {}
+
         claims: dict = self._entities["entities"][self._entity]["claims"]
         properties: dict = {}
         if self._start_shape is None:
@@ -213,6 +225,9 @@
         return response
 
     def _get_allowed_list(self, claims: dict, prop: str, expression: dict) -> list:
+        if prop not in claims:
+            return []
+
         allowed_list: list = []
         for statement in claims[prop]:
             is_it_allowed: str = ""
@@ -278,9 +293,12 @@
         :param str allowed: Whether the statement is allowed by the expression or not currently
         :return: allowed
         """
-        statement_property: str = statement["property"]
-        if "predicate" in expression and \
-                expression["predicate"].endswith(statement_property):
+        if "property" not in statement:
+            return allowed
+        if "predicate" not in expression:
+            return allowed
+
+        if expression["predicate"].endswith(statement["property"]):
             allowed = "present"
             try:
                 if expression["valueExpr"]["type"] == "NodeConstraint":
@@ -305,6 +323,9 @@
 
         :return: statements
         """
+        if "entities" not in self._entities:
+            return {}
+
         statements: dict = {}
         claims: dict = self._entities["entities"][self._entity]['claims']
         for claim in claims:
@@ -342,6 +363,13 @@
         return child, allowed
 
     def process_expressions(self, expression: dict, shape: dict, statement: dict, allowed: str) -> str:
+        if "type" not in expression:
+            return allowed
+        if "predicate" not in expression:
+            return allowed
+        if "property" not in statement:
+            return allowed
+
         if expression["type"] == "TripleConstraint" and expression["predicate"].endswith(statement["property"]):
             allowed = self._process_triple_constraint(statement,
                                                       expression,
@@ -362,9 +390,12 @@
         :param allowed: Whether the statement is allowed by the expression or not currently
         :return: allowed
         """
-        statement_property: str = statement["property"]
-        if "predicate" in expression and \
-                expression["predicate"].endswith(statement_property):
+        if "property" not in statement:
+            return allowed
+        if "predicate" not in expression:
+            return allowed
+
+        if expression["predicate"].endswith(statement["property"]):
             allowed = "allowed"
             Utilities.process_cardinalities(expression, {"mainsnak": statement})
             try:
@@ -453,6 +484,13 @@
         :param str allowed: Whether the statement is allowed by the expression or not currently
         :return: allowed
         """
+        if "snaktype" not in statement:
+            return allowed
+        if "datavalue" not in statement:
+            return allowed
+        if "type" not in statement["datavalue"]:
+            return allowed
+
         if statement["snaktype"] == "value" and \
                 statement["datavalue"]["type"] == "wikibase-entityid":
             obj = f'http://www.wikidata.org/entity/{statement["datavalue"]["value"]["id"]}'

compareshape.py

@@ -150,7 +150,8 @@ def _get_entity_json(self):
         Downloads the entity from wikidata
         """
         url: str = f"https://www.wikidata.org/wiki/Special:EntityData/{self._entity}.json"
-        response: Response = requests.get(url)
+        response: Response = requests.get(url=url,
+                                          headers={'User-Agent': 'Userscript Entityshape by User:Teester'})
         self._entities = response.json()
 
     def _get_props(self, claims: dict):
@@ -175,9 +176,14 @@ def _get_property_names(self, language: str):
                                         for i in range((len(self._props) + 48) // 48)]
         for element in wikidata_property_list:
             required_properties: str = "|".join(element)
-            url: str = f"https://www.wikidata.org/w/api.php?action=wbgetentities&ids=" \
-                       f"{required_properties}&props=labels&languages={language}&format=json"
-            response: Response = requests.get(url)
+            response: Response = requests.get(url="https://www.wikidata.org/w/api.php",
+                                              params={"action": "wbgetentities",
+                                                      "ids": required_properties,
+                                                      "props": "labels",
+                                                      "languages": language,
+                                                      "format": "json"},
+                                              headers={'User-Agent': 'Entityshape API by User:Teester'}
+                                              )
             json_text: dict = response.json()
             for item in element:
                 try:
@@ -206,9 +212,12 @@ def _process_required_in_shape_claim(shape_claim, datavalue):
             required_value: str = shape_claim["required"][required_property][0]
 
         query_entity: str = datavalue["value"]["id"]
-        url: str = f"https://www.wikidata.org/w/api.php?action=wbgetclaims" \
-                   f"&entity={query_entity}&property={required_property}&format=json"
-        response: Response = requests.get(url)
+        response: Response = requests.get(url="https://www.wikidata.org/w/api.php",
+                                          params={"action": "wbgetclaims",
+                                                  "entity": query_entity,
+                                                  "property": required_property,
+                                                  "format": "json"},
+                                          headers={'User-Agent': 'Entityshape API by User:Teester'})
         json_text: dict = response.json()
         if required_property in json_text["claims"]:
             for key in json_text["claims"][required_property]:

shape.py

@@ -164,7 +164,8 @@ def _get_schema_json(self, schema) -> None:
         :param schema: the entityschema to be downloaded
         """
         url: str = f"https://www.wikidata.org/wiki/EntitySchema:{schema}?action=raw"
-        response = requests.get(url)
+        response = requests.get(url=url,
+                                headers={'User-Agent': 'Userscript Entityshape by User:Teester'})
         self._json_text: dict = response.json()
 
     def _strip_schema_comments(self) -> None:

test_schemas.py

@@ -19,10 +19,6 @@ def setUp(self) -> None:
         app.config['DEBUG'] = False
         self.app = app.test_client()
 
-    def tearDown(self) -> None:
-        # Wait before performing the next test to avoid running into Wikidata's request limits when using Github Actions
-        time.sleep(4)
-
     def test_specific_wikidata_item_against_schema(self):
         """
         Tests a specific entity against a certain schema and checks that
@@ -52,6 +48,7 @@ def test_lexical_category(self):
                 self.assertIsNotNone(response.json["general"][0]["lexicalCategory"])
                 self.assertIsNotNone(response.json["general"][0]["language"])
 
+    @unittest.skip("Not running check on all wikidata schemas as they take too long")
     def test_wikidata_entityschemas(self) -> None:
         """
         Tests all wikidata entityschemas return 200
@@ -75,7 +72,8 @@ def test_wikidata_entityschemas(self) -> None:
             with self.subTest(schema=schema):
                 if schema in skips:
                     self.skipTest(f"Schema {schema} not supported")
-                # Wait before performing the next test to avoid running into Wikidata's request limits when using Github Actions
+                # Wait before performing the next test to avoid running into Wikidata's request limits
+                # when using GitHub Actions
                 time.sleep(4)
                 response = self.app.get(f'/api/v2?entityschema={schema}&entity=Q100532807&language=en',
                                         follow_redirects=True)

tests/test_compare_properties.py

@@ -0,0 +1,173 @@
+import unittest
+
+from comparejsonld import CompareProperties
+
+
+class TestCompareProperties(unittest.TestCase):
+    def setUp(self):
+        entity = {'entities':
+                      {'Q1':
+                        {'title': 'Q1',
+                         'type': 'item',
+                         'id': 'Q1',
+                         'claims': {'P31': [{'mainsnak':
+                                                {'snaktype': 'value',
+                                                 'property': 'P31',
+                                                 'hash': '1',
+                                                 'datavalue': {'value':
+                                                                   {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'},
+                                                               'type': 'wikibase-entityid'},
+                                                 'datatype': 'wikibase-item'},
+                                            'type': 'statement',
+                                            'id': '1',
+                                            'rank': 'normal'}],
+                         },
+                         'sitelinks': {}}}}
+        entities = "Q1"
+        statement = { "type": "Shape",
+                      "id": "test",
+                      "expression": {
+                          "type": "EachOf",
+                          "expressions": [
+                              {
+                                  "type": "TripleConstraint",
+                                  "predicate": "http://www.wikidata.org/prop/direct/P31"
+                              }
+                          ]
+                      }}
+        props = ["P31"]
+        names = {"P31": "instance of"}
+        self.compare_properties = CompareProperties(entities, entity, props, names, statement)
+
+    def test_compare_properties_with_nothing(self):
+        entity = {}
+        entities = ""
+        statement = {}
+        props = []
+        names = {}
+        test_method = CompareProperties(entities, entity, props, names, statement)
+        self.assertEqual({}, test_method.compare_properties())
+
+    def test_compare_properties_with_values(self):
+        result = {'P31': {'name': 'instance of',
+                          'necessity': 'required',
+                          'response': 'present'}}
+        self.assertEqual(result, self.compare_properties.compare_properties())
+
+    def test_check_claims_for_prop_with_nothing(self):
+        claims = {}
+        prop = ""
+        self.assertEqual('not enough correct statements', self.compare_properties.check_claims_for_props(claims, prop))
+
+    def test_check_claims_for_prop_with_values(self):
+        claims = {'Q1':
+                     {'title': 'Q1',
+                      'type': 'item',
+                      'id': 'Q1',
+                      'claims': {'P31': [{'mainsnak':
+                                              {'snaktype': 'value',
+                                               'property': 'P31',
+                                               'hash': '1',
+                                               'datavalue': {'value':
+                                                                 {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'},
+                                                             'type': 'wikibase-entityid'},
+                                               'datatype': 'wikibase-item'},
+                                          'type': 'statement',
+                                          'id': '1',
+                                          'rank': 'normal'}],
+                      },
+                      'sitelinks': {}}}
+        prop = "P31"
+        self.assertEqual('not enough correct statements', self.compare_properties.check_claims_for_props(claims, prop))
+
+    def test_get_allowed_list_with_nothing(self):
+        claims = {}
+        prop = ""
+        expression = {}
+        self.assertEqual([], self.compare_properties._get_allowed_list(claims, prop, expression))
+
+    def test_get_allowed_list_with_values(self):
+        claims = {'Q1':
+                      {'title': 'Q1',
+                       'type': 'item',
+                       'id': 'Q1',
+                       'claims': {'P31': [{'mainsnak':
+                                               {'snaktype': 'value',
+                                                'property': 'P31',
+                                                'hash': '1',
+                                                'datavalue': {'value':
+                                                                  {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'},
+                                                              'type': 'wikibase-entityid'},
+                                                'datatype': 'wikibase-item'},
+                                           'type': 'statement',
+                                           'id': '1',
+                                           'rank': 'normal'}],
+                       },
+                       'sitelinks': {}}}
+        prop = "P31"
+        expression = {'type': 'TripleConstraint',
+                      'predicate': 'http://www.wikidata.org/prop/direct/P31',
+                      'valueExpr': {'type': 'NodeConstraint', 'values': ['http://www.wikidata.org/entity/Q5']}}
+        self.assertEqual([], self.compare_properties._get_allowed_list(claims, prop, expression))
+
+    def test_process_cardinalities_with_nothing(self):
+        expression = {}
+        shape = {}
+        prop = ""
+        allowed = []
+        self.assertEqual("", self.compare_properties._process_cardinalities(expression, allowed, shape, prop))
+
+    def test_process_cardinalities_with_values(self):
+        expression = {'snaktype': 'value',
+                      'property': 'P31',
+                      'hash': '1',
+                      'datavalue': {'value':
+                                       {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'},
+                                    'type': 'wikibase-entityid'},
+                      'datatype': 'wikibase-item'}
+        shape = {'type': 'TripleConstraint',
+                 'predicate': 'http://www.wikidata.org/prop/direct/P31',
+                 'valueExpr': {'type': 'NodeConstraint', 'values': ['http://www.wikidata.org/entity/Q5']}}
+        prop = "P31"
+        allowed = ["Q5"]
+        self.assertEqual("", self.compare_properties._process_cardinalities(expression, allowed, shape, prop))
+
+    def test_get_cardinalities_with_nothing(self):
+        occurrences = 1
+        expression = {}
+        self.assertEqual("correct", self.compare_properties._get_cardinalities(occurrences, expression))
+
+    def test_get_cardinalities_with_values(self):
+        occurrences = 1
+        expression = {'snaktype': 'value',
+                      'property': 'P31',
+                      'hash': '1',
+                      'datavalue': {'value':
+                                       {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'},
+                                    'type': 'wikibase-entityid'},
+                      'datatype': 'wikibase-item'}
+        self.assertEqual("correct", self.compare_properties._get_cardinalities(occurrences, expression))
+
+    def test_process_triple_constraint_with_nothing(self):
+        statement = {}
+        expression = {}
+        allowed = ""
+        self.assertEqual("", self.compare_properties._process_triple_constraint(statement, expression, allowed))
+
+    def test_process_triple_constraint_with_values(self):
+        statement = {'snaktype': 'value',
+                     'property': 'P31',
+                     'hash': '1',
+                     'datavalue': {'value':
+                                      {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'},
+                                   'type': 'wikibase-entityid'},
+                     'datatype': 'wikibase-item'}
+        expression = {'type': 'TripleConstraint',
+                      'predicate': 'http://www.wikidata.org/prop/direct/P31',
+                      'valueExpr': {'type': 'NodeConstraint', 'values': ['http://www.wikidata.org/entity/Q2']}}
+        allowed = "Q2"
+        self.assertEqual("correct", self.compare_properties._process_triple_constraint(statement, expression, allowed))
+
+
+if __name__ == '__main__':
+    unittest.main()