fix: fix import ontology from salsah-export (DSP-1532) (#59)

* Small fixes for dealing with ontolpogies from salsah-export

* fix: typo in test data

* Small fixes

* Searching for bug in CI

* …

* Debugging CI…

* Changes for passing CI tests

Co-authored-by: Balduin Landolt <balduin.landolt@hotmail.com>

knora/dsp_tools.py

@@ -81,7 +81,13 @@ def program(args):
             if args.validate:
                 validate_list(args.datamodelfile)
             else:
-                create_lists(args.datamodelfile, args.dump)
+                create_lists(input_file=args.datamodelfile,
+                             lists_file=args.listfile,
+                             server=args.server,
+                             user=args.user,
+                             password=args.password,
+                             verbose=args.verbose,
+                             dump=args.dump)
         else:
             if args.validate:
                 validate_ontology(args.datamodelfile)

knora/dsplib/models/helpers.py

@@ -397,7 +397,6 @@ def reduce_iri(self, iristr: str, ontoname: Optional[str] = None) -> str:
         knora_api = self.prefix_from_iri("http://api.knora.org/ontology/knora-api/v2#")
         salsah_gui = self.prefix_from_iri("http://api.knora.org/ontology/salsah-gui/v2#")
 
-        #if self.__is_iri(iristr):
         if IriTest.test(iristr):
             iristr = self.get_prefixed_iri(iristr)
         tmp = iristr.split(':')

knora/dsplib/models/langstring.py

@@ -193,15 +193,16 @@ def toJsonObj(self):
         if self._simplestring is not None:
             return self._simplestring
         else:
-            return list(map(lambda a: {'language': a[0].value, 'value': a[1]}, self._langstrs.items()))
+            return list(map(lambda a: {'language': a[0].value, 'value': a[1] if a[1] else "-"}, self._langstrs.items()))
 
     def toJsonLdObj(self):
         if self.isEmpty():
             return None
         if self._simplestring is not None:
             return self._simplestring
         else:
-            return list(map(lambda a: {'@language': a[0].value, '@value': a[1]}, self._langstrs.items()))
+            return [{'@language': a[0].value, '@value': a[1]} for a in self._langstrs.items()]
+            #return list(map(lambda a: {'@language': a[0].value, '@value': a[1]}, self._langstrs.items()))
 
     @classmethod
     def fromJsonLdObj(cls, obj: Optional[Union[List[Dict[str, str]], str]]) -> 'LangString':

knora/dsplib/models/listnode.py

@@ -445,6 +445,16 @@ def toJsonObj(self, action: Actions, listIri: str = None) -> Any:
                 tmp['comments'] = self._comments.toJsonObj()
             if self._name is not None and 'name' in self._changed:
                 tmp['name'] = self._name
+        #
+        # temporary fix for bug in dsp-api which prevents labels from having
+        # escaped double-quotes in the string, e.g. "this \"label\" not works"!
+        # The double quotes will be replaced by single quotes...
+        #
+        if tmp.get('labels'):
+            print(tmp['labels'])
+            tmp['labels'] = [{'language': ele['language'], 'value': ele['value'].replace('"', "'")} for ele in tmp['labels']]
+            #tmp['labels'] = {k: v.replace('"', "'") for k, v in tmp['labels'].items()}
+        # End of FIX
         return tmp
 
     def create(self) -> 'ListNode':

knora/dsplib/models/ontology.py

@@ -432,19 +432,20 @@ def createDefinitionFileObj(self):
 
         return ontology
 
-    def print(self) -> None:
+    def print(self, short: bool = True) -> None:
         print('Ontology Info:')
         print('  Id:                   {}'.format(self._id))
         print('  Label:                {}'.format(self._label))
         print('  Name:                 {}'.format(self._name))
         print('  Project:              {}'.format(self._project))
         print('  LastModificationDate: {}'.format(str(self._lastModificationDate)))
-        print('  Property Classes:')
-        if self._property_classes:
-            for pc in self._property_classes:
-                pc.print(4)
-        print('  Resource Classes:')
-        if self._resource_classes:
-            for rc in self._resource_classes:
-                rc.print(4)
+        if not short:
+            print('  Property Classes:')
+            if self._property_classes:
+                for pc in self._property_classes:
+                    pc.print(4)
+            print('  Resource Classes:')
+            if self._resource_classes:
+                for rc in self._resource_classes:
+                    rc.print(4)
 

knora/dsplib/models/propertyclass.py

@@ -311,7 +311,8 @@ def resolve_propref(resref: str):
             tmp = resref.split(':')
             if len(tmp) > 1:
                 if tmp[0]:
-                    return {"@id": resref}  # fully qualified name in the form "prefix:name"
+#                    return {"@id": resref}  # fully qualified name in the form "prefix:name"
+                    return {"@id": self._context.get_qualified_iri(resref)}  # fully qualified name in the form "prefix:name"
                 else:
                     return {"@id": self._context.prefix_from_iri(self._ontology_id) + ':' + tmp[1]}  # ":name" in current ontology
             else:

knora/dsplib/models/sipi.py

@@ -25,11 +25,12 @@ def on_api_error(self, res):
 
     def upload_bitstream(self, filepath):
         print(f"filepath=${os.path.basename(filepath)} (${filepath})")
-        files = {
-            'file': (os.path.basename(filepath), open(filepath, 'rb')),
-        }
-        req = requests.post(self.sipiserver + "/upload?token=" + self.token,
-                            files=files)
+        with open(filepath, 'rb') as bitstreamfile:
+            files = {
+                'file': (os.path.basename(filepath), bitstreamfile),
+            }
+            req = requests.post(self.sipiserver + "/upload?token=" + self.token,
+                                files=files)
         self.on_api_error(req)
         res = req.json()
         return res

knora/dsplib/utils/knora-schema-lists.json

@@ -76,8 +76,15 @@
       },
       "required": ["shortcode", "lists"],
       "additionalProperties": false
+    },
+    "prefixes": {
+      "type": "object",
+      "patternProperties": {
+        "^[\\w-]+$": { "format" : "uri" }
+      },
+      "additionalProperties": false
     }
   },
-  "required": ["project"],
+  "required": ["prefixes", "project"],
   "additionalProperties": false
-}
+}

knora/dsplib/utils/knora-schema.json

@@ -329,7 +329,7 @@
             },
             "width": {
               "type": "string",
-              "pattern": "^[0-9]*%$"
+              "pattern": "^[0-9]*%?$"
             },
             "wrap": {
               "type": "string",

knora/dsplib/utils/onto_create_lists.py

@@ -8,7 +8,8 @@
 from ..models.listnode import ListNode
 from .onto_commons import list_creator, validate_list_from_excel, json_list_from_excel
 
-def create_lists (input_file: str, output_file: str, server: str, user: str, password: str, verbose: bool) -> bool:
+
+def create_lists(input_file: str, lists_file: str, server: str, user: str, password: str, verbose: bool, dump: bool = False) -> bool:
     current_dir = os.path.dirname(os.path.realpath(__file__))
 
     # let's read the schema for the data model definition
@@ -55,6 +56,9 @@ def create_lists (input_file: str, output_file: str, server: str, user: str, pas
     con = Connection(server)
     con.login(user, password)
 
+    if dump:
+        con.start_logging()
+
     # --------------------------------------------------------------------------
     # let's read the prefixes of external ontologies that may be used
     #
@@ -84,16 +88,17 @@ def create_lists (input_file: str, output_file: str, server: str, user: str, pas
                 con=con,
                 project=project,
                 label=rootnode['labels'],
-                comment=rootnode.get('comments'),
+                #comment=rootnode.get('comments'),
                 name=rootnode['name']
             ).create()
-            listnodes = list_creator(con, project, root_list_node, rootnode['nodes'])
-            listrootnodes[rootnode['name']] = {
-                "id": root_list_node.id,
-                "nodes": listnodes
-            }
+            if rootnode.get('nodes') is not None:
+                listnodes = list_creator(con, project, root_list_node, rootnode['nodes'])
+                listrootnodes[rootnode['name']] = {
+                    "id": root_list_node.id,
+                    "nodes": listnodes
+                }
 
-    with open(output_file, 'w', encoding="utf-8") as fp:
+    with open(lists_file, 'w', encoding="utf-8") as fp:
         json.dump(listrootnodes, fp, indent=3, sort_keys=True)
-        print("The definitions of the node-id's can be found in \"{}}\"!".format(output_file))
+        print(f"The definitions of the node-id's can be found in \"{lists_file}\"!")
     return True

knora/dsplib/utils/onto_process_excel.py

@@ -40,4 +40,5 @@ def list_excel2json(excelpath: str,
     with open(os.path.join(current_dir, 'knora-schema-lists.json')) as s:
         schema = json.load(s)
     validate(jsonobj, schema)
-    json.dump(jsonobj, open(outfile, "w"), indent=4)
+    with open(outfile, "w") as outfile:
+        json.dump(jsonobj, outfile, indent=4)

knora/dsplib/utils/onto_validate.py

@@ -68,7 +68,8 @@ def validate_ontology_from_string(jsonstr: str, exceldir: Optional[str] = None)
                 newlists.append(rootnode)
         datamodel["project"]["lists"] = newlists
 
-    json.dump(datamodel, open("gaga.json", "w"), indent=4)
+    with open("gaga.json", "w") as outfile:
+        json.dump(datamodel, outfile, indent=4)
 
     # validate the data model definition in order to be sure that it is correct
     validate(datamodel, schema)

test/test_propertyclass.py

@@ -10,7 +10,7 @@
 
 class TestPropertyClass(unittest.TestCase):
     project = "http://rdfh.ch/projects/0001"
-    onto_name = 'propclass-test'
+    onto_name = 'propclass-test-a'
     onto_label = 'propclass_test_ontology'
 
     onto: Ontology
@@ -30,6 +30,9 @@ def setUp(self) -> None:
         self.con = Connection('http://0.0.0.0:3333')
         self.con.login('root@example.com', 'test')
 
+        ontos = Ontology.getAllOntologies(self.con)
+        for onto in ontos:
+            onto.print(True)
         #
         # Create a test ontology
         #

testdata/test-data.xml

@@ -112,7 +112,7 @@
               id="obj_0003"
               permissions="res-default">
         <text-prop name=":hasText">
-            <text permissions="prop-default" encoding="utf8">This is a Compoundthong</text>
+            <text permissions="prop-default" encoding="utf8">This is a Compoundthing</text>
         </text-prop>
     </resource>