feat(id2iri): add flag to remove created resources from XML (DEV-2571)

README.md

@@ -31,9 +31,12 @@ To get started quickly, without reading the details, just execute these commands
 - `curl -sSL https://install.python-poetry.org | python3 -`
 - `poetry self add poetry-exec-plugin`
 - `poetry install`
+- `poetry shell`
 - `pre-commit install`
 - `brew install imagemagick ffmpeg`
 
+To learn more about the meaning of these commands, read the remainder of this README.
+
 
 
 ## Using poetry for dependency management

docs/cli-commands.md

@@ -265,8 +265,19 @@ by IRIs provided in a mapping file.
 dsp-tools id2iri xmlfile.xml mapping.json
 ```
 
+The following options are available:
+
+- `-r` | `--remove-resources` (optional): remove resources if their ID is in the mapping 
+  (this prevents doubled resources on the DSP server,
+  because the resources occurring in the mapping already exist on the DSP server)
+
 The output file is written to `[original name]_replaced_[timestamp].xml`.
 
+If the flag `--remove-resources` is set,
+all resources of which the ID is in the mapping are removed from the XML file.
+This prevents doubled resources on the DSP server,
+because normally, the resources occurring in the mapping already exist on the DSP server.
+
 This command cannot be used isolated, 
 because it is part of a bigger procedure 
 that is documented [here](./incremental-xmlupload.md).

docs/incremental-xmlupload.md

@@ -16,11 +16,12 @@ What is this mapping used for?
 It can happen that at a later point of time,
 additional data is uploaded.
 Depending on what kind of references the additional data contains,
-there are 3 cases how this can happen:
+there are 4 cases how this can happen:
 
 1. no references to existing resources: normal xmlupload
 2. references to existing resources via IRIs: incremental xmlupload
 3. references to existing resources via internal IDs: first id2iri, then incremental xmlupload
+4. continue an interruped xmlupload: first id2iri, then incremental xmlupload
 
 
 
@@ -85,3 +86,28 @@ dsp-tools xmlupload --incremental additional_data_replaced_[timestamp].xml
 | <center>Important</center>                                                                                                                      |
 |-------------------------------------------------------------------------------------------------------------------------------------------------|
 | Internal IDs and IRIs cannot be mixed within the same file. An XML file uploaded with the incremental option must not contain any internal IDs. |
+
+
+
+## 4. Continue an interruped xmlupload
+
+If an xmlupload didn't finish successfully, 
+some resources have already been created, but others not.
+If the remaining resources have references to created ones,
+these references must be made with IRIs,
+and the created resources must be removed from the XML file
+(otherwise they would be created a second time).
+
+In such a case, proceed as follows:
+
+```bash
+dsp-tools xmlupload data.xml
+# crash: some resources have been uploaded, and a id2iri_mapping_[timestamp].json file has been written
+# fix the reason for the crash
+
+# replace the IDs and remove the created resources with:
+dsp-tools id2iri data.xml --remove-resources id2iri_mapping_[timestamp].json
+
+# upload the outputted XML file with
+dsp-tools xmlupload data_replaced_[timestamp].xml
+```

src/dsp_tools/cli.py

@@ -223,6 +223,9 @@ def _make_parser(
         help="Replace internal IDs of an XML file (resptr tags or salsah-links) by IRIs provided in a mapping file.",
     )
     parser_id2iri.set_defaults(action="id2iri")
+    parser_id2iri.add_argument(
+        "-r", "--remove-resources", action="store_true", help="remove resources if their ID is in the mapping"
+    )
     parser_id2iri.add_argument("xmlfile", help="path to the XML file containing the data to be replaced")
     parser_id2iri.add_argument("mapping", help="path to the JSON file containing the mapping of IDs to IRIs")
 

src/dsp_tools/utils/id_to_iri.py

@@ -75,7 +75,8 @@ def _replace_resptrs(
     Returns:
         a tuple of the modified XML tree and the set of the IDs that have been replaced
     """
-    resptr_elems = tree.xpath("/knora/resource/resptr-prop/resptr")
+    resptr_xpath = "|".join([f"/knora/{x}/resptr-prop/resptr" for x in ["resource", "annotation", "link", "region"]])
+    resptr_elems = tree.xpath(resptr_xpath)
     resptr_elems_replaced = 0
     for resptr_elem in resptr_elems:
         value_before = resptr_elem.text
@@ -106,9 +107,8 @@ def _replace_salsah_links(
     Returns:
         a tuple of the modified XML tree and the set of the IDs that have been replaced
     """
-    salsah_links = [
-        x for x in tree.xpath("/knora/resource/text-prop/text//a") if x.attrib.get("class") == "salsah-link"
-    ]
+    salsah_xpath = "|".join([f"/knora/{x}/text-prop/text//a" for x in ["resource", "annotation", "link", "region"]])
+    salsah_links = [x for x in tree.xpath(salsah_xpath) if x.attrib.get("class") == "salsah-link"]
     salsah_links_replaced = 0
     for salsah_link in salsah_links:
         value_before = regex.sub("IRI:|:IRI", "", salsah_link.attrib.get("href", ""))
@@ -137,7 +137,7 @@ def _replace_ids_by_iris(
         mapping: mapping of internal IDs to IRIs
 
     Returns:
-        modified XML tree
+        a tuple of the modified XML tree and the success status
     """
     success = True
     used_mapping_entries: set[str] = set()
@@ -160,6 +160,36 @@ def _replace_ids_by_iris(
     return tree, success
 
 
+def _remove_resources_if_id_in_mapping(
+    tree: etree._Element,
+    mapping: dict[str, str],
+) -> tuple[etree._Element, bool]:
+    """
+    Remove all resources from the XML file if their ID is in the mapping.
+
+    Args:
+        tree: parsed XML file
+        mapping: mapping of internal IDs to IRIs
+
+    Returns:
+        a tuple of the modified XML tree and the success status
+    """
+    success = True
+    resources = tree.xpath("|".join([f"/knora/{x}" for x in ["resource", "annotation", "link", "region"]]))
+    resources_to_remove = [x for x in resources if x.attrib.get("id") in mapping]
+    for resource in resources_to_remove:
+        resource.getparent().remove(resource)
+
+    msg = (
+        f"Removed {len(resources_to_remove)}/{len(resources)} resources from the XML file, "
+        "because their ID was in the mapping"
+    )
+    logger.info(msg)
+    print(msg)
+
+    return tree, success
+
+
 def _write_output_file(
     orig_xml_file: Path,
     tree: etree._Element,
@@ -182,6 +212,7 @@ def _write_output_file(
 def id_to_iri(
     xml_file: str,
     json_file: str,
+    remove_resource_if_id_in_mapping: bool = False,
 ) -> bool:
     """
     Replace internal IDs of an XML file
@@ -193,6 +224,7 @@ def id_to_iri(
     Args:
         xml_file: the XML file with the data to be replaced
         json_file: the JSON file with the mapping (dict) of internal IDs to IRIs
+        remove_resource_if_id_in_mapping: if True, remove all resources from the XML file if their ID is in the mapping
 
     Raises:
         BaseError: if one of the two input files is not a valid file
@@ -207,5 +239,10 @@ def id_to_iri(
         tree=tree,
         mapping=mapping,
     )
+    if remove_resource_if_id_in_mapping:
+        tree, success = _remove_resources_if_id_in_mapping(
+            tree=tree,
+            mapping=mapping,
+        )
     _write_output_file(orig_xml_file=xml_file_as_path, tree=tree)
     return success