feat(id2iri): add flag to remove created resources from XML (DEV-2571)

README.md

@@ -31,9 +31,12 @@ To get started quickly, without reading the details, just execute these commands
 - `curl -sSL https://install.python-poetry.org | python3 -`
 - `poetry self add poetry-exec-plugin`
 - `poetry install`
+- `poetry shell`
 - `pre-commit install`
 - `brew install imagemagick ffmpeg`
 
+To learn more about the meaning of these commands, read the remainder of this README.
+
 
 
 ## Using poetry for dependency management

docs/cli-commands.md

@@ -265,8 +265,17 @@ by IRIs provided in a mapping file.
 dsp-tools id2iri xmlfile.xml mapping.json
 ```
 
+The following options are available:
+
+- `-r` | `--remove-resources` (optional): remove resources if their ID is in the mapping 
+
 The output file is written to `[original name]_replaced_[timestamp].xml`.
 
+If the flag `--remove-resources` is set,
+all resources of which the ID is in the mapping are removed from the XML file.
+This prevents doubled resources on the DSP server,
+because normally, the resources occurring in the mapping already exist on the DSP server.
+
 This command cannot be used isolated, 
 because it is part of a bigger procedure 
 that is documented [here](./incremental-xmlupload.md).

docs/incremental-xmlupload.md

@@ -16,11 +16,12 @@ What is this mapping used for?
 It can happen that at a later point of time,
 additional data is uploaded.
 Depending on what kind of references the additional data contains,
-there are 3 cases how this can happen:
+there are 4 cases how this can happen:
 
 1. no references to existing resources: normal xmlupload
 2. references to existing resources via IRIs: incremental xmlupload
 3. references to existing resources via internal IDs: first id2iri, then incremental xmlupload
+4. continue an interruped xmlupload: first id2iri, then incremental xmlupload
 
 
 
@@ -85,3 +86,24 @@ dsp-tools xmlupload --incremental additional_data_replaced_[timestamp].xml
 | <center>Important</center>                                                                                                                      |
 |-------------------------------------------------------------------------------------------------------------------------------------------------|
 | Internal IDs and IRIs cannot be mixed within the same file. An XML file uploaded with the incremental option must not contain any internal IDs. |
+
+
+
+## 4. Continue an interruped xmlupload
+
+If an xmlupload didn't finish successfully, 
+some resources have already been created, but others have not.
+If one of the remaining resources references a created resource by its ID,
+this ID must be replaced by the IRI of the created resource.
+
+In addition, the created resources must be removed from the XML file,
+otherwise they would be created a second time.
+
+In such a case, proceed as follows:
+
+1. Initial xmlupload: `dsp-tools xmlupload data.xml`
+2. A crash happens. Some resources have been uploaded, and a `id2iri_mapping_[timestamp].json` file has been written
+3. Fix the reason for the crash
+4. Replace the IDs and remove the created resources with: 
+   `dsp-tools id2iri data.xml --remove-resources id2iri_mapping_[timestamp].json``
+5. Upload the outputted XML file with `dsp-tools xmlupload data_replaced_[timestamp].xml`

src/dsp_tools/cli.py

@@ -223,6 +223,9 @@ def _make_parser(
         help="Replace internal IDs of an XML file (resptr tags or salsah-links) by IRIs provided in a mapping file.",
     )
     parser_id2iri.set_defaults(action="id2iri")
+    parser_id2iri.add_argument(
+        "-r", "--remove-resources", action="store_true", help="remove resources if their ID is in the mapping"
+    )
     parser_id2iri.add_argument("xmlfile", help="path to the XML file containing the data to be replaced")
     parser_id2iri.add_argument("mapping", help="path to the JSON file containing the mapping of IDs to IRIs")
 

src/dsp_tools/utils/id_to_iri.py

@@ -74,7 +74,8 @@ def _replace_resptrs(
     Returns:
         a tuple of the modified XML tree and the set of the IDs that have been replaced
     """
-    resptr_elems = tree.xpath("/knora/resource/resptr-prop/resptr")
+    resptr_xpath = "|".join([f"/knora/{x}/resptr-prop/resptr" for x in ["resource", "annotation", "link", "region"]])
+    resptr_elems = tree.xpath(resptr_xpath)
     resptr_elems_replaced = 0
     for resptr_elem in resptr_elems:
         value_before = resptr_elem.text
@@ -105,9 +106,8 @@ def _replace_salsah_links(
     Returns:
         a tuple of the modified XML tree and the set of the IDs that have been replaced
     """
-    salsah_links = [
-        x for x in tree.xpath("/knora/resource/text-prop/text//a") if x.attrib.get("class") == "salsah-link"
-    ]
+    salsah_xpath = "|".join([f"/knora/{x}/text-prop/text//a" for x in ["resource", "annotation", "link", "region"]])
+    salsah_links = [x for x in tree.xpath(salsah_xpath) if x.attrib.get("class") == "salsah-link"]
     salsah_links_replaced = 0
     for salsah_link in salsah_links:
         value_before = regex.sub("IRI:|:IRI", "", salsah_link.attrib.get("href", ""))
@@ -125,7 +125,7 @@ def _replace_salsah_links(
 def _replace_ids_by_iris(
     tree: etree._Element,
     mapping: dict[str, str],
-) -> tuple[etree._Element, bool]:
+) -> etree._Element:
     """
     Iterate over the <resptr> tags and the salsah-links of the <text> tags,
     and replace the internal IDs by IRIs.
@@ -136,9 +136,8 @@ def _replace_ids_by_iris(
         mapping: mapping of internal IDs to IRIs
 
     Returns:
-        modified XML tree
+        a tuple of the modified XML tree
     """
-    success = True
     used_mapping_entries: set[str] = set()
 
     tree, used_mapping_entries = _replace_resptrs(
@@ -156,7 +155,36 @@ def _replace_ids_by_iris(
     logger.info(f"Used {len(used_mapping_entries)}/{len(mapping)} entries from the mapping file")
     print(f"Used {len(used_mapping_entries)}/{len(mapping)} entries from the mapping file")
 
-    return tree, success
+    return tree
+
+
+def _remove_resources_if_id_in_mapping(
+    tree: etree._Element,
+    mapping: dict[str, str],
+) -> etree._Element:
+    """
+    Remove all resources from the XML file if their ID is in the mapping.
+
+    Args:
+        tree: parsed XML file
+        mapping: mapping of internal IDs to IRIs
+
+    Returns:
+        a tuple of the modified XML tree
+    """
+    resources = tree.xpath("|".join([f"/knora/{x}" for x in ["resource", "annotation", "link", "region"]]))
+    resources_to_remove = [x for x in resources if x.attrib.get("id") in mapping]
+    for resource in resources_to_remove:
+        resource.getparent().remove(resource)
+
+    msg = (
+        f"Removed {len(resources_to_remove)}/{len(resources)} resources from the XML file, "
+        "because their ID was in the mapping"
+    )
+    logger.info(msg)
+    print(msg)
+
+    return tree
 
 
 def _write_output_file(
@@ -181,6 +209,7 @@ def _write_output_file(
 def id_to_iri(
     xml_file: str,
     json_file: str,
+    remove_resource_if_id_in_mapping: bool = False,
 ) -> bool:
     """
     Replace internal IDs of an XML file
@@ -192,19 +221,25 @@ def id_to_iri(
     Args:
         xml_file: the XML file with the data to be replaced
         json_file: the JSON file with the mapping (dict) of internal IDs to IRIs
+        remove_resource_if_id_in_mapping: if True, remove all resources from the XML file if their ID is in the mapping
 
     Raises:
         BaseError: if one of the two input files is not a valid file
 
     Returns:
-        True if everything went well, False otherwise
+        success status
     """
     xml_file_as_path, json_file_as_path = _check_input_parameters(xml_file=xml_file, json_file=json_file)
     mapping = _parse_json_file(json_file_as_path)
     tree = parse_xml_file(xml_file_as_path)
-    tree, success = _replace_ids_by_iris(
+    tree = _replace_ids_by_iris(
         tree=tree,
         mapping=mapping,
     )
+    if remove_resource_if_id_in_mapping:
+        tree = _remove_resources_if_id_in_mapping(
+            tree=tree,
+            mapping=mapping,
+        )
     _write_output_file(orig_xml_file=xml_file_as_path, tree=tree)
-    return success
+    return True