From 647c9ad3bc9a71162c5dc7109619da3299cd59d7 Mon Sep 17 00:00:00 2001 From: Nico Matentzoglu Date: Fri, 5 Aug 2022 12:33:26 +0300 Subject: [PATCH 01/10] Update Makefile.jinja2 --- template/src/ontology/Makefile.jinja2 | 40 +++++++++++++++++++++------ 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index 10afb13e..06718f55 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -814,26 +814,49 @@ $(ONT).owl: $(ONT)-{{ project.primary_release }}.owl cp $< $@ {% endif -%} +ROBOT_RELEASE_IMPORT_MODE={% if project.use_edit_file_imports -%}$(ROBOT) merge --input $< {% else -%} +$(ROBOT) remove --input $< --select imports --trim false {% if project.use_dosdps or project.components is defined -%} +merge $(patsubst %, -i %, $(OTHER_SRC)) {% endif %}{% if project.import_group is defined -%} +merge $(patsubst %, -i %, $(IMPORT_FILES)){% endif %}{% endif %} + +{% if 'base' in project.release_artefacts or project.primary_release == 'base' -%} +# base: OTHER sources of interest, such as definitions owl +$(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) + $(ROBOT_RELEASE_IMPORT_MODE) \ + reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \ + relax \ + reduce -r {{ project.reasoner }} \ + remove {% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{iri}} {% endfor %}{% else %}--base-iri $(URIBASE)/{{ id.upper() }} {% endif %}--axioms external --preserve-structure false --trim false \ + $(SHARED_ROBOT_COMMANDS) \ + annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \ + --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} --annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ +{% endif -%} + +{% if 'editors' in project.release_artefacts or project.primary_release == 'editors' -%} # base: OTHER sources of interest, such as definitions owl -$(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) +$(ONT)-editors.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(ROBOT) remove --input $< --select imports --trim false \ - {% if project.use_dosdps or project.components is defined %}merge $(patsubst %, -i %, $(OTHER_SRC)) \ - {% endif %} $(SHARED_ROBOT_COMMANDS) annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \ + $(ROBOT_RELEASE_IMPORT_MODE) \ + $(SHARED_ROBOT_COMMANDS) annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \ --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ {% if project.release_date -%} --annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ +{% endif -%} +{% if 'full' in project.release_artefacts or project.primary_release == 'full' -%} # Full: The full artefacts with imports merged, reasoned $(ONT)-full.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) - $(ROBOT) merge --input $< \ + $(ROBOT_RELEASE_IMPORT_MODE) \ reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \ relax \ reduce -r {{ project.reasoner }} \ $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ +{% endif -%} {% if 'non-classified' in project.release_artefacts or project.primary_release == 'non-classified' -%} # foo-non-classified: (edit->imports-merged) $(ONT)-non-classified.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) - $(ROBOT) merge --input $< \ + $(ROBOT_RELEASE_IMPORT_MODE) \ $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ {% endif -%} @@ -842,7 +865,7 @@ $(ONT)-non-classified.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) # drop every axiom: filter --term-file keep_terms.txt --trim true # remove --select imports --trim false $(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) - $(ROBOT) merge --input $< $(patsubst %, -i %, $(OTHER_SRC)) \ + $(ROBOT_RELEASE_IMPORT_MODE) \ reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \ relax \ remove --axioms equivalent \ @@ -858,8 +881,7 @@ $(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILE # Should this be the non-classified ontology with the drop foreign axiom filter? # Consider adding remove --term "http://www.geneontology.org/formats/oboInOwl#hasOBONamespace" $(ONT)-simple-non-classified.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) - $(ROBOT) remove --input $< --select imports --trim true \ - merge $(patsubst %, -i %, $(OTHER_SRC)) \ + $(ROBOT_RELEASE_IMPORT_MODE) \ remove --axioms equivalent \ reduce -r {{ project.reasoner }} \ filter --select ontology --term-file $(SIMPLESEED) --trim false \ @@ -871,7 +893,7 @@ $(ONT)-simple-non-classified.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED # See above (David comment) for explanation. # removes any axioms that contains one of the ops that not in the whitelist file $(ONT)-basic.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(KEEPRELATIONS) $(IMPORT_FILES) - $(ROBOT) merge --input $< $(patsubst %, -i %, $(OTHER_SRC)) \ + $(ROBOT_RELEASE_IMPORT_MODE) \ reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \ relax \ remove --axioms equivalent \ From c8ebf957d0c272487a4bcbc9682368b350f6f331 Mon Sep 17 00:00:00 2001 From: Nico Matentzoglu Date: Fri, 5 Aug 2022 12:59:24 +0300 Subject: [PATCH 02/10] Some fixes, adding release reasoner optioon --- odk/odk.py | 3 +++ template/src/ontology/Makefile.jinja2 | 32 +++++++++++++-------------- tests/test-non-import-release.yaml | 1 + 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/odk/odk.py b/odk/odk.py index 57e34757..5255d64f 100755 --- a/odk/odk.py +++ b/odk/odk.py @@ -531,6 +531,9 @@ class OntologyProject(JsonSchemaMixin): release_artefacts : List[str] = field(default_factory=lambda: ['full', 'base']) """A list of release artefacts you wish to be exported.""" + release_use_reasoner : bool = True + """If set to True, the reasoner will be used during the release process.""" + export_formats : List[str] = field(default_factory=lambda: ['owl', 'obo']) """A list of export formats you wish your release artefacts to be exported to, such as owl, obo, gz, ttl.""" diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index 06718f55..050ad6f2 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -822,10 +822,10 @@ merge $(patsubst %, -i %, $(IMPORT_FILES)){% endif %}{% endif %} {% if 'base' in project.release_artefacts or project.primary_release == 'base' -%} # base: OTHER sources of interest, such as definitions owl $(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE) \ - reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \ - relax \ - reduce -r {{ project.reasoner }} \ + $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} + reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \{% endif %} + relax \{% if project.release_use_reasoner %} + reduce -r {{ project.reasoner }} \{% endif %} remove {% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{iri}} {% endfor %}{% else %}--base-iri $(URIBASE)/{{ id.upper() }} {% endif %}--axioms external --preserve-structure false --trim false \ $(SHARED_ROBOT_COMMANDS) \ annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \ @@ -846,10 +846,10 @@ $(ONT)-editors.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) {% if 'full' in project.release_artefacts or project.primary_release == 'full' -%} # Full: The full artefacts with imports merged, reasoned $(ONT)-full.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE) \ - reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \ - relax \ - reduce -r {{ project.reasoner }} \ + $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} + reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \{% endif %} + relax \{% if project.release_use_reasoner %} + reduce -r {{ project.reasoner }} \{% endif %} $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ {% endif -%} @@ -865,13 +865,13 @@ $(ONT)-non-classified.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) # drop every axiom: filter --term-file keep_terms.txt --trim true # remove --select imports --trim false $(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE) \ - reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \ + $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} + reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \{% endif %} relax \ remove --axioms equivalent \ relax \ - filter --term-file $(SIMPLESEED) --select "annotations ontology anonymous self" --trim true --signature true \ - reduce -r {{ project.reasoner }} \ + filter --term-file $(SIMPLESEED) --select "annotations ontology anonymous self" --trim true --signature true \{% if project.release_use_reasoner %} + reduce -r {{ project.reasoner }} \{% endif %} query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru \ $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ {% endif -%} @@ -882,8 +882,8 @@ $(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILE # Consider adding remove --term "http://www.geneontology.org/formats/oboInOwl#hasOBONamespace" $(ONT)-simple-non-classified.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) $(ROBOT_RELEASE_IMPORT_MODE) \ - remove --axioms equivalent \ - reduce -r {{ project.reasoner }} \ + remove --axioms equivalent \{% if project.release_use_reasoner %} + reduce -r {{ project.reasoner }} \{% endif %} filter --select ontology --term-file $(SIMPLESEED) --trim false \ $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ {% endif -%} @@ -893,8 +893,8 @@ $(ONT)-simple-non-classified.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED # See above (David comment) for explanation. # removes any axioms that contains one of the ops that not in the whitelist file $(ONT)-basic.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(KEEPRELATIONS) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE) \ - reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \ + $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} + reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \{% endif %} relax \ remove --axioms equivalent \ remove --axioms disjoint \ diff --git a/tests/test-non-import-release.yaml b/tests/test-non-import-release.yaml index 47bea8c6..cacf28ab 100644 --- a/tests/test-non-import-release.yaml +++ b/tests/test-non-import-release.yaml @@ -9,6 +9,7 @@ release_artefacts: - editors - base primary_release: base +release_use_reasoner: FALSE import_group: release_imports: TRUE products: From ae107c9eca25651be0c7e46499e16594222ae265 Mon Sep 17 00:00:00 2001 From: Nico Matentzoglu Date: Fri, 5 Aug 2022 18:05:22 +0300 Subject: [PATCH 03/10] Update odk.py --- odk/odk.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/odk/odk.py b/odk/odk.py index e3f09985..5255d64f 100755 --- a/odk/odk.py +++ b/odk/odk.py @@ -539,7 +539,11 @@ class OntologyProject(JsonSchemaMixin): namespaces : Optional[List[str]] = None """A list of namespaces that are considered at home in this ontology. Used for certain filter commands.""" - + + use_edit_file_imports : bool = True + """If True, ODK will release the ontology with imports explicitly specified by owl:imports in the edit file. + If False, ODK will build and release the ontology with _all_ imports and _all_ components specified in the ODK config file.""" + dosdp_tools_options: str = "--obo-prefixes=true" """default parameters for dosdp-tools""" @@ -586,7 +590,7 @@ class OntologyProject(JsonSchemaMixin): pattern_pipelines_group : Optional[PatternPipelineGroup] = None """Block that includes information on all ontology imports to be generated""" - robotemplate_group : Optional[RobotTemplateGroup] = None + robot_template_group : Optional[RobotTemplateGroup] = None """Block that includes information on all ROBOT templates used""" def fill_missing(self): From 9c57f64328f5e40c245e79f8cfcdf80d2844cf27 Mon Sep 17 00:00:00 2001 From: Nico Matentzoglu Date: Fri, 5 Aug 2022 18:48:01 +0300 Subject: [PATCH 04/10] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f462a81..4791b359 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ - Adding new `make reason_test` command ([pull](https://github.com/INCATools/ontology-development-kit/pull/639), [issue](https://github.com/INCATools/ontology-development-kit/issues/645)) - Adding intermediate artefact `$(TMPDIR)/$(ONT)-preprocess.owl` to the release workflow which enable the centralisation of preprocessing in a single make step. Basically, rather than creating release artefacts directly from the editors file (e.g. bfo-edit.owl), we add intermediate step from which all releases are derived. This intermediate can then be customised by the user ([pull](https://github.com/INCATools/ontology-development-kit/pull/639), [issue](https://github.com/INCATools/ontology-development-kit/issues/544)) +- Re-defines the base from being "all axioms as edited by the ontology editors" to "all axioms and inferences as specified by the ontology editors". The new goal is much more rigorous in removing axioms from other ontologies as well. The old base file can now be exported as an "editors release" instead ([pull](https://github.com/INCATools/ontology-development-kit/pull/643), [issue](https://github.com/INCATools/ontology-development-kit/issues/646)). +- Introduces the option of skipping the use of reasoner during the release process (important for application ontologies), ([pull](https://github.com/INCATools/ontology-development-kit/pull/643), [issue](https://github.com/INCATools/ontology-development-kit/issues/644)) +- Introduces a new mode that enable the use of ODK entirely without owl:imports in the edit file (this is great in case we want to use the ODK workflows but not check in any files, imports or components, into version control, like huge application ontologies), ([pull](https://github.com/INCATools/ontology-development-kit/pull/643), [issue](https://github.com/INCATools/ontology-development-kit/issues/629)). # v1.3.1 From 19baa436965a3b6c2478200bc917a634b48a23ed Mon Sep 17 00:00:00 2001 From: Nico Matentzoglu Date: Mon, 8 Aug 2022 12:46:23 +0300 Subject: [PATCH 05/10] Update Makefile.jinja2 --- template/src/ontology/Makefile.jinja2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index 050ad6f2..36daf5f7 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -826,7 +826,7 @@ $(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \{% endif %} relax \{% if project.release_use_reasoner %} reduce -r {{ project.reasoner }} \{% endif %} - remove {% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{iri}} {% endfor %}{% else %}--base-iri $(URIBASE)/{{ id.upper() }} {% endif %}--axioms external --preserve-structure false --trim false \ + remove {% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{iri}} {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }} {% endif %}--axioms external --preserve-structure false --trim false \ $(SHARED_ROBOT_COMMANDS) \ annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \ --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ From e00bfb7013c0296da7264d6344fcba3357467ee7 Mon Sep 17 00:00:00 2001 From: Nico Matentzoglu Date: Sat, 13 Aug 2022 21:54:09 +0300 Subject: [PATCH 06/10] Update Makefile.jinja2 --- template/src/ontology/Makefile.jinja2 | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index 36daf5f7..96d835b3 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -819,10 +819,14 @@ $(ROBOT) remove --input $< --select imports --trim false {% if project.use_dosdp merge $(patsubst %, -i %, $(OTHER_SRC)) {% endif %}{% if project.import_group is defined -%} merge $(patsubst %, -i %, $(IMPORT_FILES)){% endif %}{% endif %} +ROBOT_RELEASE_IMPORT_MODE_BASE={% if project.use_edit_file_imports -%}$(ROBOT) merge --input $< {% else -%} +$(ROBOT) remove --input $< --select imports --trim false {% if project.use_dosdps or project.components is defined -%} +merge $(patsubst %, -i %, $(OTHER_SRC)) {% endif %}{% endif %} + {% if 'base' in project.release_artefacts or project.primary_release == 'base' -%} # base: OTHER sources of interest, such as definitions owl $(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} + $(ROBOT_RELEASE_IMPORT_MODE_BASE) \{% if project.release_use_reasoner %} reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \{% endif %} relax \{% if project.release_use_reasoner %} reduce -r {{ project.reasoner }} \{% endif %} @@ -836,8 +840,7 @@ $(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) {% if 'editors' in project.release_artefacts or project.primary_release == 'editors' -%} # base: OTHER sources of interest, such as definitions owl $(ONT)-editors.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) - $(ROBOT) remove --input $< --select imports --trim false \ - $(ROBOT_RELEASE_IMPORT_MODE) \ + $(ROBOT_RELEASE_IMPORT_MODE_BASE) \ $(SHARED_ROBOT_COMMANDS) annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \ --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ {% if project.release_date -%} --annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ From ae050ab809b5affa2349f318216c7c8e21bb4eb2 Mon Sep 17 00:00:00 2001 From: Nico Matentzoglu Date: Sun, 14 Aug 2022 00:56:01 +0300 Subject: [PATCH 07/10] Update Makefile.jinja2 --- template/src/ontology/Makefile.jinja2 | 41 +-------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index 96d835b3..37646b7f 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -69,7 +69,7 @@ PATTERN_RELEASE_FILES= $(PATTERNDIR)/definitions.owl $(PATTERNDIR)/pattern FORMATS = $(sort {% for format in project.export_formats %} {{ format }}{% endfor %} owl) FORMATS_INCL_TSV = $(sort $(FORMATS) tsv) -RELEASE_ARTEFACTS = $(sort {% for release in project.release_artefacts %}{% if release.startswith('custom-') %}{{ release | replace("custom-","")}}{% else %}$(ONT)-{{ release }}{% endif %} {% endfor %}$(ONT)-base $(ONT)-full) +RELEASE_ARTEFACTS = $(sort {% for release in project.release_artefacts %}{% if release.startswith('custom-') %}{{ release | replace("custom-","")}}{% else %}$(ONT)-{{ release }}{% endif %} {% endfor %}) # ---------------------------------------- # Top-level targets @@ -728,45 +728,6 @@ $(PATTERNDIR)/definitions.owl: $(DOSDP_OWL_FILES_DEFAULT) {% if project.pattern_ {% endif -%} {% endfor -%} -{% if 'base' not in project.release_artefacts -%} -# We always want a base - even if it is not explicitly configured.. -{% if 'obo' in project.export_formats -%} -$(ONT)-base.obo: $(ONT)-base.owl - $(ROBOT) convert --input $< --check false -f obo $(OBO_FORMAT_OPTIONS) -o $@.tmp.obo && grep -v ^owl-axioms $@.tmp.obo > $@ && rm $@.tmp.obo -{% endif -%} -{% if 'ttl' in project.export_formats -%} -$(ONT)-base.ttl: $(ONT)-base.owl - $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false -f ttl -o $@.tmp.ttl && mv $@.tmp.ttl $@ -{% endif -%} -{% if 'json' in project.export_formats -%} -$(ONT)-base.json: $(ONT)-base.owl - $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false -f json -o $@.tmp.json &&\ - jq -S 'walk(if type == "array" then sort else . end)' $@.tmp.json > $@ && rm $@.tmp.json -{% endif -%} -{% endif -%} - - -{% if 'full' not in project.release_artefacts -%} -# We always want a full release - even if it is not explicitly configured.. -{% if 'obo' in project.export_formats -%} -$(ONT)-full.obo: $(ONT)-full.owl - $(ROBOT) convert --input $< --check false -f obo $(OBO_FORMAT_OPTIONS) -o $@.tmp.obo && grep -v ^owl-axioms $@.tmp.obo > $@ && rm $@.tmp.obo -{% endif -%} -{% if 'ttl' in project.export_formats -%} -$(ONT)-full.ttl: $(ONT)-full.owl - $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false -f ttl -o $@.tmp.ttl && mv $@.tmp.ttl $@ -{% endif -%} -{% if 'json' in project.export_formats -%} -$(ONT)-full.json: $(ONT)-full.owl - $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false -f json -o $@.tmp.json &&\ - jq -S 'walk(if type == "array" then sort else . end)' $@.tmp.json > $@ && rm $@.tmp.json -{% endif -%} -{% endif -%} - # ---------------------------------------- # Release artefacts: main release artefacts # ---------------------------------------- From 4348b648afb46d893ca0e78e3808e44334e51f7e Mon Sep 17 00:00:00 2001 From: Nico Matentzoglu Date: Sun, 4 Dec 2022 12:45:02 +0200 Subject: [PATCH 08/10] Update Makefile.jinja2 --- template/src/ontology/Makefile.jinja2 | 36 ++++++++++++++++----------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index e04c9970..35a7b4c4 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -784,19 +784,28 @@ $(ONT).owl: $(ONT)-{{ project.primary_release }}.owl cp $< $@ {% endif -%} +# ROBOT pipeline that merges imports, including components. ROBOT_RELEASE_IMPORT_MODE={% if project.use_edit_file_imports -%}$(ROBOT) merge --input $< {% else -%} $(ROBOT) remove --input $< --select imports --trim false {% if project.use_dosdps or project.components is defined -%} merge $(patsubst %, -i %, $(OTHER_SRC)) {% endif %}{% if project.import_group is defined -%} merge $(patsubst %, -i %, $(IMPORT_FILES)){% endif %}{% endif %} -ROBOT_RELEASE_IMPORT_MODE_BASE={% if project.use_edit_file_imports -%}$(ROBOT) merge --input $< {% else -%} -$(ROBOT) remove --input $< --select imports --trim false {% if project.use_dosdps or project.components is defined -%} -merge $(patsubst %, -i %, $(OTHER_SRC)) {% endif %}{% endif %} +# ROBOT pipeline that removes imports, then merges components. This is for release artefacts that start from "base" +ROBOT_RELEASE_IMPORT_MODE_BASE=$(ROBOT) remove --input $< --select imports --trim false {% if project.use_dosdps or project.components is defined -%} +merge $(patsubst %, -i %, $(OTHER_SRC)) {% endif %} {% if 'base' in project.release_artefacts or project.primary_release == 'base' -%} -# base: OTHER sources of interest, such as definitions owl -$(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE_BASE) \{% if project.release_use_reasoner %} +# base: All the axioms as they are editted by the editors, excluding reasoning +$(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) + $(ROBOT_RELEASE_IMPORT_MODE_BASE) \ + $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} --annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ +{% endif -%} + +{% if 'baseplus' in project.release_artefacts or project.primary_release == 'baseplus' -%} +# baseplus: A highly experimental variant of base, UNSTABLE. Only to be used experimentally (may disappear in future ODK releases). +$(ONT)-baseplus.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) + $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \{% endif %} relax \{% if project.release_use_reasoner %} reduce -r {{ project.reasoner }} \{% endif %} @@ -808,21 +817,20 @@ $(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) {% endif -%} {% if 'editors' in project.release_artefacts or project.primary_release == 'editors' -%} -# base: OTHER sources of interest, such as definitions owl +# editors: All the axioms as they are editted by the editors, excluding reasoning. This is currently the same as "base". Only to be used experimentally (may disappear in future ODK releases). $(ONT)-editors.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(ROBOT_RELEASE_IMPORT_MODE_BASE) \ - $(SHARED_ROBOT_COMMANDS) annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \ - --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ + $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ {% if project.release_date -%} --annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ {% endif -%} {% if 'full' in project.release_artefacts or project.primary_release == 'full' -%} -# Full: The full artefacts with imports merged, reasoned +# Full: The full artefacts with imports merged, reasoned. $(ONT)-full.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} - reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \{% endif %} - relax \{% if project.release_use_reasoner %} - reduce -r {{ project.reasoner }} \{% endif %} + $(ROBOT_RELEASE_IMPORT_MODE) \ + reason --reasoner {{ project.reasoner }} --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \ + relax \ + reduce -r {{ project.reasoner }} \ $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ {% endif -%} From 0aa34f30e8fff096d9fbfe223edfdeb351aa7643 Mon Sep 17 00:00:00 2001 From: Nico Matentzoglu Date: Sun, 4 Dec 2022 13:01:50 +0200 Subject: [PATCH 09/10] Update Makefile.jinja2 --- template/src/ontology/Makefile.jinja2 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index 35a7b4c4..9b202dd8 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -798,8 +798,10 @@ merge $(patsubst %, -i %, $(OTHER_SRC)) {% endif %} # base: All the axioms as they are editted by the editors, excluding reasoning $(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(ROBOT_RELEASE_IMPORT_MODE_BASE) \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - {% if project.release_date -%} --annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ + $(SHARED_ROBOT_COMMANDS) \ + annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \ + --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} --annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ {% endif -%} {% if 'baseplus' in project.release_artefacts or project.primary_release == 'baseplus' -%} @@ -862,7 +864,7 @@ $(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILE # Should this be the non-classified ontology with the drop foreign axiom filter? # Consider adding remove --term "http://www.geneontology.org/formats/oboInOwl#hasOBONamespace" $(ONT)-simple-non-classified.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE) \ + $(ROBOT_RELEASE_IMPORT_MODE_BASE) \ remove --axioms equivalent \{% if project.release_use_reasoner %} reduce -r {{ project.reasoner }} \{% endif %} filter --select ontology --term-file $(SIMPLESEED) --trim false \ From b57ded24ceb291c252b5ca3a16b595ee8659886d Mon Sep 17 00:00:00 2001 From: Nico Matentzoglu Date: Sun, 4 Dec 2022 13:04:42 +0200 Subject: [PATCH 10/10] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 524c50ca..c534de8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - Adding new `make reason_test` command ([pull](https://github.com/INCATools/ontology-development-kit/pull/639), [issue](https://github.com/INCATools/ontology-development-kit/issues/645)) - Adding intermediate artefact `$(TMPDIR)/$(ONT)-preprocess.owl` to the release workflow which enable the centralisation of preprocessing in a single make step. Basically, rather than creating release artefacts directly from the editors file (e.g. bfo-edit.owl), we add intermediate step from which all releases are derived. This intermediate can then be customised by the user ([pull](https://github.com/INCATools/ontology-development-kit/pull/639), [issue](https://github.com/INCATools/ontology-development-kit/issues/544)) -- Re-defines the base from being "all axioms as edited by the ontology editors" to "all axioms and inferences as specified by the ontology editors". The new goal is much more rigorous in removing axioms from other ontologies as well. The old base file can now be exported as an "editors release" instead ([pull](https://github.com/INCATools/ontology-development-kit/pull/643), [issue](https://github.com/INCATools/ontology-development-kit/issues/646)). +- Introduces an experimental new release called "base-plus", which includes the inferred and non-redundant classification of the ontology. This is highly experimental and may be removed in a future release of ODK. The new goal is much more rigorous in removing axioms from other ontologies as well. The old base file can now be exported as an "editors release" instead ([pull](https://github.com/INCATools/ontology-development-kit/pull/643), [issue](https://github.com/INCATools/ontology-development-kit/issues/646)). - Introduces the option of skipping the use of reasoner during the release process (important for application ontologies), ([pull](https://github.com/INCATools/ontology-development-kit/pull/643), [issue](https://github.com/INCATools/ontology-development-kit/issues/644)) - Introduces a new mode that enable the use of ODK entirely without owl:imports in the edit file (this is great in case we want to use the ODK workflows but not check in any files, imports or components, into version control, like huge application ontologies), ([pull](https://github.com/INCATools/ontology-development-kit/pull/643), [issue](https://github.com/INCATools/ontology-development-kit/issues/629)). - Changes in the `src/ontology/run.sh` wrapper script: