Permalink
Browse files

Rework YAML schema to keep config project-specific

- add YAML fields: id, base_redirect, products, term_browser,
  example_terms
- update config.schema.yml
- move translate.py to translate-entries.py
- add translate-base-redirect.py to redirecting base_url
- add translate-products.py for /obo/foo.owl files
- add translate-terms.py for /obo/FOO_12345 terms
- update tests to handle new behaviour
- update Makefile
- update example files
- update config for OBI and OBO
- update migrate.py to help with new YAML
- update README
  • Loading branch information...
1 parent bd6ad55 commit 9ecba9a420939c439ac86a5000f18c34fad79189 @jamesaoverton jamesaoverton committed Nov 19, 2015
View
@@ -3,6 +3,7 @@
.travis_build
junk/
migrations/
+temp/
tests/
tools/.vagrant/
tools/hosts
View
@@ -46,8 +46,8 @@ PRODUCTION ?= purl.obolibrary.org
MAKEFLAGS += --warn-undefined-variables
SHELL := bash
.SHELLFLAGS := -eu -o pipefail -c
-.DEFAULT_GOAL := all
.DELETE_ON_ERROR:
+.DEFAULT_GOAL := all
.SUFFIXES:
@@ -60,7 +60,7 @@ all: clean validate build
# Remove directories with generated files.
.PHONY: clean
clean:
- rm -rf tests www/obo
+ rm -rf temp tests www/obo
# Generate .htaccess files for all YAML configuration files.
.PHONY: build
@@ -82,14 +82,44 @@ validate:
#
# Convert the YAML configuration files
# to Apache .htaccess files with RedirectMatch directives.
+# There are three types:
+#
+# - base_redirects: when the project's base_url points to something
+# - product: for a project's main OWL file
+# - term: for a project's terms
+# - entries: PURLs under the project's base_url
+#
+# The first three are inserted into www/obo/.htaccess
+# while the last is in the project's www/obo/project/.htaccess
+temp/base_redirects temp/products temp/terms:
+ mkdir -p $@
+
+temp/base_redirects/%.htaccess: config/%.yml temp/base_redirects
+ tools/translate-base-redirects.py $< $@
+
+temp/products/%.htaccess: config/%.yml temp/products
+ tools/translate-products.py $< $@
+
+temp/terms/%.htaccess: config/%.yml temp/terms
+ tools/translate-terms.py $< $@
+
www/obo/%/.htaccess: config/%.yml
mkdir -p www/obo/$*
- tools/translate.py $< $@
+ tools/translate-entries.py $< $@
# Convert all YAML configuration files to .htaccess
# and move the special `obo` .htaccess file.
www/obo: $(foreach o,$(ONTOLOGY_IDS),www/obo/$o/.htaccess)
- mv www/obo/obo/.htaccess www/obo/.htaccess
+www/obo: $(foreach o,$(ONTOLOGY_IDS),temp/base_redirects/$o.htaccess)
+www/obo: $(foreach o,$(ONTOLOGY_IDS),temp/products/$o.htaccess)
+www/obo: $(foreach o,$(ONTOLOGY_IDS),temp/terms/$o.htaccess)
+ cat www/obo/obo/.htaccess > www/obo/.htaccess
+ echo '' >> www/obo/.htaccess
+ echo '### Generated from project configuration files' >> www/obo/.htaccess
+ echo '' >> www/obo/.htaccess
+ cat temp/base_redirects/*.htaccess >> www/obo/.htaccess
+ cat temp/products/*.htaccess >> www/obo/.htaccess
+ cat temp/terms/*.htaccess >> www/obo/.htaccess
rm -rf www/obo/obo
@@ -140,15 +170,31 @@ tests/examples:
mkdir -p $@
tests/examples/%.yml: tools/examples/%.xml tools/examples/%.yml tests/examples
- tools/migrate.py /obo/$* $< $@
+ tools/migrate.py $* $< $@
diff tools/examples/$*.yml $@
-tests/examples/%.htaccess: tools/examples/%.yml tools/examples/%.htaccess tests/examples
- tools/translate.py $< $@
+tests/examples/%.base_redirects.htaccess: tools/examples/%.yml tests/examples
+ tools/translate-base-redirects.py $< $@
+ diff tools/examples/$*.base_redirects.htaccess $@
+
+tests/examples/%.products.htaccess: tools/examples/%.yml tests/examples
+ tools/translate-products.py $< $@
+ diff tools/examples/$*.products.htaccess $@
+
+tests/examples/%.terms.htaccess: tools/examples/%.yml tests/examples
+ tools/translate-terms.py $< $@
+ diff tools/examples/$*.terms.htaccess $@
+
+tests/examples/%.htaccess: tools/examples/%.yml tests/examples
+ tools/translate-entries.py $< $@
diff tools/examples/$*.htaccess $@
.PHONY: test-examples
-test-examples: tests/examples/test1.yml tests/examples/test1.htaccess tests/examples/test2.htaccess
+test-examples: tests/examples/test1.yml
+test-examples: tests/examples/test2.htaccess
+test-examples: tests/examples/test2.base_redirects.htaccess
+test-examples: tests/examples/test2.products.htaccess
+test-examples: tests/examples/test2.terms.htaccess
### Update Repository
@@ -186,6 +232,6 @@ migrate-%:
|| (echo 'Refusing to overwrite config/$*.yml'; exit 1)
mkdir -p migrations
test -s migrations/$*.xml \
- || curl --fail -o migrations/$*.xml "$(PURL_XML)/obo/$*/*"
+ || curl --fail -o migrations/$*.xml "$(PURL_XML)/obo/$**"
mkdir -p config
- tools/migrate.py /obo/$* migrations/$*.xml config/$*.yml
+ tools/migrate.py $* migrations/$*.xml config/$*.yml
View
@@ -33,9 +33,39 @@ Please use one of these four options to make changes to the PURLs:
Each OBO project using this service gets a [YAML](http://yaml.org) configuration file in `config/`. That YAML configuration file is used to generate an Apache `.htaccess` file for that ontology. That Apache configuration will apply to all PURLs for that project.
-For example, PURLs for the OBI project are configured in [`config/obi.yml`](config/obi.yml), which is translated to [`obo/obi/.htaccess`](obo/obi/.htaccess), and which controls all PURLs that begin with `http://purl.obolibrary.org/obo/obi/`.
+Every YAML configuration file must have these fields:
-The `.htaccess` file should not be edited! Changes should only be made in the YAML configuration file.
+- `id:` the project's ID, usually uppercase
+- `base_url:` the part of a PURL that comes after the domain, usually lowercase
+- `term_browser:` usually [`ontobee`](http://ontobee.org) but can be `custom` (see below)
+- `products:` a list of primary files for the ontology and the URLs to redirect them to; an `.owl` file is required, and an `.obo` file is optional
+
+Optional fields include:
+
+- `example_terms:` a list of one or more term IDs for automated testing
+- `base_redirect:` If your project redirects its `base_url`, then you will need a `base_redirect:` entry. So `base_redirect: http://obi-ontology.org` will redirect <http://purl.obolibrary.org/obo/obi> to <http://obi-ontology.org>.
+- `entries:` a list of other PURLs under the `base_url`, see below
+
+Here's an example adapted from the [`config/obi.yml`](config/obi.yml) file:
+
+ id: OBI
+ base_url: /obo/obi
+
+ products:
+ - obi.owl: http://svn.code.sf.net/p/obi/code/releases/2015-09-15/obi.owl
+
+ term_browser: ontobee
+ example_terms:
+ - OBI_0000070
+
+ entries:
+ - exact: /wiki
+ replacement: http://obi-ontology.org
+
+Most of these fields are straightforward, but the `entries:` need some more explanation.
+
+
+### Entries
Each YAML configuration file contains the keyword `entries:` followed by a list of entries. Each entry defines an Apache [RedirectMatch](https://httpd.apache.org/docs/2.4/mod/mod_alias.html#redirectmatch) directive for matching URLs and redirecting to new URLs. Every entry begins with a `- `, followed by keywords and values on indented lines. There are three types of entries:
@@ -48,7 +78,7 @@ The `#` character indicates a comment, which is not considered part of the confi
See the [`tools/examples/test2.yml`](tools/examples/test2.yml) and [`tools/examples/test2.htaccess`](tools/examples/test2.htaccess) for examples.
-### Exact
+#### Exact
In the most common case, your PURL should match a unique URL and redirect to a unique URL. Here's an example from the `config/obi.yml` file:
@@ -62,7 +92,7 @@ Behind the scenes, the entry is translated into an Apache RedirectMatch directiv
RedirectMatch temp "^/2015\-09\-15/obi\.owl$" "http://svn.code.sf.net/p/obi/code/releases/2015-09-15/obi.owl"
-### Prefix
+#### Prefix
You can also match and replace just the first part of a URL, leaving the rest unchanged. This allows you to define one entry that redirects many URLs matching a common prefix. Another example from `config/obi.yml`:
@@ -76,14 +106,14 @@ The translation is similar, with the addition of `(.*)` wildcard and a `$1` "bac
RedirectMatch temp "^/branches/(.*)$" "http://obi.svn.sourceforge.net/svnroot/obi/trunk/src/ontology/branches/$1"
-### Regex
+#### Regex
Regular expression entries should only be needed very rarely, and should always be used very carefully.
For the regular expression type, the value of the `regex:` and `replacement:` keywords should contain regular expressions in exactly the format expected by Apache [RedirectMatch](https://httpd.apache.org/docs/2.4/mod/mod_alias.html#redirectmatch). The values will be quoted, but no other changes will be made to them.
-### Tests
+#### Tests
Every `prefix` or `regex` entry should also have a `tests:` keyword, with a list of additional URLs to check. Each test requires a `from:` value (like `exact:`) and a `to:` value (like `replacement:`). Here's an example:
@@ -94,26 +124,26 @@ Every `prefix` or `regex` entry should also have a `tests:` keyword, with a list
to: http://obi.svn.sourceforge.net/svnroot/obi/trunk/src/ontology/branches/obi.owl
-### Temporary and Permanent
+#### Temporary and Permanent
Any entry can have a `status:` keyword. By default, every entry uses "temporary" (HTTP 302) status. If you *really* know what you're doing, you can set the status to "permanent" (HTTP 301).
-### Order of Entries
+#### Order of Entries
Apache RedirectMatch directives are processed in the [order that they appear](https://httpd.apache.org/docs/2.4/mod/mod_alias.html#order) in the configuration file. Be careful that your `prefix` and `regex` entries do not conflict with your other entries. The YAML-to-Apache translation preserves the order of entries, so you can control the order of processing, but it's best to avoid conflicts.
-## Redirecting Terms
+## Custom Term Browsers
-The [`obo.yml`](config/obo.yml) configuration file is special, and contains (among other things) the entries for redirecting each OBO term to the term browser for its ontology. For example, `http://purl.obolibrary.org/obo/OBI_0000070` is redirected to Ontobee for browsing OBI:
+If your project does not use Ontobee as a term browser, you must specify `term_browser: custom` in your project's YAML configuration file, and provide a `regex` entry in the [`config/obo.yml`](config/obo.yml) configuration file. Here's an example for [ChEBI](https://www.ebi.ac.uk/chebi/):
- # Terms for OBI
- - regex: ^/obo/OBI_(\d+)$
- replacement: http://www.ontobee.org/browser/rdf.php?o=OBI&iri=http://purl.obolibrary.org/obo/OBI_$1
+ # Terms for CHEBI
+ - regex: ^/obo/CHEBI_(\d+)$
+ replacement: http://www.ebi.ac.uk/chebi/searchId.do?chebiId=CHEBI:$1
tests:
- - from: /OBI_0000070
- to: http://www.ontobee.org/browser/rdf.php?o=OBI&iri=http://purl.obolibrary.org/obo/OBI_0000070
+ - from: /CHEBI_15377
+ to: http://www.ebi.ac.uk/chebi/searchId.do?chebiId=CHEBI:15377
Since these are `regex` entries, and could affect multiple projects, we prefer that OBO admins are the only ones to edit `obo.yml`. If you need a change to the term redirect entry for your project, please [create a new issue](https://github.com/OBOFoundry/purl.obolibrary.org/issues/new).
@@ -122,7 +152,7 @@ Since these are `regex` entries, and could affect multiple projects, we prefer t
OBO projects currently use OCLC for managing PURLs. This project aims to replace OCLC in a straightforward way.
-The `Makefile` contains some code for fetching the PURL records for a given ontology ID from OCLC in XML format, and converting the XML to YAML. This should be a one-time migration. Going forward, the YAML configuration should be edited directly.
+The `Makefile` contains some code for fetching the PURL records for a given ontology ID from OCLC in XML format, and converting the XML to YAML. This should be a one-time migration, and it requires some manual editing and checking. Going forward, the YAML configuration should be edited directly.
The order of the migrated entries is: `exact` first (*should* be in the order they were created), followed by `prefix` entries from longest `prefix` to shortest. This order avoids nasty conflicts and has been tested to preserve the OCLC behaviour.
View
@@ -1,7 +1,16 @@
# PURL configuration for http://purl.obolibrary.org/obo/obi
+id: OBI
base_url: /obo/obi
+products:
+- obi.owl: http://svn.code.sf.net/p/obi/code/releases/2015-09-15/obi.owl
+- obi.obo: http://www.berkeleybop.org/ontologies/obi.obo
+
+term_browser: ontobee
+example_terms:
+- OBI_0000070
+
entries:
- exact: /project
replacement: http://sw.neurocommons.org/obi-303/project.rdf
View
@@ -1,13 +1,51 @@
# PURL configuration for http://purl.obolibrary.org/obo/
+id: OBO
base_url: /obo
+term_browser: custom
+
entries:
+### Term Overrides
+
+- exact: /BFO_0000050
+ replacement: http://www.ontobee.org/browser/rdf.php?o=RO&iri=http://purl.obolibrary.org/obo/BFO_0000050
+
+- exact: /BFO_0000051
+ replacement: http://www.ontobee.org/browser/rdf.php?o=RO&iri=http://purl.obolibrary.org/obo/BFO_0000051
+
+- exact: /BFO_0000066
+ replacement: http://www.ontobee.org/browser/rdf.php?o=RO&iri=http://purl.obolibrary.org/obo/BFO_0000066
+
+- exact: /BFO_0000062
+ replacement: http://www.ontobee.org/browser/rdf.php?o=RO&iri=http://purl.obolibrary.org/obo/BFO_0000062
+
+### Custom (non-Ontobee) term_browser redirects
+
+# Terms for CHEBI
+- regex: ^/obo/CHEBI_(\d+)$
+ replacement: http://www.ebi.ac.uk/chebi/searchId.do?chebiId=CHEBI:$1
+ tests:
+ - from: /CHEBI_15377
+ to: http://www.ebi.ac.uk/chebi/searchId.do?chebiId=CHEBI:15377
+
+# Terms for HAO
+- regex: ^/obo/HAO_(\d+)$
+ replacement: http://api.hymao.org/api/ontology/ontology_class/HAO_$1
+ tests:
+ - from: /HAO_0000000
+ to: http://api.hymao.org/api/ontology/ontology_class/HAO_0000000
+
+# Terms for PR and PRO
+- regex: ^/obo/PR_(\d+)$
+ replacement: http://pir.georgetown.edu/cgi-bin/pro/entry_pro?id=PR_$1
+ tests:
+ - from: /PR_000000001
+ to: http://pir.georgetown.edu/cgi-bin/pro/entry_pro?id=PR_000000001
-# Redirect term identifiers
-- regex: ^/obo/OBI_(\d+)$
- replacement: http://www.ontobee.org/browser/rdf.php?o=OBI&iri=http://purl.obolibrary.org/obo/OBI_$1
+- regex: ^/obo/PRO_(\d+)$
+ replacement: http://pir.georgetown.edu/cgi-bin/pro/entry_pro?id=PRO_$1
tests:
- - from: /OBI_0000070
- to: http://www.ontobee.org/browser/rdf.php?o=OBI&iri=http://purl.obolibrary.org/obo/OBI_0000070
+ - from: /PRO_000000001
+ to: http://pir.georgetown.edu/cgi-bin/pro/entry_pro?id=PRO_000000001
@@ -6,13 +6,33 @@
####
type: map
mapping:
+ "id":
+ type: str
+ required: true
"base_url":
type: str
pattern: /^\/obo/
required: true
- "entries":
+ "base_redirect":
+ type: str
+ required: false
+ "products":
type: seq
+ required: false
+ sequence:
+ - type: any
+ "term_browser":
+ type: str
+ pattern: /^(ontobee|custom)$/
required: true
+ "example_terms":
+ type: seq
+ required: false
+ sequence:
+ - type: str
+ "entries":
+ type: seq
+ required: false
sequence:
- type: map
mapping:
@@ -1,7 +1,16 @@
# PURL configuration for http://purl.obolibrary.org/obo/test1
+id: TEST1
base_url: /obo/test1
+products:
+- test1.owl: TODO
+- test1.obo: TODO
+
+term_browser: ontobee
+example_terms:
+- TODO
+
entries:
- exact: /project
replacement: http://example.org/project.html
@@ -0,0 +1,3 @@
+# Base redirect for TEST2
+RedirectMatch temp "^/obo/test2$" "http://example.org/test2"
+
@@ -0,0 +1,4 @@
+# Products for TEST2
+RedirectMatch temp "^/obo/test2.owl$" "http://example.org/test2.owl"
+RedirectMatch temp "^/obo/test2.obo$" "http://example.org/test2.obo"
+
@@ -0,0 +1,3 @@
+# Term redirect for TEST2
+RedirectMatch temp "^/obo/TEST2_(\d+)$" "http://www.ontobee.org/browser/rdf.php?o=TEST2&iri=http://purl.obolibrary.org/obo/TEST2_$1"
+
@@ -1,7 +1,18 @@
# PURL configuration for http://purl.obolibrary.org/obo/test2
+id: TEST2
base_url: /obo/test2
+base_redirect: http://example.org/test2
+
+products:
+- test2.owl: http://example.org/test2.owl
+- test2.obo: http://example.org/test2.obo
+
+term_browser: ontobee
+example_terms:
+- TEST2_0000001
+
entries:
- exact: /project
replacement: http://example.org/project.html
Oops, something went wrong.

0 comments on commit 9ecba9a

Please sign in to comment.